code icon Code

Fetch Workspace Messages

Fetch messages from all accessible public channels for sentiment analysis

Source Code

import fs from "fs";
import path from "path";

const [outputPath, daysBackArg = "14"] = process.argv.slice(2);
const daysBack = Math.min(parseInt(daysBackArg) || 14, 30);

if (!outputPath) {
  console.error("Usage: sentiment.slack.fetch <outputPath> [daysBack]");
  console.error("Output path is required.");
  process.exit(1);
}

const now = Date.now();
const msPerDay = 24 * 60 * 60 * 1000;
const thisWeekStart = Math.floor((now - 7 * msPerDay) / 1000);
const lastWeekStart = Math.floor((now - daysBack * msPerDay) / 1000);

const formatDate = (d) =>
  d.toLocaleDateString("en-US", { month: "short", day: "numeric" });

console.log(`Fetching workspace messages for sentiment analysis...`);
console.log(`  This week: ${formatDate(new Date(thisWeekStart * 1000))} - ${formatDate(new Date())}`);
console.log(`  Last week: ${formatDate(new Date(lastWeekStart * 1000))} - ${formatDate(new Date(thisWeekStart * 1000))}`);

try {
  // 1. Get auth, channels, and users in parallel
  console.log("Fetching workspace data...");

  const [authData, channelsData, usersData] = await Promise.all([
    (async () => {
      const res = await fetch("https://slack.com/api/auth.test", {
        headers: { Authorization: "Bearer PLACEHOLDER_TOKEN" },
      });
      const data = await res.json();
      if (!data.ok) throw new Error(`Auth failed: ${data.error}`);
      return data;
    })(),

    (async () => {
      const channels = [];
      let cursor = null;
      do {
        const params = new URLSearchParams({
          types: "public_channel",
          limit: "200",
          exclude_archived: "true",
        });
        if (cursor) params.set("cursor", cursor);
        const res = await fetch(
          "https://slack.com/api/conversations.list?" + params,
          { headers: { Authorization: "Bearer PLACEHOLDER_TOKEN" } }
        );
        const data = await res.json();
        if (!data.ok) throw new Error(`conversations.list failed: ${data.error}`);
        channels.push(...(data.channels || []));
        cursor = data.response_metadata?.next_cursor;
      } while (cursor);
      return channels;
    })(),

    (async () => {
      const users = [];
      let cursor = null;
      do {
        const params = new URLSearchParams({ limit: "200" });
        if (cursor) params.set("cursor", cursor);
        const res = await fetch("https://slack.com/api/users.list?" + params, {
          headers: { Authorization: "Bearer PLACEHOLDER_TOKEN" },
        });
        const data = await res.json();
        if (!data.ok) throw new Error(`users.list failed: ${data.error}`);
        users.push(...(data.members || []));
        cursor = data.response_metadata?.next_cursor;
      } while (cursor);
      return users;
    })(),
  ]);

  const workspace = authData.team;
  console.log(`✓ Connected to ${workspace}`);

  // Build user lookup
  const userMap = new Map();
  for (const u of usersData) {
    if (!u.is_bot && !u.deleted) {
      userMap.set(u.id, {
        name: u.profile?.display_name || u.real_name || u.name,
        realName: u.real_name,
      });
    }
  }

  // Get channels user is member of
  const myChannels = channelsData.filter((ch) => ch.is_member);
  console.log(`  ${myChannels.length} public channels, ${userMap.size} users`);

  if (myChannels.length === 0) {
    console.error("\n✗ No accessible channels found.");
    console.log(JSON.stringify({
      success: false,
      error: "no_channels",
      workspace,
    }));
    process.exit(1);
  }

  // 2. Fetch history from channels (for the full time range)
  console.log("Fetching channel message history...");

  const fetchHistory = async (channelId) => {
    const allMessages = [];
    let cursor = null;

    do {
      const params = new URLSearchParams({
        channel: channelId,
        limit: "200",
        oldest: lastWeekStart.toString(),
      });
      if (cursor) params.set("cursor", cursor);

      const res = await fetch(
        "https://slack.com/api/conversations.history?" + params,
        { headers: { Authorization: "Bearer PLACEHOLDER_TOKEN" } }
      );
      const data = await res.json();

      if (!data.ok) {
        // Some channels may fail due to permissions - skip them
        return [];
      }

      allMessages.push(...(data.messages || []));
      cursor = data.response_metadata?.next_cursor;

      // Limit to reasonable amount per channel
      if (allMessages.length >= 500) break;
    } while (cursor);

    return allMessages;
  };

  // Fetch channel histories in parallel (batch to avoid rate limits)
  const histories = [];
  for (let i = 0; i < myChannels.length; i += 15) {
    const batch = myChannels.slice(i, i + 15);
    const results = await Promise.all(
      batch.map(async (conv) => ({
        channel: { id: conv.id, name: conv.name },
        messages: await fetchHistory(conv.id),
      }))
    );
    histories.push(...results);

    if (i + 15 < myChannels.length) {
      // Brief pause between batches
      await new Promise(r => setTimeout(r, 500));
    }
  }

  // 3. Process messages into this week / last week buckets
  const thisWeekMessages = [];
  const lastWeekMessages = [];
  const channelStats = [];

  for (const { channel, messages } of histories) {
    if (messages.length === 0) continue;

    let thisWeekCount = 0;
    let lastWeekCount = 0;

    for (const msg of messages) {
      // Skip bot messages and join/leave messages
      if (msg.subtype || !msg.user || !userMap.has(msg.user)) continue;

      const ts = parseFloat(msg.ts);
      const user = userMap.get(msg.user);
      const timestamp = new Date(ts * 1000).toISOString();
      const hour = new Date(ts * 1000).getHours();
      const dayOfWeek = new Date(ts * 1000).getDay(); // 0 = Sunday

      const processed = {
        text: msg.text,
        timestamp,
        hour,
        isWeekend: dayOfWeek === 0 || dayOfWeek === 6,
        isAfterHours: hour < 8 || hour >= 18,
        channel: channel.name,
        user: user.name,
        reactions: msg.reactions?.map(r => ({ name: r.name, count: r.count })) || [],
        replyCount: msg.reply_count || 0,
        threadTs: msg.thread_ts,
      };

      if (ts >= thisWeekStart) {
        thisWeekMessages.push(processed);
        thisWeekCount++;
      } else {
        lastWeekMessages.push(processed);
        lastWeekCount++;
      }
    }

    if (thisWeekCount > 0 || lastWeekCount > 0) {
      channelStats.push({
        name: channel.name,
        thisWeek: thisWeekCount,
        lastWeek: lastWeekCount,
      });
    }
  }

  // Sort by timestamp (newest first)
  thisWeekMessages.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
  lastWeekMessages.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
  channelStats.sort((a, b) => (b.thisWeek + b.lastWeek) - (a.thisWeek + a.lastWeek));

  console.log(`  This week: ${thisWeekMessages.length} messages`);
  console.log(`  Last week: ${lastWeekMessages.length} messages`);
  console.log(`  Active channels: ${channelStats.length}`);

  if (thisWeekMessages.length < 10) {
    console.warn("\n⚠ Very few messages this week - analysis may not be meaningful");
  }

  // 4. Build summary stats
  const buildStats = (messages) => {
    const uniqueUsers = new Set(messages.map(m => m.user)).size;
    const afterHoursCount = messages.filter(m => m.isAfterHours).length;
    const weekendCount = messages.filter(m => m.isWeekend).length;
    const totalReactions = messages.reduce((sum, m) => sum + m.reactions.length, 0);
    const threadedCount = messages.filter(m => m.threadTs).length;

    return {
      messageCount: messages.length,
      uniqueUsers,
      afterHoursCount,
      afterHoursPercent: messages.length > 0 ? Math.round(afterHoursCount / messages.length * 100) : 0,
      weekendCount,
      weekendPercent: messages.length > 0 ? Math.round(weekendCount / messages.length * 100) : 0,
      totalReactions,
      avgReactionsPerMessage: messages.length > 0 ? (totalReactions / messages.length).toFixed(2) : 0,
      threadedPercent: messages.length > 0 ? Math.round(threadedCount / messages.length * 100) : 0,
    };
  };

  const output = {
    workspace,
    fetchedAt: new Date().toISOString(),
    period: {
      thisWeek: `${formatDate(new Date(thisWeekStart * 1000))} - ${formatDate(new Date())}`,
      lastWeek: `${formatDate(new Date(lastWeekStart * 1000))} - ${formatDate(new Date(thisWeekStart * 1000))}`,
    },
    summary: {
      thisWeek: buildStats(thisWeekMessages),
      lastWeek: buildStats(lastWeekMessages),
    },
    channels: channelStats.slice(0, 20), // Top 20 channels
    thisWeek: thisWeekMessages.slice(0, 300), // Cap for context window
    lastWeek: lastWeekMessages.slice(0, 300),
  };

  // Write output
  const dir = path.dirname(outputPath);
  if (dir && dir !== ".") fs.mkdirSync(dir, { recursive: true });

  const jsonData = JSON.stringify(output, null, 2);
  fs.writeFileSync(outputPath, jsonData);

  // Verify write
  const stats = fs.statSync(outputPath);
  console.log(`\n✓ Data written to: ${outputPath} (${(stats.size / 1024).toFixed(1)}KB)`);

  console.log(JSON.stringify({
    success: true,
    outputPath,
    workspace,
    thisWeekCount: thisWeekMessages.length,
    lastWeekCount: lastWeekMessages.length,
    channelCount: channelStats.length,
  }));

} catch (error) {
  console.error("Failed:", error.message);
  throw error;
}