Fetch Workspace Messages
Fetch messages from all accessible public channels for sentiment analysis
Source Code
import fs from "fs";
import path from "path";
const [outputPath, daysBackArg = "14"] = process.argv.slice(2);
const daysBack = Math.min(parseInt(daysBackArg) || 14, 30);
if (!outputPath) {
console.error("Usage: sentiment.slack.fetch <outputPath> [daysBack]");
console.error("Output path is required.");
process.exit(1);
}
const now = Date.now();
const msPerDay = 24 * 60 * 60 * 1000;
const thisWeekStart = Math.floor((now - 7 * msPerDay) / 1000);
const lastWeekStart = Math.floor((now - daysBack * msPerDay) / 1000);
const formatDate = (d) =>
d.toLocaleDateString("en-US", { month: "short", day: "numeric" });
console.log(`Fetching workspace messages for sentiment analysis...`);
console.log(` This week: ${formatDate(new Date(thisWeekStart * 1000))} - ${formatDate(new Date())}`);
console.log(` Last week: ${formatDate(new Date(lastWeekStart * 1000))} - ${formatDate(new Date(thisWeekStart * 1000))}`);
try {
// 1. Get auth, channels, and users in parallel
console.log("Fetching workspace data...");
const [authData, channelsData, usersData] = await Promise.all([
(async () => {
const res = await fetch("https://slack.com/api/auth.test", {
headers: { Authorization: "Bearer PLACEHOLDER_TOKEN" },
});
const data = await res.json();
if (!data.ok) throw new Error(`Auth failed: ${data.error}`);
return data;
})(),
(async () => {
const channels = [];
let cursor = null;
do {
const params = new URLSearchParams({
types: "public_channel",
limit: "200",
exclude_archived: "true",
});
if (cursor) params.set("cursor", cursor);
const res = await fetch(
"https://slack.com/api/conversations.list?" + params,
{ headers: { Authorization: "Bearer PLACEHOLDER_TOKEN" } }
);
const data = await res.json();
if (!data.ok) throw new Error(`conversations.list failed: ${data.error}`);
channels.push(...(data.channels || []));
cursor = data.response_metadata?.next_cursor;
} while (cursor);
return channels;
})(),
(async () => {
const users = [];
let cursor = null;
do {
const params = new URLSearchParams({ limit: "200" });
if (cursor) params.set("cursor", cursor);
const res = await fetch("https://slack.com/api/users.list?" + params, {
headers: { Authorization: "Bearer PLACEHOLDER_TOKEN" },
});
const data = await res.json();
if (!data.ok) throw new Error(`users.list failed: ${data.error}`);
users.push(...(data.members || []));
cursor = data.response_metadata?.next_cursor;
} while (cursor);
return users;
})(),
]);
const workspace = authData.team;
console.log(`✓ Connected to ${workspace}`);
// Build user lookup
const userMap = new Map();
for (const u of usersData) {
if (!u.is_bot && !u.deleted) {
userMap.set(u.id, {
name: u.profile?.display_name || u.real_name || u.name,
realName: u.real_name,
});
}
}
// Get channels user is member of
const myChannels = channelsData.filter((ch) => ch.is_member);
console.log(` ${myChannels.length} public channels, ${userMap.size} users`);
if (myChannels.length === 0) {
console.error("\n✗ No accessible channels found.");
console.log(JSON.stringify({
success: false,
error: "no_channels",
workspace,
}));
process.exit(1);
}
// 2. Fetch history from channels (for the full time range)
console.log("Fetching channel message history...");
const fetchHistory = async (channelId) => {
const allMessages = [];
let cursor = null;
do {
const params = new URLSearchParams({
channel: channelId,
limit: "200",
oldest: lastWeekStart.toString(),
});
if (cursor) params.set("cursor", cursor);
const res = await fetch(
"https://slack.com/api/conversations.history?" + params,
{ headers: { Authorization: "Bearer PLACEHOLDER_TOKEN" } }
);
const data = await res.json();
if (!data.ok) {
// Some channels may fail due to permissions - skip them
return [];
}
allMessages.push(...(data.messages || []));
cursor = data.response_metadata?.next_cursor;
// Limit to reasonable amount per channel
if (allMessages.length >= 500) break;
} while (cursor);
return allMessages;
};
// Fetch channel histories in parallel (batch to avoid rate limits)
const histories = [];
for (let i = 0; i < myChannels.length; i += 15) {
const batch = myChannels.slice(i, i + 15);
const results = await Promise.all(
batch.map(async (conv) => ({
channel: { id: conv.id, name: conv.name },
messages: await fetchHistory(conv.id),
}))
);
histories.push(...results);
if (i + 15 < myChannels.length) {
// Brief pause between batches
await new Promise(r => setTimeout(r, 500));
}
}
// 3. Process messages into this week / last week buckets
const thisWeekMessages = [];
const lastWeekMessages = [];
const channelStats = [];
for (const { channel, messages } of histories) {
if (messages.length === 0) continue;
let thisWeekCount = 0;
let lastWeekCount = 0;
for (const msg of messages) {
// Skip bot messages and join/leave messages
if (msg.subtype || !msg.user || !userMap.has(msg.user)) continue;
const ts = parseFloat(msg.ts);
const user = userMap.get(msg.user);
const timestamp = new Date(ts * 1000).toISOString();
const hour = new Date(ts * 1000).getHours();
const dayOfWeek = new Date(ts * 1000).getDay(); // 0 = Sunday
const processed = {
text: msg.text,
timestamp,
hour,
isWeekend: dayOfWeek === 0 || dayOfWeek === 6,
isAfterHours: hour < 8 || hour >= 18,
channel: channel.name,
user: user.name,
reactions: msg.reactions?.map(r => ({ name: r.name, count: r.count })) || [],
replyCount: msg.reply_count || 0,
threadTs: msg.thread_ts,
};
if (ts >= thisWeekStart) {
thisWeekMessages.push(processed);
thisWeekCount++;
} else {
lastWeekMessages.push(processed);
lastWeekCount++;
}
}
if (thisWeekCount > 0 || lastWeekCount > 0) {
channelStats.push({
name: channel.name,
thisWeek: thisWeekCount,
lastWeek: lastWeekCount,
});
}
}
// Sort by timestamp (newest first)
thisWeekMessages.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
lastWeekMessages.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
channelStats.sort((a, b) => (b.thisWeek + b.lastWeek) - (a.thisWeek + a.lastWeek));
console.log(` This week: ${thisWeekMessages.length} messages`);
console.log(` Last week: ${lastWeekMessages.length} messages`);
console.log(` Active channels: ${channelStats.length}`);
if (thisWeekMessages.length < 10) {
console.warn("\n⚠ Very few messages this week - analysis may not be meaningful");
}
// 4. Build summary stats
const buildStats = (messages) => {
const uniqueUsers = new Set(messages.map(m => m.user)).size;
const afterHoursCount = messages.filter(m => m.isAfterHours).length;
const weekendCount = messages.filter(m => m.isWeekend).length;
const totalReactions = messages.reduce((sum, m) => sum + m.reactions.length, 0);
const threadedCount = messages.filter(m => m.threadTs).length;
return {
messageCount: messages.length,
uniqueUsers,
afterHoursCount,
afterHoursPercent: messages.length > 0 ? Math.round(afterHoursCount / messages.length * 100) : 0,
weekendCount,
weekendPercent: messages.length > 0 ? Math.round(weekendCount / messages.length * 100) : 0,
totalReactions,
avgReactionsPerMessage: messages.length > 0 ? (totalReactions / messages.length).toFixed(2) : 0,
threadedPercent: messages.length > 0 ? Math.round(threadedCount / messages.length * 100) : 0,
};
};
const output = {
workspace,
fetchedAt: new Date().toISOString(),
period: {
thisWeek: `${formatDate(new Date(thisWeekStart * 1000))} - ${formatDate(new Date())}`,
lastWeek: `${formatDate(new Date(lastWeekStart * 1000))} - ${formatDate(new Date(thisWeekStart * 1000))}`,
},
summary: {
thisWeek: buildStats(thisWeekMessages),
lastWeek: buildStats(lastWeekMessages),
},
channels: channelStats.slice(0, 20), // Top 20 channels
thisWeek: thisWeekMessages.slice(0, 300), // Cap for context window
lastWeek: lastWeekMessages.slice(0, 300),
};
// Write output
const dir = path.dirname(outputPath);
if (dir && dir !== ".") fs.mkdirSync(dir, { recursive: true });
const jsonData = JSON.stringify(output, null, 2);
fs.writeFileSync(outputPath, jsonData);
// Verify write
const stats = fs.statSync(outputPath);
console.log(`\n✓ Data written to: ${outputPath} (${(stats.size / 1024).toFixed(1)}KB)`);
console.log(JSON.stringify({
success: true,
outputPath,
workspace,
thisWeekCount: thisWeekMessages.length,
lastWeekCount: lastWeekMessages.length,
channelCount: channelStats.length,
}));
} catch (error) {
console.error("Failed:", error.message);
throw error;
}