👾 Code
Fetch Recent Tweets (Multiple Handles)
Fetch up to 100 recent tweets for multiple Twitter handles in parallel using widget scraping, plus basic profile metadata from SocialData.
Source Code
const [handlesArg] = process.argv.slice(2);
const rawHandles = (handlesArg || "").trim();
const maxResults = 100;
if (!rawHandles) {
console.error("Twitter handles are required (comma-separated)");
throw new Error("Missing required parameter: handles");
}
// Parse and normalize handles
const handles = rawHandles
.split(",")
.map((h) => h.trim().replace(/^@/, ""))
.filter((h) => h.length > 0);
if (handles.length === 0) {
console.error("No valid Twitter handles provided");
throw new Error("No valid handles after parsing");
}
if (handles.length > 10) {
console.error("Too many handles. Maximum is 10 to avoid rate limiting.");
throw new Error("Maximum 10 handles allowed");
}
console.log(
`Fetching up to ${maxResults} tweets for ${
handles.length
} handle(s): @${handles.join(", @")}`
);
async function fetchSocialDataProfile(screenName) {
const apiKey = process.env.SOCIAL_DATA_API_KEY;
if (!apiKey) {
console.warn(
"SOCIAL_DATA_API_KEY is not set; skipping SocialData profile fetch."
);
return null;
}
const url = `https://api.socialdata.tools/twitter/user/${encodeURIComponent(
screenName
)}`;
const res = await fetch(url, {
method: "GET",
headers: {
Authorization: `Bearer ${apiKey}`,
Accept: "application/json",
},
});
const text = await res.text();
if (!res.ok) {
console.error(
`SocialData profile request failed with status ${res.status} for @${screenName}`
);
console.error(text.slice(0, 500));
return null;
}
let json;
try {
json = JSON.parse(text);
} catch (err) {
console.error(
`Failed to parse SocialData profile JSON for @${screenName}:`,
err.message
);
return null;
}
return {
id: json.id_str || String(json.id || ""),
name: json.name || "",
screenName: json.screen_name || screenName,
description: json.description || "",
location: json.location || "",
profileImageUrl: json.profile_image_url_https || "",
profileBannerUrl: json.profile_banner_url || "",
followersCount:
typeof json.followers_count === "number" ? json.followers_count : 0,
friendsCount:
typeof json.friends_count === "number" ? json.friends_count : 0,
statusesCount:
typeof json.statuses_count === "number" ? json.statuses_count : 0,
createdAt: json.created_at || "",
};
}
async function fetchTimelineProfile(screenName) {
const url = `https://syndication.twitter.com/srv/timeline-profile/screen-name/${screenName}`;
const params = new URLSearchParams({
dnt: "false",
embedId: "twitter-widget-1",
frame: "false",
hideHeader: "false",
lang: "en",
showReplies: "false",
});
const headers = {
accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
"accept-language": "en-US,en;q=0.9",
cookie: `guest_id=v1%3A168987090345885498; kdt=p0eqA5087j6MNpREwxYJcudjCegtrzc9b8J7H2iI; auth_token=e4fa2e1fb11107b1648add7bbd58d3cd4e4b2d5e; ct0=f39f4820e35d45a0b64651eb46acae1694bfeb221af2d1b650b74a81bebd95673869b2504b75d005809bebe0a71e1863c87338e5b534a02c8ce4d10b1730f501127333aadc34cf9c6dece1d74c91b00f; twid=u%3D945756809356300294; dnt=1; btc_opt_in=Y; twtr_pixel_opt_in=Y; *twitter*sess=BAh7CiIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo%250ASGFzaHsABjoKQHVzZWR7ADoPY3JlYXRlZF9hdGwrCAIPOL2LAToMY3NyZl9p%250AZCIlODJlZjUzMTE0ZmZkZWZlNzVhMzM0NTM3M2FhNWZhNmI6B2lkIiU3OGMy%250ANDlmN2Y5NWRjYmVkZTliMTYyZmI2YWVjYjgzZToVaW5pdGlhdGVkX2luX2Fw%250AcCIGMQ%253D%253D--e4fb722064815b66eb4cb5098a47fc74eef01367; fm="WW91IHdpbGwgbm8gbG9uZ2VyIHJlY2VpdmUgZW1haWxzIGxpa2UgdGhpcy4=--61a3e5587a505c23e2499300d1d7f92ff6d971e0"; guest_id_marketing=v1%3A168987090345885498; guest_id_ads=v1%3A168987090345885498; personalization_id="v1_qWZOJ07EYJQV7qtkcyHuQg=="; *ga=GA1.2.891791384.1722577601; *gid=GA1.2.262146840.1723035771; *gat=1`,
"user-agent":
"Mozilla/5.0 (Skills Bot; +https://skills.dom) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
};
const response = await fetch(`${url}?${params.toString()}`, { headers });
if (!response.ok) {
const body = await response.text();
console.error(`Widget HTTP error for @${screenName}: ${response.status}`);
console.error(body.slice(0, 300));
throw new Error(
`Failed to fetch widget timeline for @${screenName}: ${response.status}`
);
}
return await response.text();
}
function extractWidgetTweetsFromHtml(html, limit) {
const m = html.match(
/<script[^>]*id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/i
);
if (!m) {
throw new Error("__NEXT_DATA__ script not found in widget HTML");
}
let data;
try {
data = JSON.parse(m[1]);
} catch (e) {
console.error("Failed to parse __NEXT_DATA__ JSON");
throw e;
}
const entries =
data?.props?.pageProps?.timeline?.entries ||
data?.props?.pageProps?.timeline?.instructions?.[0]?.entries ||
[];
const tweets = entries
.filter(
(entry) =>
entry && entry.type === "tweet" && entry.content && entry.content.tweet
)
.map((entry) => entry.content.tweet);
if (!tweets.length) {
throw new Error("No tweet entries found in widget data");
}
tweets.sort(
(a, b) =>
new Date(b.created_at).getTime() - new Date(a.created_at).getTime()
);
return tweets.slice(0, limit);
}
function normalizeWidgetTweet(tweet) {
return {
id: tweet.id_str || tweet.id || "",
text: tweet.full_text || tweet.text || "",
createdAt: tweet.created_at || "",
likeCount: tweet.favorite_count || 0,
retweetCount: tweet.retweet_count || 0,
replyCount: tweet.reply_count || 0,
quoteCount: tweet.quote_count || 0,
};
}
// Fetch tweets for a single handle
async function fetchTweetsForHandle(handle) {
console.log(`\n[${handle}] Starting fetch...`);
try {
// Fetch profile first
const profile = await fetchSocialDataProfile(handle).catch((err) => {
console.error(
`[${handle}] Error fetching SocialData profile:`,
err.message
);
return null;
});
// Fetch tweets via widget
const html = await fetchTimelineProfile(handle);
const widgetTweets = extractWidgetTweetsFromHtml(html, maxResults);
if (!widgetTweets || widgetTweets.length === 0) {
console.log(`[${handle}] No tweets found`);
return {
handle,
success: true,
profile,
tweets: [],
};
}
const normalized = widgetTweets.map(normalizeWidgetTweet);
console.log(
`[${handle}] ✓ Fetched ${normalized.length} tweet(s)${
profile ? " with profile metadata" : ""
}`
);
return {
handle,
success: true,
profile,
tweets: normalized,
};
} catch (error) {
console.error(`[${handle}] ✗ Error:`, error.message);
return {
handle,
success: false,
error: error.message,
profile: null,
tweets: [],
};
}
}
// Fetch all handles in parallel
try {
console.log("Starting parallel Twitter fetch for multiple handles...");
const results = await Promise.all(
handles.map((handle) => fetchTweetsForHandle(handle))
);
// Summarize results
const successful = results.filter((r) => r.success);
const failed = results.filter((r) => !r.success);
console.log("\n=== FETCH SUMMARY ===");
console.log(`Total handles: ${handles.length}`);
console.log(`Successful: ${successful.length}`);
console.log(`Failed: ${failed.length}`);
if (failed.length > 0) {
console.log("\nFailed handles:");
failed.forEach((f) => console.log(` - @${f.handle}: ${f.error}`));
}
// Output complete results as JSON
console.log("\n=== COMPLETE RESULTS ===");
console.log(JSON.stringify({ results }, null, 2));
} catch (error) {
console.error("Fatal error during parallel fetch:", error.message);
throw error;
}