👾 Code

Fetch Recent Tweets (Multiple Handles)

Fetch up to 100 recent tweets for multiple Twitter handles in parallel using widget scraping, plus basic profile metadata from SocialData.

Source Code


              const [handlesArg] = process.argv.slice(2);
const rawHandles = (handlesArg || "").trim();
const maxResults = 100;

if (!rawHandles) {
  console.error("Twitter handles are required (comma-separated)");
  throw new Error("Missing required parameter: handles");
}

// Parse and normalize handles
const handles = rawHandles
  .split(",")
  .map((h) => h.trim().replace(/^@/, ""))
  .filter((h) => h.length > 0);

if (handles.length === 0) {
  console.error("No valid Twitter handles provided");
  throw new Error("No valid handles after parsing");
}

if (handles.length > 10) {
  console.error("Too many handles. Maximum is 10 to avoid rate limiting.");
  throw new Error("Maximum 10 handles allowed");
}

console.log(
  `Fetching up to ${maxResults} tweets for ${
    handles.length
  } handle(s): @${handles.join(", @")}`
);

async function fetchSocialDataProfile(screenName) {
  const apiKey = process.env.SOCIAL_DATA_API_KEY;

  if (!apiKey) {
    console.warn(
      "SOCIAL_DATA_API_KEY is not set; skipping SocialData profile fetch."
    );
    return null;
  }

  const url = `https://api.socialdata.tools/twitter/user/${encodeURIComponent(
    screenName
  )}`;

  const res = await fetch(url, {
    method: "GET",
    headers: {
      Authorization: `Bearer ${apiKey}`,
      Accept: "application/json",
    },
  });

  const text = await res.text();

  if (!res.ok) {
    console.error(
      `SocialData profile request failed with status ${res.status} for @${screenName}`
    );
    console.error(text.slice(0, 500));
    return null;
  }

  let json;
  try {
    json = JSON.parse(text);
  } catch (err) {
    console.error(
      `Failed to parse SocialData profile JSON for @${screenName}:`,
      err.message
    );
    return null;
  }

  return {
    id: json.id_str || String(json.id || ""),
    name: json.name || "",
    screenName: json.screen_name || screenName,
    description: json.description || "",
    location: json.location || "",
    profileImageUrl: json.profile_image_url_https || "",
    profileBannerUrl: json.profile_banner_url || "",
    followersCount:
      typeof json.followers_count === "number" ? json.followers_count : 0,
    friendsCount:
      typeof json.friends_count === "number" ? json.friends_count : 0,
    statusesCount:
      typeof json.statuses_count === "number" ? json.statuses_count : 0,
    createdAt: json.created_at || "",
  };
}

async function fetchTimelineProfile(screenName) {
  const url = `https://syndication.twitter.com/srv/timeline-profile/screen-name/${screenName}`;
  const params = new URLSearchParams({
    dnt: "false",
    embedId: "twitter-widget-1",
    frame: "false",
    hideHeader: "false",
    lang: "en",
    showReplies: "false",
  });

  const headers = {
    accept:
      "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
    "accept-language": "en-US,en;q=0.9",
    cookie: `guest_id=v1%3A168987090345885498; kdt=p0eqA5087j6MNpREwxYJcudjCegtrzc9b8J7H2iI; auth_token=e4fa2e1fb11107b1648add7bbd58d3cd4e4b2d5e; ct0=f39f4820e35d45a0b64651eb46acae1694bfeb221af2d1b650b74a81bebd95673869b2504b75d005809bebe0a71e1863c87338e5b534a02c8ce4d10b1730f501127333aadc34cf9c6dece1d74c91b00f; twid=u%3D945756809356300294; dnt=1; btc_opt_in=Y; twtr_pixel_opt_in=Y; *twitter*sess=BAh7CiIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo%250ASGFzaHsABjoKQHVzZWR7ADoPY3JlYXRlZF9hdGwrCAIPOL2LAToMY3NyZl9p%250AZCIlODJlZjUzMTE0ZmZkZWZlNzVhMzM0NTM3M2FhNWZhNmI6B2lkIiU3OGMy%250ANDlmN2Y5NWRjYmVkZTliMTYyZmI2YWVjYjgzZToVaW5pdGlhdGVkX2luX2Fw%250AcCIGMQ%253D%253D--e4fb722064815b66eb4cb5098a47fc74eef01367; fm="WW91IHdpbGwgbm8gbG9uZ2VyIHJlY2VpdmUgZW1haWxzIGxpa2UgdGhpcy4=--61a3e5587a505c23e2499300d1d7f92ff6d971e0"; guest_id_marketing=v1%3A168987090345885498; guest_id_ads=v1%3A168987090345885498; personalization_id="v1_qWZOJ07EYJQV7qtkcyHuQg=="; *ga=GA1.2.891791384.1722577601; *gid=GA1.2.262146840.1723035771; *gat=1`,
    "user-agent":
      "Mozilla/5.0 (Skills Bot; +https://skills.dom) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
  };

  const response = await fetch(`${url}?${params.toString()}`, { headers });
  if (!response.ok) {
    const body = await response.text();
    console.error(`Widget HTTP error for @${screenName}: ${response.status}`);
    console.error(body.slice(0, 300));
    throw new Error(
      `Failed to fetch widget timeline for @${screenName}: ${response.status}`
    );
  }
  return await response.text();
}

function extractWidgetTweetsFromHtml(html, limit) {
  const m = html.match(
    /<script[^>]*id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/i
  );
  if (!m) {
    throw new Error("__NEXT_DATA__ script not found in widget HTML");
  }

  let data;
  try {
    data = JSON.parse(m[1]);
  } catch (e) {
    console.error("Failed to parse __NEXT_DATA__ JSON");
    throw e;
  }

  const entries =
    data?.props?.pageProps?.timeline?.entries ||
    data?.props?.pageProps?.timeline?.instructions?.[0]?.entries ||
    [];

  const tweets = entries
    .filter(
      (entry) =>
        entry && entry.type === "tweet" && entry.content && entry.content.tweet
    )
    .map((entry) => entry.content.tweet);

  if (!tweets.length) {
    throw new Error("No tweet entries found in widget data");
  }

  tweets.sort(
    (a, b) =>
      new Date(b.created_at).getTime() - new Date(a.created_at).getTime()
  );

  return tweets.slice(0, limit);
}

function normalizeWidgetTweet(tweet) {
  return {
    id: tweet.id_str || tweet.id || "",
    text: tweet.full_text || tweet.text || "",
    createdAt: tweet.created_at || "",
    likeCount: tweet.favorite_count || 0,
    retweetCount: tweet.retweet_count || 0,
    replyCount: tweet.reply_count || 0,
    quoteCount: tweet.quote_count || 0,
  };
}

// Fetch tweets for a single handle
async function fetchTweetsForHandle(handle) {
  console.log(`\n[${handle}] Starting fetch...`);

  try {
    // Fetch profile first
    const profile = await fetchSocialDataProfile(handle).catch((err) => {
      console.error(
        `[${handle}] Error fetching SocialData profile:`,
        err.message
      );
      return null;
    });

    // Fetch tweets via widget
    const html = await fetchTimelineProfile(handle);
    const widgetTweets = extractWidgetTweetsFromHtml(html, maxResults);

    if (!widgetTweets || widgetTweets.length === 0) {
      console.log(`[${handle}] No tweets found`);
      return {
        handle,
        success: true,
        profile,
        tweets: [],
      };
    }

    const normalized = widgetTweets.map(normalizeWidgetTweet);

    console.log(
      `[${handle}] ✓ Fetched ${normalized.length} tweet(s)${
        profile ? " with profile metadata" : ""
      }`
    );

    return {
      handle,
      success: true,
      profile,
      tweets: normalized,
    };
  } catch (error) {
    console.error(`[${handle}] ✗ Error:`, error.message);
    return {
      handle,
      success: false,
      error: error.message,
      profile: null,
      tweets: [],
    };
  }
}

// Fetch all handles in parallel
try {
  console.log("Starting parallel Twitter fetch for multiple handles...");

  const results = await Promise.all(
    handles.map((handle) => fetchTweetsForHandle(handle))
  );

  // Summarize results
  const successful = results.filter((r) => r.success);
  const failed = results.filter((r) => !r.success);

  console.log("\n=== FETCH SUMMARY ===");
  console.log(`Total handles: ${handles.length}`);
  console.log(`Successful: ${successful.length}`);
  console.log(`Failed: ${failed.length}`);

  if (failed.length > 0) {
    console.log("\nFailed handles:");
    failed.forEach((f) => console.log(`  - @${f.handle}: ${f.error}`));
  }

  // Output complete results as JSON
  console.log("\n=== COMPLETE RESULTS ===");
  console.log(JSON.stringify({ results }, null, 2));
} catch (error) {
  console.error("Fatal error during parallel fetch:", error.message);
  throw error;
}