Source-Web-23070-DataAnalysis/v2/scrapper/main.js

import { getThread } from "./scrapper.js";
import { searchThreads } from "./search.js";
import { writeFileSync } from "fs";

const cookies = [
  {
    name: 'sessionid',
    value: '64605724719%3ALlZCmwghVyOAck%3A23%3AAYhptDoKttkGRWkpa5583neohBfLXlGfOlwPPmdP1w',
    domain: '.threads.com'
  },
  {
    name: 'ds_user_id',
    value: '64605724719',
    domain: '.threads.com'
  },
  {
    name:'csrftoken',
    value:'SI5YedKIeuSAgAbdtfynUwzrmSAGquxH',
    domain:'.threads.com'
  }
];

async function main(){

    const searchResults = await searchThreads("厭世", 20, cookies);
    console.log(JSON.stringify(searchResults));

    for(const url of searchResults.urls){
        try{
            const threadItems=await getThread(url);
            console.log(JSON.stringify(threadItems));

            // save to filesystem
            const fileName=url.split("/post/")[1].split("/")[0];
            writeFileSync(`scrapped/${fileName}.json`, JSON.stringify(threadItems, null, 2) );
        } catch(err){
            console.error("Error processing", url, ":", err);
        }
    }

//     const threadUrl="https://www.threads.com/@bobolove0828/post/DQ63pgpklmi";
//     const threadItems=await getThread(threadUrl, cookies);
//     console.log(JSON.stringify(threadItems));
//     const fileName=threadUrl.split("/post/")[1].split("/")[0];
//     writeFileSync(`scrapped/${fileName}.json`, JSON.stringify(threadItems, null, 2) );
}

main();