Source-Web-23070-DataAnalysis/v2/app/src/utils/parsing.js

import { fetch } from '@tauri-apps/plugin-http';

const COLLECTION_DATA = 'data-v3';
const COLLECTION_THEME = 'theme-v1';
const COLLECTION_KEYWORD = 'keywords_v1';

async function clearCollection(collection){
    // clear Qdrant collection
    const res_delete=await fetch(`http://localhost:6333/collections/${collection}`, {
        method: 'DELETE',
    });
    const data_delete = await res_delete.json();
    console.log(data_delete);

    const res=await fetch(`http://localhost:6333/collections/${collection}`, {
        method: 'PUT',
        body: JSON.stringify({
            vectors: {
                size: 1536,
                distance: "Cosine"
            },
            payload: {
                text: "string"
            }
        }),
    });
    const data = await res.json();
    console.log(data);

}


async function textToEmbeddings(text){
   // call embedding API
   const response=await fetch('https://api.openai.com/v1/embeddings', {
       method: 'POST',
       headers: {
           'Content-Type': 'application/json',
           'Authorization': `Bearer ${import.meta.env.VITE_OPENAI_API_KEY}`
       },
       body: JSON.stringify({
           model: "text-embedding-3-small",
           input: text
       })
   });
   const result = await response.json();
   console.log(result);

   return result.data[0]?.embedding;
}



export async function processTheme(themes){

    await clearCollection(COLLECTION_THEME);

    themes.forEach(async (item, index)=>{
        const text=await themeToText(item);
        console.log(text);

        const embeddings=await textToEmbeddings(text);
        await writeToQdrant(index, embeddings, JSON.stringify(item),COLLECTION_THEME);
    });
}

export async function processKeywords(keywords, collection){
    await clearCollection(collection);

    keywords.forEach(async (item, index)=>{
        const text= item.title;
        console.log(text);

        const embeddings=await textToEmbeddings(text);
        await writeToQdrant(index, embeddings, JSON.stringify(item), collection);
    });
}

async function themeToText(item){
    let text = "";
    text += `Title: ${item.title} `;
    text += `Keywords: ${item.keywords.join(", ")} `;
    return text;
}

async function writeToQdrant(id,embeddings, text, collection){
    // write embeddings to Qdrant
    const res=await fetch(`http://localhost:6333/collections/${collection}/points?wait=true`, {
        method: 'PUT',
        headers: {
            'Content-Type': 'application/json',
        },
        body: JSON.stringify({
            points: [{
                id: id,
                vector: embeddings,
                payload: {
                    text: text
                }
            }]
        })
    });

    const data = await res.json();
    console.log(data);

    return data;
}

export async function searchByText(query){
    const queryEmbeddings=await textToEmbeddings(query);
    return await searchQdrant(queryEmbeddings);
}

export async function searchByTheme(themeId, limit){
    // get theme embeddings from Qdrant
    const res=await fetch(`http://localhost:6333/collections/${COLLECTION_THEME}/points/${themeId}`, {
        method: 'GET',
        headers: {
            'Content-Type': 'application/json',
        }
    });

    if (!res.ok) {
        const errorData = await res.json();
        console.error("Search failed:", errorData);
        return null;
    }

    const data = await res.json();
    console.log(data);

    const allResults=await searchQdrant(data.result.vector, limit);
    normalizeResultTime(allResults);

    return allResults;
}

export async function searchByKeywords(keywordIds, limit){
    // get keyword embeddings from Qdrant
    let allResults = [];
    for(const keywordId of keywordIds){
        const res=await fetch(`http://localhost:6333/collections/${COLLECTION_KEYWORD}/points/${keywordId}`, {
            method: 'GET',
            headers: {
                'Content-Type': 'application/json',
            }
        });
        if (!res.ok) {
            const errorData = await res.json();
            console.error("Search failed:", errorData);
            continue;
        }

        const data = await res.json();

        const results = await searchQdrant(data.result.vector, Math.floor(limit/keywordIds.length));

        allResults.push(...results);
    }
    
    normalizeResultTime(allResults);

    return allResults;
}

async function searchQdrant(query_embeddings, limit=10){
    // search Qdrant
    const res=await fetch(`http://localhost:6333/collections/${COLLECTION_DATA}/points/search`, {
        method: 'POST',
        headers: {
            'Content-Type': 'application/json',
        },
        body: JSON.stringify({
            vector: query_embeddings,
            limit: limit,
            with_payload: true,
            with_vector: true
        })
    });

    if (!res.ok) {
        const errorData = await res.json();
        console.error("Search failed:", errorData);
        return null;
    }

    const data = await res.json();
    console.log("Search Result:", data);

    return data.result;
}

function normalizeResultTime(results){

    let min, max;
    results.forEach(item=>{
        const time = JSON.parse(item.payload.metadata).published_on;
        if(!min || time<min) min=time;
        if(!max || time>max) max=time;
    });

    console.log("Time range:", min, max);

    results.forEach(item=>{
        const time = JSON.parse(item.payload.metadata).published_on;
        item.payload.normalized_time = (time - min) / (max - min);
    });

    return results;

}