import { fetch } from '@tauri-apps/plugin-http'; const COLLECTION_DATA = 'data-v3'; const COLLECTION_THEME = 'theme-v1'; const COLLECTION_KEYWORD = 'keywords_v1'; async function clearCollection(collection){ // clear Qdrant collection const res_delete=await fetch(`http://localhost:6333/collections/${collection}`, { method: 'DELETE', }); const data_delete = await res_delete.json(); console.log(data_delete); const res=await fetch(`http://localhost:6333/collections/${collection}`, { method: 'PUT', body: JSON.stringify({ vectors: { size: 1536, distance: "Cosine" }, payload: { text: "string" } }), }); const data = await res.json(); console.log(data); } async function textToEmbeddings(text){ // call embedding API const response=await fetch('https://api.openai.com/v1/embeddings', { method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${import.meta.env.VITE_OPENAI_API_KEY}` }, body: JSON.stringify({ model: "text-embedding-3-small", input: text }) }); const result = await response.json(); console.log(result); return result.data[0]?.embedding; } export async function processTheme(themes){ await clearCollection(COLLECTION_THEME); themes.forEach(async (item, index)=>{ const text=await themeToText(item); console.log(text); const embeddings=await textToEmbeddings(text); await writeToQdrant(index, embeddings, JSON.stringify(item),COLLECTION_THEME); }); } export async function processKeywords(keywords, collection){ await clearCollection(collection); keywords.forEach(async (item, index)=>{ const text= item.title; console.log(text); const embeddings=await textToEmbeddings(text); await writeToQdrant(index, embeddings, JSON.stringify(item), collection); }); } async function themeToText(item){ let text = ""; text += `Title: ${item.title} `; text += `Keywords: ${item.keywords.join(", ")} `; return text; } async function writeToQdrant(id,embeddings, text, collection){ // write embeddings to Qdrant const res=await fetch(`http://localhost:6333/collections/${collection}/points?wait=true`, { method: 'PUT', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify({ points: [{ id: id, vector: embeddings, payload: { text: text } }] }) }); const data = await res.json(); console.log(data); return data; } export async function searchByText(query){ const queryEmbeddings=await textToEmbeddings(query); return await searchQdrant(queryEmbeddings); } export async function searchByTheme(themeId, limit){ // get theme embeddings from Qdrant const res=await fetch(`http://localhost:6333/collections/${COLLECTION_THEME}/points/${themeId}`, { method: 'GET', headers: { 'Content-Type': 'application/json', } }); if (!res.ok) { const errorData = await res.json(); console.error("Search failed:", errorData); return null; } const data = await res.json(); console.log(data); const allResults=await searchQdrant(data.result.vector, limit); normalizeResultTime(allResults); return allResults; } export async function searchByKeywords(keywordIds, limit){ // get keyword embeddings from Qdrant let allResults = []; for(const keywordId of keywordIds){ const res=await fetch(`http://localhost:6333/collections/${COLLECTION_KEYWORD}/points/${keywordId}`, { method: 'GET', headers: { 'Content-Type': 'application/json', } }); if (!res.ok) { const errorData = await res.json(); console.error("Search failed:", errorData); continue; } const data = await res.json(); const results = await searchQdrant(data.result.vector, Math.floor(limit/keywordIds.length)); allResults.push(...results); } normalizeResultTime(allResults); return allResults; } async function searchQdrant(query_embeddings, limit=10){ // search Qdrant const res=await fetch(`http://localhost:6333/collections/${COLLECTION_DATA}/points/search`, { method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify({ vector: query_embeddings, limit: limit, with_payload: true, with_vector: true }) }); if (!res.ok) { const errorData = await res.json(); console.error("Search failed:", errorData); return null; } const data = await res.json(); console.log("Search Result:", data); return data.result; } function normalizeResultTime(results){ let min, max; results.forEach(item=>{ const time = JSON.parse(item.payload.metadata).published_on; if(!min || timemax) max=time; }); console.log("Time range:", min, max); results.forEach(item=>{ const time = JSON.parse(item.payload.metadata).published_on; item.payload.normalized_time = (time - min) / (max - min); }); return results; }