Added vdb-v3-wikksplitter metadata

app.py

@@ -11,7 +11,7 @@ MAX_NEW_TOKENS = 700
 SHOW_MODEL_PARAMETERS_IN_UI = os.environ.get("SHOW_MODEL_PARAMETERS_IN_UI", default="False") == "True"
 import logging
 
-logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] - %(message)s')
+logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(name)s][%(levelname)s] - %(message)s')
 
 setup()
 

rag.py

@@ -9,9 +9,10 @@ from huggingface_hub import snapshot_download, InferenceClient
 from langchain_community.vectorstores import FAISS
 from langchain_community.embeddings import HuggingFaceEmbeddings
 
+from termcolor import cprint
 
-logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s] - %(message)s')
 # logging.getLogger().setLevel(logging.INFO)
+logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(name)s][%(levelname)s] - %(message)s')
 
 
 class RAG:
@@ -99,6 +100,15 @@ class RAG:
         documents_retrieved = self.vectore_store.similarity_search_with_score_by_vector(embedding, k=number_of_contexts)
         logging.info(f"Documents retrieved: {len(documents_retrieved)}")
 
+        for i, (doc, score) in enumerate(documents_retrieved):
+            logging.info(f"Document {i+1}:")
+            logging.info(f"Score: {score:.5f}")
+            logging.info(f"Title: {doc.metadata}")
+            # logging.info(f"Source: {doc.metadata['src']}")
+            # logging.info(f"Redirected: {doc.metadata['redirected']}")
+            # url = doc.metadata['url']
+            # logging.info(f"Revision ID: {url}")
+            # logging.info(f'URL: <a href="{url}" target="_blank">{url}</a><br>')
 
         # Reranking
         # ==============================================================================================================
@@ -137,6 +147,7 @@ class RAG:
 
         return response.json()[0]["generated_text"].split("###")[-1][8:]
 
+    
     def predict_completion(self, instruction, context, model_parameters):
         
         client = OpenAI(
@@ -183,22 +194,39 @@ class RAG:
 
         return text_context, full_context, source_context
 
+    
     def get_response(self, prompt: str, model_parameters: dict) -> str:
         try:
             docs = self.get_context(prompt, model_parameters["NUM_CHUNKS"])
 
             response = ""
 
-            for i, (doc, score) in enumerate(docs):
+            for i, (doc, score) in enumerate(docs):                
                 
+                # ----------------------------------------------------------------------------
+                # vector_db__BAAI__bge-m3__cfg-v3-wikisplitter => metadata
+                # ----------------------------------------------------------------------------
+                # {
+                #     'document_id': '1535', 
+                #     'title': 'Intel·ligència artificial', 
+                #     'url': 'https://ca.wikipedia.org/wiki?curid=1535', 
+                #     'language': 'ca', 
+                #     'src': '/gpfs/projects/bsc88/apps/projects/__wiki-rag__/_data/json_extractor/cawiki-20250501/wiki_00.jsonl', 
+                #     'section_title': 'Centres tecnològics a Catalunya i les seves aportacions i investigacions en la IA.', 
+                #     'section_id': 32, 
+                #     'section_len': 3403, 
+                #     'split_level': 'section'
+                # }
+                # ----------------------------------------------------------------------------
                 response += "\n\n" + "="*100
                 response += f"\nDocument {i+1}"
                 response += "\n" + "="*100
                 response += f"\nScore: {score:.5f}"
                 response += f"\nTitle: {doc.metadata['title']}"
+                response += f"\nSection title: {doc.metadata['section_title']}"
                 response += f"\nURL: {doc.metadata['url']}"
-                response += f"\nID: {doc.metadata['id']}"
-                response += f"\nStart index: {doc.metadata['start_index']}"
+                response += f"\nID: {doc.metadata['document_id']}"
+                # response += f"\nStart index: {doc.metadata['start_index']}"
                 # response += f"\nSource: {doc.metadata['src']}"
                 # response += f"\nRedirected: {doc.metadata['redirected']}"
                 # url = doc.metadata['url']