move code from papers in separate file

app.py

@@ -1,11 +1,9 @@
 from climateqa.engine.embeddings import get_embeddings_function
 embeddings_function = get_embeddings_function()
 
-from climateqa.knowledge.openalex import OpenAlex
 from sentence_transformers import CrossEncoder
 
 # reranker = CrossEncoder("mixedbread-ai/mxbai-rerank-xsmall-v1")
-oa = OpenAlex()
 
 import gradio as gr
 from gradio_modal import Modal
@@ -44,10 +42,9 @@ from climateqa.engine.chains.prompts import audience_prompts
 from climateqa.sample_questions import QUESTIONS
 from climateqa.constants import POSSIBLE_REPORTS, OWID_CATEGORIES
 from climateqa.utils import get_image_from_azure_blob_storage
-from climateqa.engine.keywords import make_keywords_chain
-from climateqa.engine.chains.answer_rag import make_rag_papers_chain
 from climateqa.engine.graph import make_graph_agent
 from climateqa.engine.embeddings import get_embeddings_function
+from climateqa.engine.chains.retrieve_papers import find_papers
 
 from front.utils import serialize_docs,process_figures,make_html_df
 
@@ -249,84 +246,9 @@ def log_on_azure(file, logs, share_client):
     file_client.upload_file(logs)
 
 
-def generate_keywords(query):
-    chain = make_keywords_chain(llm)
-    keywords = chain.invoke(query)
-    keywords = " AND ".join(keywords["keywords"])
-    return keywords
 
 
 
-papers_cols_widths = {
-    "id":100,
-    "title":300,
-    "doi":100,
-    "publication_year":100,
-    "abstract":500,
-    "is_oa":50,
-}
-
-papers_cols = list(papers_cols_widths.keys())
-papers_cols_widths = list(papers_cols_widths.values())
-
-
-async def find_papers(query,after, relevant_content_sources):
-    if "OpenAlex" in relevant_content_sources:
-        summary = ""
-        keywords = generate_keywords(query)
-        df_works = oa.search(keywords,after = after)
-        df_works = df_works.dropna(subset=["abstract"])
-        df_works = oa.rerank(query,df_works,reranker)
-        df_works = df_works.sort_values("rerank_score",ascending=False)
-        docs_html = []
-        for i in range(10):
-            docs_html.append(make_html_df(df_works, i))
-        docs_html = "".join(docs_html)
-        print(docs_html)
-        G = oa.make_network(df_works)
-
-        height = "750px"
-        network = oa.show_network(G,color_by = "rerank_score",notebook=False,height = height)
-        network_html = network.generate_html()
-
-        network_html = network_html.replace("'", "\"")
-        css_to_inject = "<style>#mynetwork { border: none !important; } .card { border: none !important; }</style>"
-        network_html = network_html + css_to_inject
-
-        
-        network_html = f"""<iframe style="width: 100%; height: {height};margin:0 auto" name="result" allow="midi; geolocation; microphone; camera; 
-        display-capture; encrypted-media;" sandbox="allow-modals allow-forms 
-        allow-scripts allow-same-origin allow-popups 
-        allow-top-navigation-by-user-activation allow-downloads" allowfullscreen="" 
-        allowpaymentrequest="" frameborder="0" srcdoc='{network_html}'></iframe>"""
-
-
-        docs = df_works["content"].head(10).tolist()
-
-        df_works = df_works.reset_index(drop = True).reset_index().rename(columns = {"index":"doc"})
-        df_works["doc"] = df_works["doc"] + 1
-        df_works = df_works[papers_cols]
-
-        yield docs_html, network_html, summary 
-
-        chain = make_rag_papers_chain(llm)
-        result = chain.astream_log({"question": query,"docs": docs,"language":"English"})
-        path_answer = "/logs/StrOutputParser/streamed_output/-"
-
-        async for op in result:
-
-            op = op.ops[0]
-
-            if op['path'] == path_answer: # reforulated question
-                new_token = op['value'] # str
-                summary += new_token
-            else:
-                continue
-            yield docs_html, network_html, summary
-    else :
-        yield "","", ""
-
-
 # --------------------------------------------------------------------
 # Gradio
 # --------------------------------------------------------------------
@@ -430,7 +352,10 @@ with gr.Blocks(title="Climate Q&A", css_paths=os.getcwd()+ "/style.css", theme=t
                     with gr.Tab("Configuration", id = 10, ) as tab_config:
                             gr.Markdown("Reminders: You can talk in any language, ClimateQ&A is multi-lingual!")
 
+
+                            
                             with gr.Row():
+                                
                                 dropdown_sources = gr.CheckboxGroup(
                                     ["IPCC", "IPBES","IPOS"],
                                     label="Select source",
@@ -443,7 +368,7 @@ with gr.Blocks(title="Climate Q&A", css_paths=os.getcwd()+ "/style.css", theme=t
                                     value=["IPCC figures"],
                                     interactive=True,
                                 )
-                            
+
                             dropdown_reports = gr.Dropdown(
                                 POSSIBLE_REPORTS,
                                 label="Or select specific reports",
@@ -452,6 +377,9 @@ with gr.Blocks(title="Climate Q&A", css_paths=os.getcwd()+ "/style.css", theme=t
                                 interactive=True,
                             )
 
+                            search_only = gr.Checkbox(label="Search only without chating", value=False, interactive=True, elem_id="checkbox-chat")
+
+
                             dropdown_audience = gr.Dropdown(
                                 ["Children","General public","Experts"],
                                 label="Select audience",

climateqa/engine/chains/retrieve_papers.py

@@ -0,0 +1,95 @@
+from climateqa.engine.keywords import make_keywords_chain
+from climateqa.engine.llm import get_llm
+from climateqa.knowledge.openalex import OpenAlex
+from climateqa.engine.chains.answer_rag import make_rag_papers_chain
+from front.utils import make_html_df
+from climateqa.engine.reranker import get_reranker
+
+oa = OpenAlex()
+
+llm = get_llm(provider="openai",max_tokens = 1024,temperature = 0.0)
+reranker = get_reranker("nano")
+
+
+papers_cols_widths = {
+    "id":100,
+    "title":300,
+    "doi":100,
+    "publication_year":100,
+    "abstract":500,
+    "is_oa":50,
+}
+
+papers_cols = list(papers_cols_widths.keys())
+papers_cols_widths = list(papers_cols_widths.values())
+
+
+
+def generate_keywords(query):
+    chain = make_keywords_chain(llm)
+    keywords = chain.invoke(query)
+    keywords = " AND ".join(keywords["keywords"])
+    return keywords
+
+
+async def find_papers(query,after, relevant_content_sources, reranker= reranker):
+    if "OpenAlex" in relevant_content_sources:
+        summary = ""
+        keywords = generate_keywords(query)
+        df_works = oa.search(keywords,after = after)
+        
+        print(f"Found {len(df_works)} papers")
+        
+        if not df_works.empty:
+            df_works = df_works.dropna(subset=["abstract"])
+            df_works = df_works[df_works["abstract"] != ""].reset_index(drop = True)
+            df_works = oa.rerank(query,df_works,reranker)
+            df_works = df_works.sort_values("rerank_score",ascending=False)
+            docs_html = []
+            for i in range(10):
+                docs_html.append(make_html_df(df_works, i))
+            docs_html = "".join(docs_html)
+            G = oa.make_network(df_works)
+
+            height = "750px"
+            network = oa.show_network(G,color_by = "rerank_score",notebook=False,height = height)
+            network_html = network.generate_html()
+
+            network_html = network_html.replace("'", "\"")
+            css_to_inject = "<style>#mynetwork { border: none !important; } .card { border: none !important; }</style>"
+            network_html = network_html + css_to_inject
+
+            
+            network_html = f"""<iframe style="width: 100%; height: {height};margin:0 auto" name="result" allow="midi; geolocation; microphone; camera; 
+            display-capture; encrypted-media;" sandbox="allow-modals allow-forms 
+            allow-scripts allow-same-origin allow-popups 
+            allow-top-navigation-by-user-activation allow-downloads" allowfullscreen="" 
+            allowpaymentrequest="" frameborder="0" srcdoc='{network_html}'></iframe>"""
+
+
+            docs = df_works["content"].head(10).tolist()
+
+            df_works = df_works.reset_index(drop = True).reset_index().rename(columns = {"index":"doc"})
+            df_works["doc"] = df_works["doc"] + 1
+            df_works = df_works[papers_cols]
+
+            yield docs_html, network_html, summary 
+
+            chain = make_rag_papers_chain(llm)
+            result = chain.astream_log({"question": query,"docs": docs,"language":"English"})
+            path_answer = "/logs/StrOutputParser/streamed_output/-"
+
+            async for op in result:
+
+                op = op.ops[0]
+
+                if op['path'] == path_answer: # reforulated question
+                    new_token = op['value'] # str
+                    summary += new_token
+                else:
+                    continue
+                yield docs_html, network_html, summary
+        else : 
+            print("No papers found")
+    else :
+        yield "","", ""

climateqa/engine/keywords.py

@@ -11,10 +11,12 @@ class KeywordsOutput(BaseModel):
     
     keywords: list = Field(
         description="""
-        Generate 1 or 2 relevant keywords from the user query to ask a search engine for scientific research papers.
+        Generate 1 or 2 relevant keywords from the user query to ask a search engine for scientific research papers. Answer only with English keywords. 
+        Do not use special characters or accents.
         
         Example:
         - "What is the impact of deep sea mining ?" -> ["deep sea mining"]
+        - "Quel est l'impact de l'exploitation minière en haute mer ?" -> ["deep sea mining"]
         - "How will El Nino be impacted by climate change" -> ["el nino"]
         - "Is climate change a hoax" -> [Climate change","hoax"]
         """

climateqa/knowledge/openalex.py

@@ -41,6 +41,10 @@ class OpenAlex():
                 break
 
             df_works = pd.DataFrame(page)
+            
+            if df_works.empty:
+                return df_works
+            
             df_works = df_works.dropna(subset = ["title"])
             df_works["primary_location"] = df_works["primary_location"].map(replace_nan_with_empty_dict)
             df_works["abstract"] = df_works["abstract_inverted_index"].apply(lambda x: self.get_abstract_from_inverted_index(x)).fillna("")