included weeds into insects identifier as well; and removed domain dropdowno

app.py

@@ -78,11 +78,9 @@ def get_species_list_from_db(db_name):
 
 
 # default_persist_directory = './db5' # For deployement
-default_persist_directory_insects='./vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species'
-default_persist_directory_weeds='./vector-databases-deployed/db5-agllm-data-isu-field-weeds-all-species'
+default_persist_directory='./vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species'
 
-species_list_insects=get_species_list_from_db(default_persist_directory_insects)
-species_list_weeds=get_species_list_from_db(default_persist_directory_weeds)
+species_list=get_species_list_from_db(default_persist_directory)
 # default_persist_directory = 'vector-databases/db5-pre-completion' # For Development
 csv_filepath1 = "./agllm-data/corrected/Corrected_supplemented-insect_data-2500-sorted.xlsx"
 csv_filepath2 = "./agllm-data/corrected/Corrected_supplemented-insect_data-remaining.xlsx"
@@ -251,7 +249,7 @@ filter = {
 # )
 # 
 
-def initialize_qa_chain(specie_selector, application_mode, model_name, region, database_persistent_directory=default_persist_directory_insects, domain_name="Insects"):
+def initialize_qa_chain(specie_selector, application_mode, model_name, region, database_persistent_directory=default_persist_directory):
     # Add helper function for India info (kept for potential future use, but removed from RAG prompt)
     def read_and_format_filtered_csv_better(dataframe_given, insect_specie):
             filtered_data = dataframe_given[dataframe_given['species'] == insect_specie]
@@ -325,7 +323,7 @@ def initialize_qa_chain(specie_selector, application_mode, model_name, region, d
             availability_message = f"Information for **{specie_selector}** is available in region(s): **{', '.join(available_regions_list)}**."
         else:
             available_regions_list = []
-            availability_message = f"No regional information found for **{specie_selector}** in the {domain_name} database."
+            availability_message = f"No regional information found for **{specie_selector}** in the database."
     except Exception as e:
         print(f"Error checking region availability: {e}")
         available_regions_list = []
@@ -401,7 +399,7 @@ def initialize_qa_chain(specie_selector, application_mode, model_name, region, d
     
     # Updated prompt template for multi-part response with region-specific contexts
     general_system_template = f"""
-You are an AI assistant specialized in providing information about {domain_name.lower()} ({specie_selector}). The user is primarily interested in the '{region}' region.
+You are an AI assistant specialized in providing information about agricultural pests ({specie_selector}). The user is primarily interested in the '{region}' region.
 
 The following context has been retrieved from a database organized by region:
 
@@ -454,7 +452,7 @@ IMPORTANT:
 4. Apply this language constraint: {language_constraint}
 5. Keep your summaries concise and directly related to the user's question.
 
-User Question about {specie_selector} ({domain_name}): {{question}}
+User Question about {specie_selector}: {{question}}
 """
 
     class RegionFormattingLLMChain:
@@ -530,15 +528,6 @@ with gr.Blocks(theme=gr.themes.Soft())  as demo:
     with gr.Row():
         with gr.Column(scale=1):
             with gr.Row():
-                    domain_name = gr.Dropdown(
-                        list(["Insects", "Weeds"]),
-                        value="Insects",
-                        label="Domain",
-                        info="Select Domain",
-                        interactive=True,
-                        scale=1,
-                        visible=True
-                    )
                     region_selector = gr.Dropdown(
                         list(["United States", "India", "Africa"]), # Updated regions
                         value="United States", # Updated default
@@ -551,8 +540,8 @@ with gr.Blocks(theme=gr.themes.Soft())  as demo:
 
                     # Model selection
                     specie_selector = gr.Dropdown(
-                        list(set(species_list_insects)),
-                        value=species_list_insects[0],
+                        list(set(species_list)),
+                        value=species_list[0],
                         label="Species",
                         info="Select the Species",
                         interactive=True,
@@ -578,7 +567,7 @@ with gr.Blocks(theme=gr.themes.Soft())  as demo:
                         scale=1,
                         visible=True
                     )
-            region_availability_display = gr.Markdown(value="Select species/domain to see region availability.") # Added display area
+            region_availability_display = gr.Markdown(value="Select species to see region availability.") # Added display area
 
         with gr.Column(scale=2):
             # User input prompt text field
@@ -599,13 +588,13 @@ with gr.Blocks(theme=gr.themes.Soft())  as demo:
             return history + [["Invalid prompts - user prompt cannot be empty", None]]
 
     # Chatbot logic for configuration, sending the prompts, rendering the streamed back generations, etc.
-    def bot(model_name, application_mode, user_prompt_message, history, messages_history, qa_chain, domain_name, region): # Removed use_rag
+    def bot(model_name, application_mode, user_prompt_message, history, messages_history, qa_chain, region): # Removed use_rag
         if qa_chain == None:
             # Initial QA chain setup if not already done (uses default species for the selected domain)
-            initial_species = species_list_insects[0] if domain_name == "Insects" else species_list_weeds[0]
+            initial_species = species_list[0]
             # Need to handle the tuple returned by init_qa_chain now
             # Use the currently selected region for initialization if qa_chain is None
-            qa_chain, _ = init_qa_chain(initial_species, application_mode, model_name, domain_name, region) # Pass region
+            qa_chain, _ = init_qa_chain(initial_species, application_mode, model_name, region) # Pass region
 
         history[-1][1] = ""  # Placeholder for the answer
 
@@ -640,19 +629,12 @@ with gr.Blocks(theme=gr.themes.Soft())  as demo:
     def input_cleanup():
         return ""
 
-    def init_qa_chain(specie_selector, application_mode, model_name, domain_name, region): # Removed use_rag
-        print(f"--- init_qa_chain wrapper called with domain: '{domain_name}' ---") # DIAGNOSTIC PRINT
+    def init_qa_chain(specie_selector, application_mode, model_name, region): # Removed use_rag
+        print(f"--- init_qa_chain wrapper called ---") # DIAGNOSTIC PRINT
         qa_chain_instance = None
         availability_msg = "Error initializing QA chain."
         try:
-            if domain_name=="Insects":
-                qa_chain_instance, availability_msg = initialize_qa_chain(specie_selector, application_mode, model_name, region, default_persist_directory_insects, domain_name) # Removed use_rag
-            elif domain_name=="Weeds":
-                qa_chain_instance, availability_msg = initialize_qa_chain(specie_selector, application_mode, model_name, region, default_persist_directory_weeds, domain_name) # Removed use_rag
-            else:
-                print("No Appropriate Chain Selected")
-                availability_msg = "Invalid domain selected."
-                # Return None for chain and the message
+            qa_chain_instance, availability_msg = initialize_qa_chain(specie_selector, application_mode, model_name, region) # Removed use_rag
         except Exception as e:
             print(f"Error in init_qa_chain wrapper: {e}")
             availability_msg = f"Error initializing: {e}"
@@ -660,7 +642,7 @@ with gr.Blocks(theme=gr.themes.Soft())  as demo:
         return qa_chain_instance, availability_msg # Return both chain and message
 
     # Update QA chain AND availability message when relevant inputs change
-    inputs_for_qa_chain = [specie_selector, application_mode, model_name, domain_name, region_selector] # CORRECT ORDER
+    inputs_for_qa_chain = [specie_selector, application_mode, model_name, region_selector] # CORRECT ORDER
     outputs_for_qa_chain = [qa_chain_state, region_availability_display]
 
     # specie_selector.change(init_qa_chain, inputs=inputs_for_qa_chain, outputs=outputs_for_qa_chain)
@@ -672,20 +654,8 @@ with gr.Blocks(theme=gr.themes.Soft())  as demo:
     model_name.change(init_qa_chain, inputs=inputs_for_qa_chain, outputs=outputs_for_qa_chain)
     region_selector.change(init_qa_chain, inputs=inputs_for_qa_chain, outputs=outputs_for_qa_chain)
     application_mode.change(init_qa_chain, inputs=inputs_for_qa_chain, outputs=outputs_for_qa_chain)
-    domain_name.change(init_qa_chain, inputs=inputs_for_qa_chain, outputs=outputs_for_qa_chain)
 
     #####
-    def update_species_list(domain):
-        if domain == "Insects":
-            return gr.Dropdown( species_list_insects, value=species_list_insects[0], label="Species", info="Select the Species", interactive=True, scale=1, visible=True )
-        elif domain == "Weeds":
-            return gr.Dropdown( species_list_weeds, value=species_list_weeds[0], label="Species", info="Select the Species", interactive=True, scale=1, visible=True )
-
-    domain_name.change(
-        update_species_list,
-        inputs=[domain_name],
-        outputs=[specie_selector]
-    )
 
     # When the user clicks Enter and the user message is submitted
     user_prompt_message.submit(
@@ -695,7 +665,7 @@ with gr.Blocks(theme=gr.themes.Soft())  as demo:
         queue=False
     ).then(
         bot,
-        [model_name, application_mode, user_prompt_message, chatbot, state, qa_chain_state, domain_name, region_selector], # Removed use_rag
+        [model_name, application_mode, user_prompt_message, chatbot, state, qa_chain_state, region_selector], # Removed use_rag
         [chatbot, state]
     ).then(input_cleanup,
         [],
@@ -711,7 +681,7 @@ with gr.Blocks(theme=gr.themes.Soft())  as demo:
         queue=False
     ).then(
         bot,
-        [model_name, application_mode, user_prompt_message, chatbot, state, qa_chain_state, domain_name, region_selector], # Removed use_rag
+        [model_name, application_mode, user_prompt_message, chatbot, state, qa_chain_state, region_selector], # Removed use_rag
         [chatbot, state]
     ).then(
         input_cleanup,

app_database_prep.py

@@ -101,14 +101,24 @@ def process_excel_sheet(
 
 # --- Main Script Logic ---
 
-# loader = DirectoryLoader('./agllm-data/', glob="./*.pdf", loader_cls=PyMuPDFLoader)
-# loader = DirectoryLoader('/u/marshad/data/agllm-data/', glob='**/*.pdf', loader_cls=PyMuPDFLoader)
-data_domain_identifier="agllm-data-isu-field-insects-all-species"
-persist_directory = f'vector-databases-deployed/db5-{data_domain_identifier}' # was full
-loader = DirectoryLoader(f'agllm-data/{data_domain_identifier}', glob='**/*.pdf', loader_cls=PyMuPDFLoader)#,# was full, loader_kwargs={'chunk_size':512})
-chunk_size_input=512 
-metadata_raw = pd.read_csv(f"./agllm-data/{data_domain_identifier}/matched_species_results_v2.csv")
-documents = loader.load()
+# --- INSECTS DATA PROCESSING ---
+insects_data_domain_identifier = "agllm-data-isu-field-insects-all-species"
+persist_directory = f'vector-databases-deployed/db5-{insects_data_domain_identifier}'
+insects_loader = DirectoryLoader(f'agllm-data/{insects_data_domain_identifier}', glob='**/*.pdf', loader_cls=PyMuPDFLoader)
+chunk_size_input = 512
+insects_metadata_raw = pd.read_csv(f"./agllm-data/{insects_data_domain_identifier}/matched_species_results_v2.csv")
+insects_documents = insects_loader.load()
+
+# --- WEEDS DATA PROCESSING ---
+weeds_data_domain_identifier = "agllm-data-isu-field-weeds-all-species"
+weeds_loader = DirectoryLoader(f'agllm-data/{weeds_data_domain_identifier}', glob='**/*.pdf', loader_cls=PyMuPDFLoader)
+weeds_metadata_raw = pd.read_csv(f"./agllm-data/{weeds_data_domain_identifier}/matched_species_results_v2.csv")
+weeds_documents = weeds_loader.load()
+
+# Combine documents from both sources before processing
+documents = insects_documents + weeds_documents
+metadata_raw = pd.concat([insects_metadata_raw, weeds_metadata_raw], ignore_index=True)
+
 
 ## Load Excel File Path (Define once)
 excel_file_path = "agllm-data/PestID Species.xlsx"

vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species/chroma.sqlite3

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e4c853859ede8bd23b2d2fa7a6754c568b5b55ee5cb1263176179372d6e3837
-size 5844992
+oid sha256:d7b0db4eaef0c4bc975ba1ce231b7a2bcb3134b13139fe7586a58d5dcc1f97e9
+size 9072640

vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species/{34b62619-98c9-4834-894c-fbbb81d34690 → e82d58e5-16f1-41a6-9289-211464329861}/data_level0.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f18abd8c514282db82706e52b0a33ed659cd534e925a6f149deb7af9ce34bd8e
-size 6284000
+oid sha256:bb76ae0f6ca830a9a048b0cf53962a78c88c5c7fcda63fc846077d3456eb3890
+size 62840000

vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species/{34b62619-98c9-4834-894c-fbbb81d34690 → e82d58e5-16f1-41a6-9289-211464329861}/header.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:effaa959ce2b30070fdafc2fe82096fc46e4ee7561b75920dd3ce43d09679b21
+oid sha256:ec666c9828420c69fc6b597461d8c18487becec1527c7d1cff9b898cbb393c2d
 size 100

vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species/{34b62619-98c9-4834-894c-fbbb81d34690 → e82d58e5-16f1-41a6-9289-211464329861}/length.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc19b1997119425765295aeab72d76faa6927d4f83985d328c26f20468d6cc76
-size 4000
+oid sha256:e7e2dcff542de95352682dc186432e98f0188084896773f1973276b0577d5305
+size 40000

vector-databases-deployed/db5-agllm-data-isu-field-weeds-all-species/chroma.sqlite3

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d6c522f18861ff080a6073fa9c7fcfd1e08d5e65f1b64c8748b52c851387f0d7
-size 155648
+oid sha256:9bdbd2acc8e88eaf4e6b6bc8a8687b36fe5c624b986515bad74e37010f3924ba
+size 167936