Add remaining files from agllm-development state

.cursorignore

@@ -0,0 +1,5 @@
+# Add directories or file patterns to ignore during indexing (e.g. foo/ or *.csv)
+*.ipynb
+*.xlsx
+*.csv
+ 

.gitattributes

@@ -0,0 +1,40 @@
+*.xlsx filter=lfs diff=lfs merge=lfs -text
+db5/** filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+vector-databases-deployed/** filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,42 @@
+# First exclude everything in agllm-data
+agllm-data/*
+
+# Then explicitly include the corrected and india directories and their contents
+!agllm-data/corrected/
+!agllm-data/corrected/**
+!agllm-data/india/
+!agllm-data/india/**
+
+# Other exclusions
+agllm/
+db3/
+# db5/
+db4/
+mlruns/
+/agllm-data.zip
+.env
+vector-databases/
+vector-databases-deployement/
+writing/
+# but include the analysis folder
+!writing/*/analysis/
+# Include specific file
+!writing/65d4fadc59fceb1a54d1aae6/main.tex
+*.pdf
+
+# Chloropleth related files
+chloropleth/
+chloropleth_backup/
+*.shp
+*.png
+
+# Vector database backups
+vector-databases-deployed-backup/
+*.sqlite3
+
+# Binary files
+*.pyc
+__pycache__/
+.DS_Store
+
+vector-databases-deployed/

.vscode/settings.json

@@ -0,0 +1,2 @@
+{
+}

AgLLM.code-workspace

@@ -0,0 +1,8 @@
+{
+	"folders": [
+		{
+			"path": "."
+		}
+	],
+	"settings": {}
+}

agllm-data/corrected/Corrected_supplemented-insect_data-2500-sorted.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6aef24ba96ca9be054b4f87344d93d3f8f59cbd280ff29d45991977482067717
+size 1185132

agllm-data/corrected/Corrected_supplemented-insect_data-remaining.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc953489e31d114f689783aaa50d1e23ce3fcc58618a9b22562e0cad5750af57
+size 980300

agllm-data/corrected/Corrected_weed-prepared_results-all.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e97a0eaa24c101b940d48f1dd1e11345246e2455521b2d79a45c3da108c63195
+size 602025

agllm-data/india/species.csv

@@ -0,0 +1,2 @@
+Common Name,species,ETL,Alternate Hosts,Management,Remarks
+Brown Marmorated Stink Bug,Halyomorpha Halys,,horticulture and field crops,"Crush any eggs nymphs or adults you find on plants. Repeat daily for several days.|Botanical extracts: Spray plants with a mixture of 50 g of neem seed extract in 2 L of water. Boil for 15 minutes let cool and spray three times at two-week intervals.|Soapy water: Spray plants with soapy water to remove insects",

api_request_parallel_processor.py

@@ -0,0 +1,508 @@
+"""
+API REQUEST PARALLEL PROCESSOR
+
+Using the OpenAI API to process lots of text quickly takes some care.
+If you trickle in a million API requests one by one, they'll take days to complete.
+If you flood a million API requests in parallel, they'll exceed the rate limits and fail with errors.
+To maximize throughput, parallel requests need to be throttled to stay under rate limits.
+
+This script parallelizes requests to the OpenAI API while throttling to stay under rate limits.
+
+Features:
+- Streams requests from file, to avoid running out of memory for giant jobs
+- Makes requests concurrently, to maximize throughput
+- Throttles request and token usage, to stay under rate limits
+- Retries failed requests up to {max_attempts} times, to avoid missing data
+- Logs errors, to diagnose problems with requests
+
+Example command to call script:
+```
+python examples/api_request_parallel_processor.py \
+  --requests_filepath examples/data/example_requests_to_parallel_process.jsonl \
+  --save_filepath examples/data/example_requests_to_parallel_process_results.jsonl \
+  --request_url https://api.openai.com/v1/embeddings \
+  --max_requests_per_minute 1500 \
+  --max_tokens_per_minute 6250000 \
+  --token_encoding_name cl100k_base \
+  --max_attempts 5 \
+  --logging_level 20
+```
+
+Inputs:
+- requests_filepath : str
+    - path to the file containing the requests to be processed
+    - file should be a jsonl file, where each line is a json object with API parameters and an optional metadata field
+    - e.g., {"model": "text-embedding-ada-002", "input": "embed me", "metadata": {"row_id": 1}}
+    - as with all jsonl files, take care that newlines in the content are properly escaped (json.dumps does this automatically)
+    - an example file is provided at examples/data/example_requests_to_parallel_process.jsonl
+    - the code to generate the example file is appended to the bottom of this script
+- save_filepath : str, optional
+    - path to the file where the results will be saved
+    - file will be a jsonl file, where each line is an array with the original request plus the API response
+    - e.g., [{"model": "text-embedding-ada-002", "input": "embed me"}, {...}]
+    - if omitted, results will be saved to {requests_filename}_results.jsonl
+- request_url : str, optional
+    - URL of the API endpoint to call
+    - if omitted, will default to "https://api.openai.com/v1/embeddings"
+- api_key : str, optional
+    - API key to use
+    - if omitted, the script will attempt to read it from an environment variable {os.getenv("OPENAI_API_KEY")}
+- max_requests_per_minute : float, optional
+    - target number of requests to make per minute (will make less if limited by tokens)
+    - leave headroom by setting this to 50% or 75% of your limit
+    - if requests are limiting you, try batching multiple embeddings or completions into one request
+    - if omitted, will default to 1,500
+- max_tokens_per_minute : float, optional
+    - target number of tokens to use per minute (will use less if limited by requests)
+    - leave headroom by setting this to 50% or 75% of your limit
+    - if omitted, will default to 125,000
+- token_encoding_name : str, optional
+    - name of the token encoding used, as defined in the `tiktoken` package
+    - if omitted, will default to "cl100k_base" (used by `text-embedding-ada-002`)
+- max_attempts : int, optional
+    - number of times to retry a failed request before giving up
+    - if omitted, will default to 5
+- logging_level : int, optional
+    - level of logging to use; higher numbers will log fewer messages
+    - 40 = ERROR; will log only when requests fail after all retries
+    - 30 = WARNING; will log when requests his rate limits or other errors
+    - 20 = INFO; will log when requests start and the status at finish
+    - 10 = DEBUG; will log various things as the loop runs to see when they occur
+    - if omitted, will default to 20 (INFO).
+
+The script is structured as follows:
+    - Imports
+    - Define main()
+        - Initialize things
+        - In main loop:
+            - Get next request if one is not already waiting for capacity
+            - Update available token & request capacity
+            - If enough capacity available, call API
+            - The loop pauses if a rate limit error is hit
+            - The loop breaks when no tasks remain
+    - Define dataclasses
+        - StatusTracker (stores script metadata counters; only one instance is created)
+        - APIRequest (stores API inputs, outputs, metadata; one method to call API)
+    - Define functions
+        - api_endpoint_from_url (extracts API endpoint from request URL)
+        - append_to_jsonl (writes to results file)
+        - num_tokens_consumed_from_request (bigger function to infer token usage from request)
+        - task_id_generator_function (yields 0, 1, 2, ...)
+    - Run main()
+"""
+
+# imports
+import aiohttp  # for making API calls concurrently
+import argparse  # for running script from command line
+import asyncio  # for running API calls concurrently
+import json  # for saving results to a jsonl file
+import logging  # for logging rate limit warnings and other messages
+import os  # for reading API key
+import re  # for matching endpoint from request URL
+import tiktoken  # for counting tokens
+import time  # for sleeping after rate limit is hit
+from dataclasses import (
+    dataclass,
+    field,
+)  # for storing API inputs, outputs, and metadata
+
+
+async def process_api_requests_from_file(
+    requests_filepath: str,
+    save_filepath: str,
+    request_url: str,
+    api_key: str,
+    max_requests_per_minute: float,
+    max_tokens_per_minute: float,
+    token_encoding_name: str,
+    max_attempts: int,
+    logging_level: int,
+):
+    """Processes API requests in parallel, throttling to stay under rate limits."""
+    # constants
+    seconds_to_pause_after_rate_limit_error = 15
+    seconds_to_sleep_each_loop = (
+        0.001  # 1 ms limits max throughput to 1,000 requests per second
+    )
+
+    # initialize logging
+    logging.basicConfig(level=logging_level)
+    logging.debug(f"Logging initialized at level {logging_level}")
+
+    # infer API endpoint and construct request header
+    api_endpoint = api_endpoint_from_url(request_url)
+    request_header = {"Authorization": f"Bearer {api_key}"}
+    # use api-key header for Azure deployments
+    if '/deployments' in request_url:
+        request_header = {"api-key": f"{api_key}"}
+
+    # initialize trackers
+    queue_of_requests_to_retry = asyncio.Queue()
+    task_id_generator = (
+        task_id_generator_function()
+    )  # generates integer IDs of 0, 1, 2, ...
+    status_tracker = (
+        StatusTracker()
+    )  # single instance to track a collection of variables
+    next_request = None  # variable to hold the next request to call
+
+    # initialize available capacity counts
+    available_request_capacity = max_requests_per_minute
+    available_token_capacity = max_tokens_per_minute
+    last_update_time = time.time()
+
+    # initialize flags
+    file_not_finished = True  # after file is empty, we'll skip reading it
+    logging.debug(f"Initialization complete.")
+
+    # initialize file reading
+    with open(requests_filepath) as file:
+        # `requests` will provide requests one at a time
+        requests = file.__iter__()
+        logging.debug(f"File opened. Entering main loop")
+        async with aiohttp.ClientSession() as session:  # Initialize ClientSession here
+            while True:
+                # get next request (if one is not already waiting for capacity)
+                if next_request is None:
+                    if not queue_of_requests_to_retry.empty():
+                        next_request = queue_of_requests_to_retry.get_nowait()
+                        logging.debug(
+                            f"Retrying request {next_request.task_id}: {next_request}"
+                        )
+                    elif file_not_finished:
+                        try:
+                            # get new request
+                            request_json = json.loads(next(requests))
+                            next_request = APIRequest(
+                                task_id=next(task_id_generator),
+                                request_json=request_json,
+                                token_consumption=num_tokens_consumed_from_request(
+                                    request_json, api_endpoint, token_encoding_name
+                                ),
+                                attempts_left=max_attempts,
+                                metadata=request_json.pop("metadata", None),
+                            )
+                            status_tracker.num_tasks_started += 1
+                            status_tracker.num_tasks_in_progress += 1
+                            logging.debug(
+                                f"Reading request {next_request.task_id}: {next_request}"
+                            )
+                        except StopIteration:
+                            # if file runs out, set flag to stop reading it
+                            logging.debug("Read file exhausted")
+                            file_not_finished = False
+
+                # update available capacity
+                current_time = time.time()
+                seconds_since_update = current_time - last_update_time
+                available_request_capacity = min(
+                    available_request_capacity
+                    + max_requests_per_minute * seconds_since_update / 60.0,
+                    max_requests_per_minute,
+                )
+                available_token_capacity = min(
+                    available_token_capacity
+                    + max_tokens_per_minute * seconds_since_update / 60.0,
+                    max_tokens_per_minute,
+                )
+                last_update_time = current_time
+
+                # if enough capacity available, call API
+                if next_request:
+                    next_request_tokens = next_request.token_consumption
+                    if (
+                        available_request_capacity >= 1
+                        and available_token_capacity >= next_request_tokens
+                    ):
+                        # update counters
+                        available_request_capacity -= 1
+                        available_token_capacity -= next_request_tokens
+                        next_request.attempts_left -= 1
+
+                        # call API
+                        asyncio.create_task(
+                            next_request.call_api(
+                                session=session,
+                                request_url=request_url,
+                                request_header=request_header,
+                                retry_queue=queue_of_requests_to_retry,
+                                save_filepath=save_filepath,
+                                status_tracker=status_tracker,
+                            )
+                        )
+                        next_request = None  # reset next_request to empty
+
+                # if all tasks are finished, break
+                if status_tracker.num_tasks_in_progress == 0:
+                    break
+
+                # main loop sleeps briefly so concurrent tasks can run
+                await asyncio.sleep(seconds_to_sleep_each_loop)
+
+                # if a rate limit error was hit recently, pause to cool down
+                seconds_since_rate_limit_error = (
+                    time.time() - status_tracker.time_of_last_rate_limit_error
+                )
+                if (
+                    seconds_since_rate_limit_error
+                    < seconds_to_pause_after_rate_limit_error
+                ):
+                    remaining_seconds_to_pause = (
+                        seconds_to_pause_after_rate_limit_error
+                        - seconds_since_rate_limit_error
+                    )
+                    await asyncio.sleep(remaining_seconds_to_pause)
+                    # ^e.g., if pause is 15 seconds and final limit was hit 5 seconds ago
+                    logging.warn(
+                        f"Pausing to cool down until {time.ctime(status_tracker.time_of_last_rate_limit_error + seconds_to_pause_after_rate_limit_error)}"
+                    )
+
+        # after finishing, log final status
+        logging.info(
+            f"""Parallel processing complete. Results saved to {save_filepath}"""
+        )
+        if status_tracker.num_tasks_failed > 0:
+            logging.warning(
+                f"{status_tracker.num_tasks_failed} / {status_tracker.num_tasks_started} requests failed. Errors logged to {save_filepath}."
+            )
+        if status_tracker.num_rate_limit_errors > 0:
+            logging.warning(
+                f"{status_tracker.num_rate_limit_errors} rate limit errors received. Consider running at a lower rate."
+            )
+
+
+# dataclasses
+
+
+@dataclass
+class StatusTracker:
+    """Stores metadata about the script's progress. Only one instance is created."""
+
+    num_tasks_started: int = 0
+    num_tasks_in_progress: int = 0  # script ends when this reaches 0
+    num_tasks_succeeded: int = 0
+    num_tasks_failed: int = 0
+    num_rate_limit_errors: int = 0
+    num_api_errors: int = 0  # excluding rate limit errors, counted above
+    num_other_errors: int = 0
+    time_of_last_rate_limit_error: int = 0  # used to cool off after hitting rate limits
+
+
+@dataclass
+class APIRequest:
+    """Stores an API request's inputs, outputs, and other metadata. Contains a method to make an API call."""
+
+    task_id: int
+    request_json: dict
+    token_consumption: int
+    attempts_left: int
+    metadata: dict
+    result: list = field(default_factory=list)
+
+    async def call_api(
+        self,
+        session: aiohttp.ClientSession,
+        request_url: str,
+        request_header: dict,
+        retry_queue: asyncio.Queue,
+        save_filepath: str,
+        status_tracker: StatusTracker,
+    ):
+        """Calls the OpenAI API and saves results."""
+        logging.info(f"Starting request #{self.task_id}")
+        error = None
+        try:
+            async with session.post(
+                url=request_url, headers=request_header, json=self.request_json
+            ) as response:
+                response = await response.json()
+            if "error" in response:
+                logging.warning(
+                    f"Request {self.task_id} failed with error {response['error']}"
+                )
+                status_tracker.num_api_errors += 1
+                error = response
+                if "Rate limit" in response["error"].get("message", ""):
+                    status_tracker.time_of_last_rate_limit_error = time.time()
+                    status_tracker.num_rate_limit_errors += 1
+                    status_tracker.num_api_errors -= (
+                        1  # rate limit errors are counted separately
+                    )
+
+        except (
+            Exception
+        ) as e:  # catching naked exceptions is bad practice, but in this case we'll log & save them
+            logging.warning(f"Request {self.task_id} failed with Exception {e}")
+            status_tracker.num_other_errors += 1
+            error = e
+        if error:
+            self.result.append(error)
+            if self.attempts_left:
+                retry_queue.put_nowait(self)
+            else:
+                logging.error(
+                    f"Request {self.request_json} failed after all attempts. Saving errors: {self.result}"
+                )
+                data = (
+                    [self.request_json, [str(e) for e in self.result], self.metadata]
+                    if self.metadata
+                    else [self.request_json, [str(e) for e in self.result]]
+                )
+                append_to_jsonl(data, save_filepath)
+                status_tracker.num_tasks_in_progress -= 1
+                status_tracker.num_tasks_failed += 1
+        else:
+            data = (
+                [self.request_json, response, self.metadata]
+                if self.metadata
+                else [self.request_json, response]
+            )
+            append_to_jsonl(data, save_filepath)
+            status_tracker.num_tasks_in_progress -= 1
+            status_tracker.num_tasks_succeeded += 1
+            logging.debug(f"Request {self.task_id} saved to {save_filepath}")
+
+
+# functions
+
+
+def api_endpoint_from_url(request_url):
+    """Extract the API endpoint from the request URL."""
+    match = re.search("^https://[^/]+/v\\d+/(.+)$", request_url)
+    if match is None:
+        # for Azure OpenAI deployment urls
+        match = re.search(r"^https://[^/]+/openai/deployments/[^/]+/(.+?)(\?|$)", request_url)
+    return match[1]
+
+
+def append_to_jsonl(data, filename: str) -> None:
+    """Append a json payload to the end of a jsonl file."""
+    json_string = json.dumps(data)
+    with open(filename, "a") as f:
+        f.write(json_string + "\n")
+
+
+def num_tokens_consumed_from_request(
+    request_json: dict,
+    api_endpoint: str,
+    token_encoding_name: str,
+):
+    """Count the number of tokens in the request. Only supports completion and embedding requests."""
+    encoding = tiktoken.get_encoding(token_encoding_name)
+    # if completions request, tokens = prompt + n * max_tokens
+    if api_endpoint.endswith("completions"):
+        max_tokens = request_json.get("max_tokens", 15)
+        n = request_json.get("n", 1)
+        completion_tokens = n * max_tokens
+
+        # chat completions
+        if api_endpoint.startswith("chat/"):
+            num_tokens = 0
+            for message in request_json["messages"]:
+                num_tokens += 4  # every message follows <im_start>{role/name}\n{content}<im_end>\n
+                for key, value in message.items():
+                    num_tokens += len(encoding.encode(value))
+                    if key == "name":  # if there's a name, the role is omitted
+                        num_tokens -= 1  # role is always required and always 1 token
+            num_tokens += 2  # every reply is primed with <im_start>assistant
+            return num_tokens + completion_tokens
+        # normal completions
+        else:
+            prompt = request_json["prompt"]
+            if isinstance(prompt, str):  # single prompt
+                prompt_tokens = len(encoding.encode(prompt))
+                num_tokens = prompt_tokens + completion_tokens
+                return num_tokens
+            elif isinstance(prompt, list):  # multiple prompts
+                prompt_tokens = sum([len(encoding.encode(p)) for p in prompt])
+                num_tokens = prompt_tokens + completion_tokens * len(prompt)
+                return num_tokens
+            else:
+                raise TypeError(
+                    'Expecting either string or list of strings for "prompt" field in completion request'
+                )
+    # if embeddings request, tokens = input tokens
+    elif api_endpoint == "embeddings":
+        input = request_json["input"]
+        if isinstance(input, str):  # single input
+            num_tokens = len(encoding.encode(input))
+            return num_tokens
+        elif isinstance(input, list):  # multiple inputs
+            num_tokens = sum([len(encoding.encode(i)) for i in input])
+            return num_tokens
+        else:
+            raise TypeError(
+                'Expecting either string or list of strings for "inputs" field in embedding request'
+            )
+    # more logic needed to support other API calls (e.g., edits, inserts, DALL-E)
+    else:
+        raise NotImplementedError(
+            f'API endpoint "{api_endpoint}" not implemented in this script'
+        )
+
+
+def task_id_generator_function():
+    """Generate integers 0, 1, 2, and so on."""
+    task_id = 0
+    while True:
+        yield task_id
+        task_id += 1
+
+
+# run script
+
+
+if __name__ == "__main__":
+    # parse command line arguments
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--requests_filepath")
+    parser.add_argument("--save_filepath", default=None)
+    parser.add_argument("--request_url", default="https://api.openai.com/v1/embeddings")
+    parser.add_argument("--api_key", default=os.getenv("OPENAI_API_KEY"))
+    parser.add_argument("--max_requests_per_minute", type=int, default=3_000 * 0.5)
+    parser.add_argument("--max_tokens_per_minute", type=int, default=250_000 * 0.5)
+    parser.add_argument("--token_encoding_name", default="cl100k_base")
+    parser.add_argument("--max_attempts", type=int, default=5)
+    parser.add_argument("--logging_level", default=logging.INFO)
+    args = parser.parse_args()
+
+    if args.save_filepath is None:
+        args.save_filepath = args.requests_filepath.replace(".jsonl", "_results.jsonl")
+
+    # run script
+    asyncio.run(
+        process_api_requests_from_file(
+            requests_filepath=args.requests_filepath,
+            save_filepath=args.save_filepath,
+            request_url=args.request_url,
+            api_key=args.api_key,
+            max_requests_per_minute=float(args.max_requests_per_minute),
+            max_tokens_per_minute=float(args.max_tokens_per_minute),
+            token_encoding_name=args.token_encoding_name,
+            max_attempts=int(args.max_attempts),
+            logging_level=int(args.logging_level),
+        )
+    )
+
+
+"""
+APPENDIX
+
+The example requests file at openai-cookbook/examples/data/example_requests_to_parallel_process.jsonl contains 10,000 requests to text-embedding-ada-002.
+
+It was generated with the following code:
+
+```python
+import json
+
+filename = "data/example_requests_to_parallel_process.jsonl"
+n_requests = 10_000
+jobs = [{"model": "text-embedding-ada-002", "input": str(x) + "\n"} for x in range(n_requests)]
+with open(filename, "w") as f:
+    for job in jobs:
+        json_string = json.dumps(job)
+        f.write(json_string + "\n")
+```
+
+As with all jsonl files, take care that newlines in the content are properly escaped (json.dumps does this automatically).
+"""

api_request_parallel_processor_anthropic.py

@@ -0,0 +1,473 @@
+print("IMPORTED")
+"""
+API REQUEST PARALLEL PROCESSOR
+
+Using the OpenAI API to process lots of text quickly takes some care.
+If you trickle in a million API requests one by one, they'll take days to complete.
+If you flood a million API requests in parallel, they'll exceed the rate limits and fail with errors.
+To maximize throughput, parallel requests need to be throttled to stay under rate limits.
+
+This script parallelizes requests to the OpenAI API while throttling to stay under rate limits.
+
+Features:
+- Streams requests from file, to avoid running out of memory for giant jobs
+- Makes requests concurrently, to maximize throughput
+- Throttles request and token usage, to stay under rate limits
+- Retries failed requests up to {max_attempts} times, to avoid missing data
+- Logs errors, to diagnose problems with requests
+
+Example command to call script:
+```
+python examples/api_request_parallel_processor.py \
+  --requests_filepath examples/data/example_requests_to_parallel_process.jsonl \
+  --save_filepath examples/data/example_requests_to_parallel_process_results.jsonl \
+  --request_url https://api.openai.com/v1/embeddings \
+  --max_requests_per_minute 1500 \
+  --max_tokens_per_minute 6250000 \
+  --token_encoding_name cl100k_base \
+  --max_attempts 5 \
+  --logging_level 20
+```
+
+Inputs:
+- requests_filepath : str
+    - path to the file containing the requests to be processed
+    - file should be a jsonl file, where each line is a json object with API parameters and an optional metadata field
+    - e.g., {"model": "text-embedding-ada-002", "input": "embed me", "metadata": {"row_id": 1}}
+    - as with all jsonl files, take care that newlines in the content are properly escaped (json.dumps does this automatically)
+    - an example file is provided at examples/data/example_requests_to_parallel_process.jsonl
+    - the code to generate the example file is appended to the bottom of this script
+- save_filepath : str, optional
+    - path to the file where the results will be saved
+    - file will be a jsonl file, where each line is an array with the original request plus the API response
+    - e.g., [{"model": "text-embedding-ada-002", "input": "embed me"}, {...}]
+    - if omitted, results will be saved to {requests_filename}_results.jsonl
+- request_url : str, optional
+    - URL of the API endpoint to call
+    - if omitted, will default to "https://api.openai.com/v1/embeddings"
+- api_key : str, optional
+    - API key to use
+    - if omitted, the script will attempt to read it from an environment variable {os.getenv("OPENAI_API_KEY")}
+- max_requests_per_minute : float, optional
+    - target number of requests to make per minute (will make less if limited by tokens)
+    - leave headroom by setting this to 50% or 75% of your limit
+    - if requests are limiting you, try batching multiple embeddings or completions into one request
+    - if omitted, will default to 1,500
+- max_tokens_per_minute : float, optional
+    - target number of tokens to use per minute (will use less if limited by requests)
+    - leave headroom by setting this to 50% or 75% of your limit
+    - if omitted, will default to 125,000
+- token_encoding_name : str, optional
+    - name of the token encoding used, as defined in the `tiktoken` package
+    - if omitted, will default to "cl100k_base" (used by `text-embedding-ada-002`)
+- max_attempts : int, optional
+    - number of times to retry a failed request before giving up
+    - if omitted, will default to 5
+- logging_level : int, optional
+    - level of logging to use; higher numbers will log fewer messages
+    - 40 = ERROR; will log only when requests fail after all retries
+    - 30 = WARNING; will log when requests his rate limits or other errors
+    - 20 = INFO; will log when requests start and the status at finish
+    - 10 = DEBUG; will log various things as the loop runs to see when they occur
+    - if omitted, will default to 20 (INFO).
+
+The script is structured as follows:
+    - Imports
+    - Define main()
+        - Initialize things
+        - In main loop:
+            - Get next request if one is not already waiting for capacity
+            - Update available token & request capacity
+            - If enough capacity available, call API
+            - The loop pauses if a rate limit error is hit
+            - The loop breaks when no tasks remain
+    - Define dataclasses
+        - StatusTracker (stores script metadata counters; only one instance is created)
+        - APIRequest (stores API inputs, outputs, metadata; one method to call API)
+    - Define functions
+        - api_endpoint_from_url (extracts API endpoint from request URL)
+        - append_to_jsonl (writes to results file)
+        - num_tokens_consumed_from_request (bigger function to infer token usage from request)
+        - task_id_generator_function (yields 0, 1, 2, ...)
+    - Run main()
+"""
+
+# imports
+import aiohttp  # for making API calls concurrently
+import argparse  # for running script from command line
+import asyncio  # for running API calls concurrently
+import json  # for saving results to a jsonl file
+import logging  # for logging rate limit warnings and other messages
+import os  # for reading API key
+import re  # for matching endpoint from request URL
+import tiktoken  # for counting tokens
+import time  # for sleeping after rate limit is hit
+from dataclasses import (
+    dataclass,
+    field,
+)  # for storing API inputs, outputs, and metadata
+
+
+async def process_api_requests_from_file(
+    requests_filepath: str,
+    save_filepath: str,
+    request_url: str,
+    api_key: str,
+    max_requests_per_minute: float,
+    max_tokens_per_minute: float,
+    token_encoding_name: str,
+    max_attempts: int,
+    logging_level: int,
+):
+    """Processes API requests in parallel, throttling to stay under rate limits."""
+    # constants
+    seconds_to_pause_after_rate_limit_error = 15
+    seconds_to_sleep_each_loop = (
+        0.001  # 1 ms limits max throughput to 1,000 requests per second
+    )
+
+    # initialize logging
+    logging.basicConfig(level=logging_level)
+    logging.debug(f"Logging initialized at level {logging_level}")
+
+    # infer API endpoint and construct request header
+    api_endpoint = api_endpoint_from_url(request_url)
+    # request_header = {"Authorization": f"Bearer {api_key}"}
+    request_header = {
+    "x-api-key": api_key,
+    "anthropic-version": "2023-06-01",
+    "content-type": "application/json",
+    }
+    # use api-key header for Azure deployments
+    if '/deployments' in request_url:
+        request_header = {"api-key": f"{api_key}"}
+
+    # initialize trackers
+    queue_of_requests_to_retry = asyncio.Queue()
+    task_id_generator = (
+        task_id_generator_function()
+    )  # generates integer IDs of 0, 1, 2, ...
+    status_tracker = (
+        StatusTracker()
+    )  # single instance to track a collection of variables
+    next_request = None  # variable to hold the next request to call
+
+    # initialize available capacity counts
+    available_request_capacity = max_requests_per_minute
+    available_token_capacity = max_tokens_per_minute
+    last_update_time = time.time()
+
+    # initialize flags
+    file_not_finished = True  # after file is empty, we'll skip reading it
+    logging.debug(f"Initialization complete.")
+
+    # initialize file reading
+    with open(requests_filepath) as file:
+        # `requests` will provide requests one at a time
+        requests = file.__iter__()
+        logging.debug(f"File opened. Entering main loop")
+        async with aiohttp.ClientSession() as session:  # Initialize ClientSession here
+            while True:
+                # get next request (if one is not already waiting for capacity)
+                if next_request is None:
+                    if not queue_of_requests_to_retry.empty():
+                        next_request = queue_of_requests_to_retry.get_nowait()
+                        logging.debug(
+                            f"Retrying request {next_request.task_id}: {next_request}"
+                        )
+                    elif file_not_finished:
+                        try:
+                            # get new request
+                            request_json = json.loads(next(requests))
+                            next_request = APIRequest(
+                                task_id=next(task_id_generator),
+                                request_json=request_json,
+                                token_consumption=num_tokens_consumed_from_request(
+                                    request_json, api_endpoint, token_encoding_name
+                                ),
+                                attempts_left=max_attempts,
+                                metadata=request_json.pop("metadata", None),
+                            )
+                            status_tracker.num_tasks_started += 1
+                            status_tracker.num_tasks_in_progress += 1
+                            logging.debug(
+                                f"Reading request {next_request.task_id}: {next_request}"
+                            )
+                        except StopIteration:
+                            # if file runs out, set flag to stop reading it
+                            logging.debug("Read file exhausted")
+                            file_not_finished = False
+
+                # update available capacity
+                current_time = time.time()
+                seconds_since_update = current_time - last_update_time
+                available_request_capacity = min(
+                    available_request_capacity
+                    + max_requests_per_minute * seconds_since_update / 60.0,
+                    max_requests_per_minute,
+                )
+                available_token_capacity = min(
+                    available_token_capacity
+                    + max_tokens_per_minute * seconds_since_update / 60.0,
+                    max_tokens_per_minute,
+                )
+                last_update_time = current_time
+
+                # if enough capacity available, call API
+                if next_request:
+                    next_request_tokens = next_request.token_consumption
+                    if (
+                        available_request_capacity >= 1
+                        and available_token_capacity >= next_request_tokens
+                    ):
+                        # update counters
+                        available_request_capacity -= 1
+                        available_token_capacity -= next_request_tokens
+                        next_request.attempts_left -= 1
+
+                        # call API
+                        asyncio.create_task(
+                            next_request.call_api(
+                                session=session,
+                                request_url=request_url,
+                                request_header=request_header,
+                                retry_queue=queue_of_requests_to_retry,
+                                save_filepath=save_filepath,
+                                status_tracker=status_tracker,
+                            )
+                        )
+                        next_request = None  # reset next_request to empty
+
+                # if all tasks are finished, break
+                if status_tracker.num_tasks_in_progress == 0:
+                    break
+
+                # main loop sleeps briefly so concurrent tasks can run
+                await asyncio.sleep(seconds_to_sleep_each_loop)
+
+                # if a rate limit error was hit recently, pause to cool down
+                seconds_since_rate_limit_error = (
+                    time.time() - status_tracker.time_of_last_rate_limit_error
+                )
+                if (
+                    seconds_since_rate_limit_error
+                    < seconds_to_pause_after_rate_limit_error
+                ):
+                    remaining_seconds_to_pause = (
+                        seconds_to_pause_after_rate_limit_error
+                        - seconds_since_rate_limit_error
+                    )
+                    await asyncio.sleep(remaining_seconds_to_pause)
+                    # ^e.g., if pause is 15 seconds and final limit was hit 5 seconds ago
+                    logging.warn(
+                        f"Pausing to cool down until {time.ctime(status_tracker.time_of_last_rate_limit_error + seconds_to_pause_after_rate_limit_error)}"
+                    )
+
+        # after finishing, log final status
+        logging.info(
+            f"""Parallel processing complete. Results saved to {save_filepath}"""
+        )
+        if status_tracker.num_tasks_failed > 0:
+            logging.warning(
+                f"{status_tracker.num_tasks_failed} / {status_tracker.num_tasks_started} requests failed. Errors logged to {save_filepath}."
+            )
+        if status_tracker.num_rate_limit_errors > 0:
+            logging.warning(
+                f"{status_tracker.num_rate_limit_errors} rate limit errors received. Consider running at a lower rate."
+            )
+
+
+# dataclasses
+
+
+@dataclass
+class StatusTracker:
+    """Stores metadata about the script's progress. Only one instance is created."""
+
+    num_tasks_started: int = 0
+    num_tasks_in_progress: int = 0  # script ends when this reaches 0
+    num_tasks_succeeded: int = 0
+    num_tasks_failed: int = 0
+    num_rate_limit_errors: int = 0
+    num_api_errors: int = 0  # excluding rate limit errors, counted above
+    num_other_errors: int = 0
+    time_of_last_rate_limit_error: int = 0  # used to cool off after hitting rate limits
+
+
+@dataclass
+class APIRequest:
+    """Stores an API request's inputs, outputs, and other metadata. Contains a method to make an API call."""
+
+    task_id: int
+    request_json: dict
+    token_consumption: int
+    attempts_left: int
+    metadata: dict
+    result: list = field(default_factory=list)
+
+    async def call_api(
+        self,
+        session: aiohttp.ClientSession,
+        request_url: str,
+        request_header: dict,
+        retry_queue: asyncio.Queue,
+        save_filepath: str,
+        status_tracker: StatusTracker,
+    ):
+        """Calls the OpenAI API and saves results."""
+        logging.info(f"Starting request #{self.task_id}")
+        error = None
+        try:
+            async with session.post(
+                url=request_url, headers=request_header, json=self.request_json
+            ) as response:
+                response = await response.json()
+            if "error" in response:
+                logging.warning(
+                    f"Request {self.task_id} failed with error {response['error']}"
+                )
+                status_tracker.num_api_errors += 1
+                error = response
+                if "Rate limit" in response["error"].get("message", ""):
+                    status_tracker.time_of_last_rate_limit_error = time.time()
+                    status_tracker.num_rate_limit_errors += 1
+                    status_tracker.num_api_errors -= (
+                        1  # rate limit errors are counted separately
+                    )
+
+        except (
+            Exception
+        ) as e:  # catching naked exceptions is bad practice, but in this case we'll log & save them
+            logging.warning(f"Request {self.task_id} failed with Exception {e}")
+            status_tracker.num_other_errors += 1
+            error = e
+        if error:
+            self.result.append(error)
+            if self.attempts_left:
+                retry_queue.put_nowait(self)
+            else:
+                logging.error(
+                    f"Request {self.request_json} failed after all attempts. Saving errors: {self.result}"
+                )
+                data = (
+                    [self.request_json, [str(e) for e in self.result], self.metadata]
+                    if self.metadata
+                    else [self.request_json, [str(e) for e in self.result]]
+                )
+                append_to_jsonl(data, save_filepath)
+                status_tracker.num_tasks_in_progress -= 1
+                status_tracker.num_tasks_failed += 1
+        else:
+            data = (
+                [self.request_json, response, self.metadata]
+                if self.metadata
+                else [self.request_json, response]
+            )
+            append_to_jsonl(data, save_filepath)
+            status_tracker.num_tasks_in_progress -= 1
+            status_tracker.num_tasks_succeeded += 1
+            logging.debug(f"Request {self.task_id} saved to {save_filepath}")
+
+
+# functions
+
+
+def api_endpoint_from_url(request_url):
+    print(request_url)
+    """Extract the API endpoint from the request URL."""
+    match = re.search(r"^https://[^/]+/v1/(.+)$", request_url)
+    if match:
+        return match[1]
+    else:
+        raise ValueError(f"Invalid API URL: {request_url}")
+
+
+def append_to_jsonl(data, filename: str) -> None:
+    """Append a json payload to the end of a jsonl file."""
+    json_string = json.dumps(data)
+    with open(filename, "a") as f:
+        f.write(json_string + "\n")
+
+
+def num_tokens_consumed_from_request(
+    request_json: dict,
+    api_endpoint: str,
+    token_encoding_name: str,
+):
+    encoding = tiktoken.get_encoding(token_encoding_name)
+    print(api_endpoint)
+    if api_endpoint == "messages":
+        num_tokens = 0
+        for message in request_json["messages"]:
+            num_tokens += len(encoding.encode(message["content"]))
+        return num_tokens
+    else:
+        raise NotImplementedError(
+            f'API endpoint "{api_endpoint}" not implemented in this script'
+        )
+
+def task_id_generator_function():
+    """Generate integers 0, 1, 2, and so on."""
+    task_id = 0
+    while True:
+        yield task_id
+        task_id += 1
+
+
+# run script
+
+
+if __name__ == "__main__":
+    # parse command line arguments
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--requests_filepath")
+    parser.add_argument("--save_filepath", default=None)
+    parser.add_argument("--request_url", default="https://api.openai.com/v1/embeddings")
+    parser.add_argument("--api_key", default=os.getenv("ANTHROPIC_API_KEY"))
+    parser.add_argument("--max_requests_per_minute", type=int, default=3_000 * 0.5)
+    parser.add_argument("--max_tokens_per_minute", type=int, default=250_000 * 0.5)
+    parser.add_argument("--token_encoding_name", default="cl100k_base")
+    parser.add_argument("--max_attempts", type=int, default=5)
+    parser.add_argument("--logging_level", default=logging.INFO)
+    args = parser.parse_args()
+
+    if args.save_filepath is None:
+        args.save_filepath = args.requests_filepath.replace(".jsonl", "_results.jsonl")
+
+    # run script
+    asyncio.run(
+        process_api_requests_from_file(
+            requests_filepath=args.requests_filepath,
+            save_filepath=args.save_filepath,
+            request_url=args.request_url,
+            api_key=args.api_key,
+            max_requests_per_minute=float(args.max_requests_per_minute),
+            max_tokens_per_minute=float(args.max_tokens_per_minute),
+            token_encoding_name=args.token_encoding_name,
+            max_attempts=int(args.max_attempts),
+            logging_level=int(args.logging_level),
+        )
+    )
+
+
+"""
+APPENDIX
+
+The example requests file at openai-cookbook/examples/data/example_requests_to_parallel_process.jsonl contains 10,000 requests to text-embedding-ada-002.
+
+It was generated with the following code:
+
+```python
+import json
+
+filename = "data/example_requests_to_parallel_process.jsonl"
+n_requests = 10_000
+jobs = [{"model": "text-embedding-ada-002", "input": str(x) + "\n"} for x in range(n_requests)]
+with open(filename, "w") as f:
+    for job in jobs:
+        json_string = json.dumps(job)
+        f.write(json_string + "\n")
+```
+
+As with all jsonl files, take care that newlines in the content are properly escaped (json.dumps does this automatically).
+"""

api_request_parallel_processor_universal.py

@@ -0,0 +1,492 @@
+"""
+API REQUEST PARALLEL PROCESSOR
+
+Using the OpenAI API to process lots of text quickly takes some care.
+If you trickle in a million API requests one by one, they'll take days to complete.
+If you flood a million API requests in parallel, they'll exceed the rate limits and fail with errors.
+To maximize throughput, parallel requests need to be throttled to stay under rate limits.
+
+This script parallelizes requests to the OpenAI API while throttling to stay under rate limits.
+
+Features:
+- Streams requests from file, to avoid running out of memory for giant jobs
+- Makes requests concurrently, to maximize throughput
+- Throttles request and token usage, to stay under rate limits
+- Retries failed requests up to {max_attempts} times, to avoid missing data
+- Logs errors, to diagnose problems with requests
+
+Example command to call script:
+```
+python examples/api_request_parallel_processor.py \
+  --requests_filepath examples/data/example_requests_to_parallel_process.jsonl \
+  --save_filepath examples/data/example_requests_to_parallel_process_results.jsonl \
+  --request_url https://api.openai.com/v1/embeddings \
+  --max_requests_per_minute 1500 \
+  --max_tokens_per_minute 6250000 \
+  --token_encoding_name cl100k_base \
+  --max_attempts 5 \
+  --logging_level 20
+```
+
+Inputs:
+- requests_filepath : str
+    - path to the file containing the requests to be processed
+    - file should be a jsonl file, where each line is a json object with API parameters and an optional metadata field
+    - e.g., {"model": "text-embedding-ada-002", "input": "embed me", "metadata": {"row_id": 1}}
+    - as with all jsonl files, take care that newlines in the content are properly escaped (json.dumps does this automatically)
+    - an example file is provided at examples/data/example_requests_to_parallel_process.jsonl
+    - the code to generate the example file is appended to the bottom of this script
+- save_filepath : str, optional
+    - path to the file where the results will be saved
+    - file will be a jsonl file, where each line is an array with the original request plus the API response
+    - e.g., [{"model": "text-embedding-ada-002", "input": "embed me"}, {...}]
+    - if omitted, results will be saved to {requests_filename}_results.jsonl
+- request_url : str, optional
+    - URL of the API endpoint to call
+    - if omitted, will default to "https://api.openai.com/v1/embeddings"
+- api_key : str, optional
+    - API key to use
+    - if omitted, the script will attempt to read it from an environment variable {os.getenv("OPENAI_API_KEY")}
+- max_requests_per_minute : float, optional
+    - target number of requests to make per minute (will make less if limited by tokens)
+    - leave headroom by setting this to 50% or 75% of your limit
+    - if requests are limiting you, try batching multiple embeddings or completions into one request
+    - if omitted, will default to 1,500
+- max_tokens_per_minute : float, optional
+    - target number of tokens to use per minute (will use less if limited by requests)
+    - leave headroom by setting this to 50% or 75% of your limit
+    - if omitted, will default to 125,000
+- token_encoding_name : str, optional
+    - name of the token encoding used, as defined in the `tiktoken` package
+    - if omitted, will default to "cl100k_base" (used by `text-embedding-ada-002`)
+- max_attempts : int, optional
+    - number of times to retry a failed request before giving up
+    - if omitted, will default to 5
+- logging_level : int, optional
+    - level of logging to use; higher numbers will log fewer messages
+    - 40 = ERROR; will log only when requests fail after all retries
+    - 30 = WARNING; will log when requests his rate limits or other errors
+    - 20 = INFO; will log when requests start and the status at finish
+    - 10 = DEBUG; will log various things as the loop runs to see when they occur
+    - if omitted, will default to 20 (INFO).
+
+The script is structured as follows:
+    - Imports
+    - Define main()
+        - Initialize things
+        - In main loop:
+            - Get next request if one is not already waiting for capacity
+            - Update available token & request capacity
+            - If enough capacity available, call API
+            - The loop pauses if a rate limit error is hit
+            - The loop breaks when no tasks remain
+    - Define dataclasses
+        - StatusTracker (stores script metadata counters; only one instance is created)
+        - APIRequest (stores API inputs, outputs, metadata; one method to call API)
+    - Define functions
+        - api_endpoint_from_url (extracts API endpoint from request URL)
+        - append_to_jsonl (writes to results file)
+        - num_tokens_consumed_from_request (bigger function to infer token usage from request)
+        - task_id_generator_function (yields 0, 1, 2, ...)
+    - Run main()
+"""
+
+# imports
+import aiohttp  # for making API calls concurrently
+import argparse  # for running script from command line
+import asyncio  # for running API calls concurrently
+import json  # for saving results to a jsonl file
+import logging  # for logging rate limit warnings and other messages
+import os  # for reading API key
+import re  # for matching endpoint from request URL
+import tiktoken  # for counting tokens
+import time  # for sleeping after rate limit is hit
+from dataclasses import (
+    dataclass,
+    field,
+)  # for storing API inputs, outputs, and metadata
+
+
+async def process_api_requests_from_file(
+    vendor_name: str,
+    requests_filepath: str,
+    save_filepath: str,
+    request_url: str,
+    api_key: str,
+    max_requests_per_minute: float,
+    max_tokens_per_minute: float,
+    token_encoding_name: str,
+    max_attempts: int,
+    logging_level: int,
+):
+    """Processes API requests in parallel, throttling to stay under rate limits."""
+    # constants
+    seconds_to_pause_after_rate_limit_error = 15
+    seconds_to_sleep_each_loop = (
+        0.001  # 1 ms limits max throughput to 1,000 requests per second
+    )
+
+    # initialize logging
+    logging.basicConfig(level=logging_level)
+    logging.debug(f"Logging initialized at level {logging_level}")
+
+    # infer API endpoint and construct request header
+    api_endpoint = api_endpoint_from_url(request_url, vendor_name)
+    request_header=None
+    if vendor_name=="openai":
+        request_header = {"Authorization": f"Bearer {api_key}"}
+    elif vendor_name=="anthropic":
+        request_header = {
+        "x-api-key": api_key,
+        "anthropic-version": "2023-06-01",
+        "content-type": "application/json",
+        }
+    elif vendor_name == "meta" or vendor_name == "google" :
+        request_header = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {api_key}",
+        }
+    else:
+        print("Error. Invalid Model Input. Exiting")
+        # exit()
+
+    # use api-key header for Azure deployments
+    # if '/deployments' in request_url:
+    #     request_header = {"api-key": f"{api_key}"}
+
+    # initialize trackers
+    queue_of_requests_to_retry = asyncio.Queue()
+    task_id_generator = (
+        task_id_generator_function()
+    )  # generates integer IDs of 0, 1, 2, ...
+    status_tracker = (
+        StatusTracker()
+    )  # single instance to track a collection of variables
+    next_request = None  # variable to hold the next request to call
+
+    # initialize available capacity counts
+    available_request_capacity = max_requests_per_minute
+    available_token_capacity = max_tokens_per_minute
+    last_update_time = time.time()
+
+    # initialize flags
+    file_not_finished = True  # after file is empty, we'll skip reading it
+    logging.debug(f"Initialization complete.")
+
+    # initialize file reading
+    with open(requests_filepath) as file:
+        # `requests` will provide requests one at a time
+        requests = file.__iter__()
+        logging.debug(f"File opened. Entering main loop")
+        async with aiohttp.ClientSession() as session:  # Initialize ClientSession here
+            while True:
+                # get next request (if one is not already waiting for capacity)
+                if next_request is None:
+                    if not queue_of_requests_to_retry.empty():
+                        next_request = queue_of_requests_to_retry.get_nowait()
+                        logging.debug(
+                            f"Retrying request {next_request.task_id}: {next_request}"
+                        )
+                    elif file_not_finished:
+                        try:
+                            # get new request
+                            request_json = json.loads(next(requests))
+                            next_request = APIRequest(
+                                task_id=next(task_id_generator),
+                                request_json=request_json,
+                                # token_consumption=num_tokens_consumed_from_request(
+                                #     request_json, api_endpoint, token_encoding_name
+                                # ), # just disabled the tokens consumption. Not worried about that. 
+                                token_consumption=0, 
+                                attempts_left=max_attempts,
+                                metadata=request_json.pop("metadata", None),
+                            )
+                            status_tracker.num_tasks_started += 1
+                            status_tracker.num_tasks_in_progress += 1
+                            logging.debug(
+                                f"Reading request {next_request.task_id}: {next_request}"
+                            )
+                        except StopIteration:
+                            # if file runs out, set flag to stop reading it
+                            logging.debug("Read file exhausted")
+                            file_not_finished = False
+
+                # update available capacity
+                current_time = time.time()
+                seconds_since_update = current_time - last_update_time
+                available_request_capacity = min(
+                    available_request_capacity
+                    + max_requests_per_minute * seconds_since_update / 60.0,
+                    max_requests_per_minute,
+                )
+                available_token_capacity = min(
+                    available_token_capacity
+                    + max_tokens_per_minute * seconds_since_update / 60.0,
+                    max_tokens_per_minute,
+                )
+                last_update_time = current_time
+
+                # if enough capacity available, call API
+                if next_request:
+                    next_request_tokens = next_request.token_consumption
+                    if (
+                        available_request_capacity >= 1
+                        and available_token_capacity >= next_request_tokens
+                    ):
+                        # update counters
+                        available_request_capacity -= 1
+                        available_token_capacity -= next_request_tokens
+                        next_request.attempts_left -= 1
+
+                        # call API
+                        asyncio.create_task(
+                            next_request.call_api(
+                                session=session,
+                                request_url=request_url,
+                                request_header=request_header,
+                                retry_queue=queue_of_requests_to_retry,
+                                save_filepath=save_filepath,
+                                status_tracker=status_tracker,
+                            )
+                        )
+                        next_request = None  # reset next_request to empty
+
+                # if all tasks are finished, break
+                if status_tracker.num_tasks_in_progress == 0:
+                    break
+
+                # main loop sleeps briefly so concurrent tasks can run
+                await asyncio.sleep(seconds_to_sleep_each_loop)
+
+                # if a rate limit error was hit recently, pause to cool down
+                seconds_since_rate_limit_error = (
+                    time.time() - status_tracker.time_of_last_rate_limit_error
+                )
+                if (
+                    seconds_since_rate_limit_error
+                    < seconds_to_pause_after_rate_limit_error
+                ):
+                    remaining_seconds_to_pause = (
+                        seconds_to_pause_after_rate_limit_error
+                        - seconds_since_rate_limit_error
+                    )
+                    await asyncio.sleep(remaining_seconds_to_pause)
+                    # ^e.g., if pause is 15 seconds and final limit was hit 5 seconds ago
+                    logging.warn(
+                        f"Pausing to cool down until {time.ctime(status_tracker.time_of_last_rate_limit_error + seconds_to_pause_after_rate_limit_error)}"
+                    )
+
+        # after finishing, log final status
+        logging.info(
+            f"""Parallel processing complete. Results saved to {save_filepath}"""
+        )
+        if status_tracker.num_tasks_failed > 0:
+            logging.warning(
+                f"{status_tracker.num_tasks_failed} / {status_tracker.num_tasks_started} requests failed. Errors logged to {save_filepath}."
+            )
+        if status_tracker.num_rate_limit_errors > 0:
+            logging.warning(
+                f"{status_tracker.num_rate_limit_errors} rate limit errors received. Consider running at a lower rate."
+            )
+
+
+# dataclasses
+
+
+@dataclass
+class StatusTracker:
+    """Stores metadata about the script's progress. Only one instance is created."""
+
+    num_tasks_started: int = 0
+    num_tasks_in_progress: int = 0  # script ends when this reaches 0
+    num_tasks_succeeded: int = 0
+    num_tasks_failed: int = 0
+    num_rate_limit_errors: int = 0
+    num_api_errors: int = 0  # excluding rate limit errors, counted above
+    num_other_errors: int = 0
+    time_of_last_rate_limit_error: int = 0  # used to cool off after hitting rate limits
+
+
+@dataclass
+class APIRequest:
+    """Stores an API request's inputs, outputs, and other metadata. Contains a method to make an API call."""
+
+    task_id: int
+    request_json: dict
+    token_consumption: int
+    attempts_left: int
+    metadata: dict
+    result: list = field(default_factory=list)
+
+    async def call_api(
+        self,
+        session: aiohttp.ClientSession,
+        request_url: str,
+        request_header: dict,
+        retry_queue: asyncio.Queue,
+        save_filepath: str,
+        status_tracker: StatusTracker,
+    ):
+        """Calls the OpenAI API and saves results."""
+        logging.info(f"Starting request #{self.task_id}")
+        error = None
+        try:
+            async with session.post(
+                url=request_url, headers=request_header, json=self.request_json
+            ) as response:
+                response = await response.json()
+            if "error" in response:
+                logging.warning(
+                    f"Request {self.task_id} failed with error {response['error']}"
+                )
+                status_tracker.num_api_errors += 1
+                error = response
+                if "Rate limit" in response["error"].get("message", ""):
+                    status_tracker.time_of_last_rate_limit_error = time.time()
+                    status_tracker.num_rate_limit_errors += 1
+                    status_tracker.num_api_errors -= (
+                        1  # rate limit errors are counted separately
+                    )
+
+        except (
+            Exception
+        ) as e:  # catching naked exceptions is bad practice, but in this case we'll log & save them
+            logging.warning(f"Request {self.task_id} failed with Exception {e}")
+            status_tracker.num_other_errors += 1
+            error = e
+        if error:
+            self.result.append(error)
+            if self.attempts_left:
+                retry_queue.put_nowait(self)
+            else:
+                logging.error(
+                    f"Request {self.request_json} failed after all attempts. Saving errors: {self.result}"
+                )
+                data = (
+                    [self.request_json, [str(e) for e in self.result], self.metadata]
+                    if self.metadata
+                    else [self.request_json, [str(e) for e in self.result]]
+                )
+                append_to_jsonl(data, save_filepath)
+                status_tracker.num_tasks_in_progress -= 1
+                status_tracker.num_tasks_failed += 1
+        else:
+            data = (
+                [self.request_json, response, self.metadata]
+                if self.metadata
+                else [self.request_json, response]
+            )
+            append_to_jsonl(data, save_filepath)
+            status_tracker.num_tasks_in_progress -= 1
+            status_tracker.num_tasks_succeeded += 1
+            logging.debug(f"Request {self.task_id} saved to {save_filepath}")
+
+
+# functions
+
+
+def api_endpoint_from_url(request_url, vendor_name):
+    """Extract the API endpoint from the request URL."""
+    match=None
+    if vendor_name=="openai":
+        match = re.search("^https://[^/]+/v\\d+/(.+)$", request_url)
+    elif vendor_name=="anthropic":
+        match = re.search(r"^https://[^/]+/v1/(.+)$", request_url)
+    elif vendor_name == "meta" or vendor_name == "google":
+        match = re.search(r"^https://[^/]+/api/v1/(.+)$", request_url)
+    else:
+        print("Error. Invalid Model Input. Exiting")
+        # exit()
+    if match is None:
+        # for Azure OpenAI deployment urls
+        match = re.search(r"^https://[^/]+/openai/deployments/[^/]+/(.+?)(\?|$)", request_url)
+    return match[1]
+
+
+def append_to_jsonl(data, filename: str) -> None:
+    """Append a json payload to the end of a jsonl file."""
+    json_string = json.dumps(data)
+    with open(filename, "a") as f:
+        f.write(json_string + "\n")
+
+
+
+def task_id_generator_function():
+    """Generate integers 0, 1, 2, and so on."""
+    task_id = 0
+    while True:
+        yield task_id
+        task_id += 1
+
+
+# run script
+
+
+if __name__ == "__main__":
+    # parse command line arguments
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vendor_name", default=None)
+    parser.add_argument("--requests_filepath")
+    parser.add_argument("--save_filepath", default=None)
+    parser.add_argument("--request_url", default=None)
+    parser.add_argument("--api_key", default=None)
+    parser.add_argument("--max_requests_per_minute", type=int, default=3_000 * 0.5)
+    parser.add_argument("--max_tokens_per_minute", type=int, default=250_000 * 0.5)
+    parser.add_argument("--token_encoding_name", default="cl100k_base")
+    parser.add_argument("--max_attempts", type=int, default=5)
+    parser.add_argument("--logging_level", default=logging.INFO)
+    
+    args = parser.parse_args()
+    if args.vendor_name=="openai":
+        args.api_key=os.getenv("OPENAI_API_KEY")
+        args.request_url="https://api.openai.com/v1/chat/completions"
+    elif args.vendor_name=="anthropic":
+        args.api_key=os.getenv("ANTHROPIC_API_KEY")
+        args.request_url="https://api.anthropic.com/v1/messages"
+    elif args.vendor_name == "meta" or args.vendor_name == "google" :
+        args.api_key = os.getenv("OPENROUTER_API_KEY")
+        args.request_url = "https://openrouter.ai/api/v1/chat/completions"
+    else:
+        print("Error. Invalid Model Input. Exiting")
+        # exit()
+    if args.save_filepath is None:
+        args.save_filepath = args.requests_filepath.replace(".jsonl", "_results.jsonl")
+
+    # run script
+    asyncio.run(
+        process_api_requests_from_file(
+            vendor_name=args.vendor_name,
+            requests_filepath=args.requests_filepath,
+            save_filepath=args.save_filepath,
+            request_url=args.request_url,
+            api_key=args.api_key,
+            max_requests_per_minute=float(args.max_requests_per_minute),
+            max_tokens_per_minute=float(args.max_tokens_per_minute),
+            token_encoding_name=args.token_encoding_name,
+            max_attempts=int(args.max_attempts),
+            logging_level=int(args.logging_level),
+        )
+    )
+
+
+"""
+APPENDIX
+
+The example requests file at openai-cookbook/examples/data/example_requests_to_parallel_process.jsonl contains 10,000 requests to text-embedding-ada-002.
+
+It was generated with the following code:
+
+```python
+import json
+
+filename = "data/example_requests_to_parallel_process.jsonl"
+n_requests = 10_000
+jobs = [{"model": "text-embedding-ada-002", "input": str(x) + "\n"} for x in range(n_requests)]
+with open(filename, "w") as f:
+    for job in jobs:
+        json_string = json.dumps(job)
+        f.write(json_string + "\n")
+```
+
+As with all jsonl files, take care that newlines in the content are properly escaped (json.dumps does this automatically).
+"""

api_request_parallel_processor_universal_SEQUENTIAL.py

@@ -0,0 +1,420 @@
+"""
+API REQUEST PARALLEL PROCESSOR
+
+Using the OpenAI API to process lots of text quickly takes some care.
+If you trickle in a million API requests one by one, they'll take days to complete.
+If you flood a million API requests in parallel, they'll exceed the rate limits and fail with errors.
+To maximize throughput, parallel requests need to be throttled to stay under rate limits.
+
+This script parallelizes requests to the OpenAI API while throttling to stay under rate limits.
+
+Features:
+- Streams requests from file, to avoid running out of memory for giant jobs
+- Makes requests concurrently, to maximize throughput
+- Throttles request and token usage, to stay under rate limits
+- Retries failed requests up to {max_attempts} times, to avoid missing data
+- Logs errors, to diagnose problems with requests
+
+Example command to call script:
+```
+python examples/api_request_parallel_processor.py \
+  --requests_filepath examples/data/example_requests_to_parallel_process.jsonl \
+  --save_filepath examples/data/example_requests_to_parallel_process_results.jsonl \
+  --request_url https://api.openai.com/v1/embeddings \
+  --max_requests_per_minute 1500 \
+  --max_tokens_per_minute 6250000 \
+  --token_encoding_name cl100k_base \
+  --max_attempts 5 \
+  --logging_level 20
+```
+
+Inputs:
+- requests_filepath : str
+    - path to the file containing the requests to be processed
+    - file should be a jsonl file, where each line is a json object with API parameters and an optional metadata field
+    - e.g., {"model": "text-embedding-ada-002", "input": "embed me", "metadata": {"row_id": 1}}
+    - as with all jsonl files, take care that newlines in the content are properly escaped (json.dumps does this automatically)
+    - an example file is provided at examples/data/example_requests_to_parallel_process.jsonl
+    - the code to generate the example file is appended to the bottom of this script
+- save_filepath : str, optional
+    - path to the file where the results will be saved
+    - file will be a jsonl file, where each line is an array with the original request plus the API response
+    - e.g., [{"model": "text-embedding-ada-002", "input": "embed me"}, {...}]
+    - if omitted, results will be saved to {requests_filename}_results.jsonl
+- request_url : str, optional
+    - URL of the API endpoint to call
+    - if omitted, will default to "https://api.openai.com/v1/embeddings"
+- api_key : str, optional
+    - API key to use
+    - if omitted, the script will attempt to read it from an environment variable {os.getenv("OPENAI_API_KEY")}
+- max_requests_per_minute : float, optional
+    - target number of requests to make per minute (will make less if limited by tokens)
+    - leave headroom by setting this to 50% or 75% of your limit
+    - if requests are limiting you, try batching multiple embeddings or completions into one request
+    - if omitted, will default to 1,500
+- max_tokens_per_minute : float, optional
+    - target number of tokens to use per minute (will use less if limited by requests)
+    - leave headroom by setting this to 50% or 75% of your limit
+    - if omitted, will default to 125,000
+- token_encoding_name : str, optional
+    - name of the token encoding used, as defined in the `tiktoken` package
+    - if omitted, will default to "cl100k_base" (used by `text-embedding-ada-002`)
+- max_attempts : int, optional
+    - number of times to retry a failed request before giving up
+    - if omitted, will default to 5
+- logging_level : int, optional
+    - level of logging to use; higher numbers will log fewer messages
+    - 40 = ERROR; will log only when requests fail after all retries
+    - 30 = WARNING; will log when requests his rate limits or other errors
+    - 20 = INFO; will log when requests start and the status at finish
+    - 10 = DEBUG; will log various things as the loop runs to see when they occur
+    - if omitted, will default to 20 (INFO).
+
+The script is structured as follows:
+    - Imports
+    - Define main()
+        - Initialize things
+        - In main loop:
+            - Get next request if one is not already waiting for capacity
+            - Update available token & request capacity
+            - If enough capacity available, call API
+            - The loop pauses if a rate limit error is hit
+            - The loop breaks when no tasks remain
+    - Define dataclasses
+        - StatusTracker (stores script metadata counters; only one instance is created)
+        - APIRequest (stores API inputs, outputs, metadata; one method to call API)
+    - Define functions
+        - api_endpoint_from_url (extracts API endpoint from request URL)
+        - append_to_jsonl (writes to results file)
+        - num_tokens_consumed_from_request (bigger function to infer token usage from request)
+        - task_id_generator_function (yields 0, 1, 2, ...)
+    - Run main()
+"""
+
+# imports
+import aiohttp  # for making API calls concurrently
+import argparse  # for running script from command line
+import asyncio  # for running API calls concurrently
+import json  # for saving results to a jsonl file
+import logging  # for logging rate limit warnings and other messages
+import os  # for reading API key
+import re  # for matching endpoint from request URL
+import tiktoken  # for counting tokens
+import time  # for sleeping after rate limit is hit
+from dataclasses import (
+    dataclass,
+    field,
+)  # for storing API inputs, outputs, and metadata
+
+
+def process_api_requests_from_file(
+    vendor_name: str,
+    requests_filepath: str,
+    save_filepath: str,
+    request_url: str,
+    api_key: str,
+    max_requests_per_minute: float,
+    max_tokens_per_minute: float,
+    token_encoding_name: str,
+    max_attempts: int,
+    logging_level: int,
+):
+    """Processes API requests sequentially."""
+    # initialize logging
+    logging.basicConfig(level=logging_level)
+    logging.debug(f"Logging initialized at level {logging_level}")
+
+    # infer API endpoint and construct request header
+    api_endpoint = api_endpoint_from_url(request_url, vendor_name)
+    request_header = None
+    if vendor_name == "openai":
+        request_header = {"Authorization": f"Bearer {api_key}"}
+    elif vendor_name == "anthropic":
+        request_header = {
+            "x-api-key": api_key,
+            "anthropic-version": "2023-06-01",
+            "content-type": "application/json",
+        }
+    elif vendor_name == "meta" or vendor_name == "google":
+        request_header = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {api_key}",
+        }
+    else:
+        print("Error. Invalid Model Input. Exiting")
+
+    # initialize trackers
+    task_id_generator = task_id_generator_function()
+    status_tracker = StatusTracker()
+
+    # process requests sequentially
+    with open(requests_filepath) as file, requests.Session() as session:
+        for line in file:
+            request_json = json.loads(line)
+            request = APIRequest(
+                task_id=next(task_id_generator),
+                request_json=request_json,
+                token_consumption=0,
+                attempts_left=max_attempts,
+                metadata=request_json.pop("metadata", None),
+            )
+            status_tracker.num_tasks_started += 1
+            logging.debug(f"Processing request {request.task_id}: {request}")
+
+            while request.attempts_left > 0:
+                error = None
+                try:
+                    response = session.post(
+                        url=request_url,
+                        headers=request_header,
+                        json=request.request_json,
+                    ).json()
+                    if "error" in response:
+                        logging.warning(
+                            f"Request {request.task_id} failed with error {response['error']}"
+                        )
+                        status_tracker.num_api_errors += 1
+                        error = response
+                        if "Rate limit" in response["error"].get("message", ""):
+                            status_tracker.num_rate_limit_errors += 1
+                            status_tracker.num_api_errors -= 1
+                except Exception as e:
+                    logging.warning(f"Request {request.task_id} failed with Exception {e}")
+                    status_tracker.num_other_errors += 1
+                    error = e
+
+                if error:
+                    request.result.append(error)
+                    request.attempts_left -= 1
+                    if request.attempts_left == 0:
+                        logging.error(
+                            f"Request {request.request_json} failed after all attempts. Saving errors: {request.result}"
+                        )
+                        data = (
+                            [request.request_json, [str(e) for e in request.result], request.metadata]
+                            if request.metadata
+                            else [request.request_json, [str(e) for e in request.result]]
+                        )
+                        append_to_jsonl(data, save_filepath)
+                        status_tracker.num_tasks_failed += 1
+                else:
+                    data = (
+                        [request.request_json, response, request.metadata]
+                        if request.metadata
+                        else [request.request_json, response]
+                    )
+                    append_to_jsonl(data, save_filepath)
+                    status_tracker.num_tasks_succeeded += 1
+                    logging.debug(f"Request {request.task_id} saved to {save_filepath}")
+                    break
+
+    # after finishing, log final status
+    logging.info(f"Sequential processing complete. Results saved to {save_filepath}")
+    if status_tracker.num_tasks_failed > 0:
+        logging.warning(
+            f"{status_tracker.num_tasks_failed} / {status_tracker.num_tasks_started} requests failed. Errors logged to {save_filepath}."
+        )
+    if status_tracker.num_rate_limit_errors > 0:
+        logging.warning(
+            f"{status_tracker.num_rate_limit_errors} rate limit errors received. Consider running at a lower rate."
+        )
+# dataclasses
+
+
+@dataclass
+class StatusTracker:
+    """Stores metadata about the script's progress. Only one instance is created."""
+
+    num_tasks_started: int = 0
+    num_tasks_in_progress: int = 0  # script ends when this reaches 0
+    num_tasks_succeeded: int = 0
+    num_tasks_failed: int = 0
+    num_rate_limit_errors: int = 0
+    num_api_errors: int = 0  # excluding rate limit errors, counted above
+    num_other_errors: int = 0
+    time_of_last_rate_limit_error: int = 0  # used to cool off after hitting rate limits
+
+
+@dataclass
+class APIRequest:
+    """Stores an API request's inputs, outputs, and other metadata. Contains a method to make an API call."""
+
+    task_id: int
+    request_json: dict
+    token_consumption: int
+    attempts_left: int
+    metadata: dict
+    result: list = field(default_factory=list)
+
+    async def call_api(
+        self,
+        session: aiohttp.ClientSession,
+        request_url: str,
+        request_header: dict,
+        retry_queue: asyncio.Queue,
+        save_filepath: str,
+        status_tracker: StatusTracker,
+    ):
+        """Calls the OpenAI API and saves results."""
+        logging.info(f"Starting request #{self.task_id}")
+        error = None
+        try:
+            async with session.post(
+                url=request_url, headers=request_header, json=self.request_json
+            ) as response:
+                response = await response.json()
+            if "error" in response:
+                logging.warning(
+                    f"Request {self.task_id} failed with error {response['error']}"
+                )
+                status_tracker.num_api_errors += 1
+                error = response
+                if "Rate limit" in response["error"].get("message", ""):
+                    status_tracker.time_of_last_rate_limit_error = time.time()
+                    status_tracker.num_rate_limit_errors += 1
+                    status_tracker.num_api_errors -= (
+                        1  # rate limit errors are counted separately
+                    )
+
+        except (
+            Exception
+        ) as e:  # catching naked exceptions is bad practice, but in this case we'll log & save them
+            logging.warning(f"Request {self.task_id} failed with Exception {e}")
+            status_tracker.num_other_errors += 1
+            error = e
+        if error:
+            self.result.append(error)
+            if self.attempts_left:
+                retry_queue.put_nowait(self)
+            else:
+                logging.error(
+                    f"Request {self.request_json} failed after all attempts. Saving errors: {self.result}"
+                )
+                data = (
+                    [self.request_json, [str(e) for e in self.result], self.metadata]
+                    if self.metadata
+                    else [self.request_json, [str(e) for e in self.result]]
+                )
+                append_to_jsonl(data, save_filepath)
+                status_tracker.num_tasks_in_progress -= 1
+                status_tracker.num_tasks_failed += 1
+        else:
+            data = (
+                [self.request_json, response, self.metadata]
+                if self.metadata
+                else [self.request_json, response]
+            )
+            append_to_jsonl(data, save_filepath)
+            status_tracker.num_tasks_in_progress -= 1
+            status_tracker.num_tasks_succeeded += 1
+            logging.debug(f"Request {self.task_id} saved to {save_filepath}")
+
+
+# functions
+
+
+def api_endpoint_from_url(request_url, vendor_name):
+    """Extract the API endpoint from the request URL."""
+    match=None
+    if vendor_name=="openai":
+        match = re.search("^https://[^/]+/v\\d+/(.+)$", request_url)
+    elif vendor_name=="anthropic":
+        match = re.search(r"^https://[^/]+/v1/(.+)$", request_url)
+    elif vendor_name == "meta" or vendor_name == "google":
+        match = re.search(r"^https://[^/]+/api/v1/(.+)$", request_url)
+    else:
+        print("Error. Invalid Model Input. Exiting")
+        # exit()
+    if match is None:
+        # for Azure OpenAI deployment urls
+        match = re.search(r"^https://[^/]+/openai/deployments/[^/]+/(.+?)(\?|$)", request_url)
+    return match[1]
+
+
+def append_to_jsonl(data, filename: str) -> None:
+    """Append a json payload to the end of a jsonl file."""
+    json_string = json.dumps(data)
+    with open(filename, "a") as f:
+        f.write(json_string + "\n")
+
+
+
+def task_id_generator_function():
+    """Generate integers 0, 1, 2, and so on."""
+    task_id = 0
+    while True:
+        yield task_id
+        task_id += 1
+
+
+# run script
+
+
+if __name__ == "__main__":
+    # parse command line arguments
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vendor_name", default=None)
+    parser.add_argument("--requests_filepath")
+    parser.add_argument("--save_filepath", default=None)
+    parser.add_argument("--request_url", default=None)
+    parser.add_argument("--api_key", default=None)
+    parser.add_argument("--max_requests_per_minute", type=int, default=3_000 * 0.5)
+    parser.add_argument("--max_tokens_per_minute", type=int, default=250_000 * 0.5)
+    parser.add_argument("--token_encoding_name", default="cl100k_base")
+    parser.add_argument("--max_attempts", type=int, default=5)
+    parser.add_argument("--logging_level", default=logging.INFO)
+    
+    args = parser.parse_args()
+    if args.vendor_name=="openai":
+        args.api_key=os.getenv("OPENAI_API_KEY")
+        args.request_url="https://api.openai.com/v1/chat/completions"
+    elif args.vendor_name=="anthropic":
+        args.api_key=os.getenv("ANTHROPIC_API_KEY")
+        args.request_url="https://api.anthropic.com/v1/messages"
+    elif args.vendor_name == "meta" or args.vendor_name == "google" :
+        args.api_key = os.getenv("OPENROUTER_API_KEY")
+        args.request_url = "https://openrouter.ai/api/v1/chat/completions"
+    else:
+        print("Error. Invalid Model Input. Exiting")
+        # exit()
+    if args.save_filepath is None:
+        args.save_filepath = args.requests_filepath.replace(".jsonl", "_results.jsonl")
+
+    # run script
+    asyncio.run(
+        process_api_requests_from_file(
+            vendor_name=args.vendor_name,
+            requests_filepath=args.requests_filepath,
+            save_filepath=args.save_filepath,
+            request_url=args.request_url,
+            api_key=args.api_key,
+            max_requests_per_minute=float(args.max_requests_per_minute),
+            max_tokens_per_minute=float(args.max_tokens_per_minute),
+            token_encoding_name=args.token_encoding_name,
+            max_attempts=int(args.max_attempts),
+            logging_level=int(args.logging_level),
+        )
+    )
+
+
+"""
+APPENDIX
+
+The example requests file at openai-cookbook/examples/data/example_requests_to_parallel_process.jsonl contains 10,000 requests to text-embedding-ada-002.
+
+It was generated with the following code:
+
+```python
+import json
+
+filename = "data/example_requests_to_parallel_process.jsonl"
+n_requests = 10_000
+jobs = [{"model": "text-embedding-ada-002", "input": str(x) + "\n"} for x in range(n_requests)]
+with open(filename, "w") as f:
+    for job in jobs:
+        json_string = json.dumps(job)
+        f.write(json_string + "\n")
+```
+
+As with all jsonl files, take care that newlines in the content are properly escaped (json.dumps does this automatically).
+"""

app.py

@@ -0,0 +1,727 @@
+import os
+# https://stackoverflow.com/questions/76175046/how-to-add-prompt-to-langchain-conversationalretrievalchain-chat-over-docs-with
+# again from: 
+# https://python.langchain.com/docs/integrations/providers/vectara/vectara_chat
+from langchain.document_loaders import PyPDFDirectoryLoader
+import pandas as pd
+import langchain
+from queue import Queue
+from typing import Any
+from langchain.llms.huggingface_text_gen_inference import HuggingFaceTextGenInference
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from langchain.schema import LLMResult
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.prompts.prompt import PromptTemplate
+from anyio.from_thread import start_blocking_portal #For model callback streaming
+
+
+from langchain.prompts import SystemMessagePromptTemplate, HumanMessagePromptTemplate, ChatPromptTemplate
+import os
+from dotenv import load_dotenv
+
+import streamlit as st
+import json
+from langchain.document_loaders import PyPDFLoader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.chains.question_answering import load_qa_chain
+from langchain.chat_models import ChatOpenAI
+# from langchain.chat_models import ChatAnthropic
+from langchain_anthropic import ChatAnthropic
+from langchain.vectorstores import Chroma
+import chromadb
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.llms import OpenAI
+from langchain.chains import RetrievalQA
+from langchain.document_loaders import TextLoader
+from langchain.document_loaders import DirectoryLoader
+from langchain_community.document_loaders import PyMuPDFLoader
+from langchain.schema import Document
+
+from langchain.memory import ConversationBufferMemory
+
+from langchain.chains.conversational_retrieval.prompts import CONDENSE_QUESTION_PROMPT
+from langchain.chains.conversational_retrieval.prompts import QA_PROMPT
+import gradio as gr
+from langchain.memory import ConversationBufferMemory
+from langchain.chains import ConversationalRetrievalChain
+from langchain.chains import ConversationChain
+from langchain.prompts import PromptTemplate
+from langchain.chains import LLMChain
+print("Started")
+
+# Function to map UI region selection to DB metadata region
+# def get_db_region(ui_region: str) -> str:
+#     """Maps UI region selection (e.g., 'Iowa') to the region string stored in metadata (e.g., 'United States')."""
+#     if ui_region == "Iowa":
+#         return "United States"
+#     # Add more mappings if needed (e.g., Africa)
+#     return ui_region # Default to using the UI string if no specific mapping
+
+def get_species_list_from_db(db_name):
+    embedding = OpenAIEmbeddings()
+    vectordb_temp = Chroma(persist_directory=db_name, 
+                  embedding_function=embedding)
+    species_list=[]
+    for meta in vectordb_temp.get()["metadatas"] :
+        try:
+            matched_first_species = meta['matched_specie_0']
+        except KeyError:
+            continue
+        # Since each document is considered as a single chunk, the chunk_index is 0 for all
+        species_list.append( matched_first_species)
+
+    return species_list
+
+
+
+# default_persist_directory = './db5' # For deployement
+default_persist_directory_insects='./vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species'
+default_persist_directory_weeds='./vector-databases-deployed/db5-agllm-data-isu-field-weeds-all-species'
+
+species_list_insects=get_species_list_from_db(default_persist_directory_insects)
+species_list_weeds=get_species_list_from_db(default_persist_directory_weeds)
+# default_persist_directory = 'vector-databases/db5-pre-completion' # For Development
+csv_filepath1 = "./agllm-data/corrected/Corrected_supplemented-insect_data-2500-sorted.xlsx"
+csv_filepath2 = "./agllm-data/corrected/Corrected_supplemented-insect_data-remaining.xlsx"
+
+# India data
+cv_india="./agllm-data/india/species.csv"
+model_name=4
+max_tokens=400
+system_message = {"role": "system", "content": "You are a helpful assistant."} # TODO: double check how this plays out later. 
+langchain.debug=False # TODO: DOUBLE CHECK
+from langchain import globals
+globals.set_debug(False)
+
+retriever_k_value=3
+embedding = OpenAIEmbeddings()
+print("Started....")
+class ChatOpenRouter(ChatOpenAI):
+    openai_api_base: str
+    openai_api_key: str
+    model_name: str
+
+    def __init__(self,
+                 model_name: str,
+                 openai_api_key: [str] = None,
+                 openai_api_base: str = "https://openrouter.ai/api/v1",
+                 **kwargs):
+        openai_api_key = openai_api_key or os.getenv('OPENROUTER_API_KEY')
+        super().__init__(openai_api_base=openai_api_base,
+                         openai_api_key=openai_api_key,
+                         model_name=model_name, **kwargs)
+
+
+######### todo: skipping the first step
+
+
+
+# print(# Single example
+# vectordb.as_retriever(k=2, search_kwargs={"filter": {"matched_specie_0": "Hypagyrtis unipunctata"}, 'k':1}).get_relevant_documents(
+#     "Checking if retriever is correctly initalized?"
+# ))
+
+columns = ['species', 'common name', 'order', 'family',
+           'genus', 'Updated role in ecosystem', 'Proof',
+           'ipm strategies', 'size of insect', 'geographical spread',
+           'life cycle specifics', 'pest for plant species', 'species status',
+           'distribution area', 'appearance', 'identification']
+
+df1 = pd.read_excel(csv_filepath1, usecols=columns)
+df2 = pd.read_excel(csv_filepath2, usecols=columns)
+df_india = pd.read_csv(cv_india)
+all_insects_data = pd.concat([df1, df2], ignore_index=True)
+
+def get_prompt_with_vetted_info_from_specie_name(search_for_specie, mode):
+
+    def read_and_format_filtered_csv_better(dataframe_given, insect_specie):
+            filtered_data = dataframe_given[dataframe_given['species'] == insect_specie]
+            formatted_data = ""
+            # Format the filtered data
+            for index, row in filtered_data.iterrows():
+                    row_data = [f"{col}: {row[col]}" for col in filtered_data.columns]
+                    formatted_row = "\n".join(row_data)
+                    formatted_data += f"{formatted_row}\n"
+
+            return formatted_data
+
+            # Use the path to your CSV file here
+            
+    vetted_info=read_and_format_filtered_csv_better(all_insects_data, search_for_specie)
+    india_info=read_and_format_filtered_csv_better(df_india, search_for_specie)
+    
+
+    if mode=="Farmer":
+            language_constraint="The language should be acustomed to the Farmers. Given question is likely to be asked by a farmer in the field will ask which will help to make decisions which are immediate and practical."
+    elif mode=="Researcher":
+            language_constraint="The language should be acustomed to a researcher. Given question is likely to be asked by a scientist which are comprehensive and aimed at exploring new knowledge or refining existing methodologies"
+    else:
+            print("No  valid mode provided. Exiting")
+            exit()
+#    general_system_template = """ 
+#    In every question you are provided information about the insect/weed. Two types of information are: First,  Vetted Information (which is same in every questinon) and Second, some context from external documents about an insect/weed species and a question by the user. answer the question according to these two types of informations. 
+#    ---- 
+#    Vetted info is as follows:
+#    {vetted_info}
+#    ----
+#    The context retrieved for documents about this particular question is as follows:
+#    {context}
+#    ----
+#    Additional Instruction:
+#    1. Reference Constraint
+#    At the end of each answer provide the source/reference for the given data in following format: 
+#    \n\n[enter two new lines before writing below] References:
+#    Vetted Information Used: Write what was used from the document for coming up with the answer above. Write exact part of lines. If nothing, write 'Nothing'. 
+#    Documents Used: Write what was used from the document for coming up with the answer above. If nothing, write 'Nothing'. Write exact part of lines and document used. 
+#    2. Information Constraint: 
+#    Only answer the question from information provided otherwise say you dont know. You have to answer in 50 words including references. Prioritize information in documents/context over vetted information. And first mention the warnings/things to be careful about. 
+#    3. Language constraint:
+#    {language_constraint}
+
+#    ----
+#    """.format(vetted_info=vetted_info, language_constraint=language_constraint,context="{context}", )
+
+    general_system_template = f""" 
+    You are an AI assistant specialized in providing information about insects/weeds. Answer the user's question based on the available information or your general knowledge.
+
+    The context retrieved for this question is as follows:
+    {{context}}
+
+    Instructions:
+    1. Evaluate the relevance of the provided context to the question.
+    2. If the context contains relevant information, use it to answer the question and explicitly mention "Based on provided information" in your source.
+    3. If the context does not contain relevant information, use your general knowledge to answer the question and state "Based on general knowledge" as the source.
+    4. Format your response as follows:
+       Answer: Provide a concise answer in less than 50 words.
+       Source: State either "Based on provided information" or "Based on general knowledge".
+
+    5. Language constraint:
+       {language_constraint}
+    6. Other region (India) information:
+        {india_info}
+    7. So you have two kinds of information (default from Iowa and other region (India) information). First need to ask the user what region they are interested in. and only provide information from that region.
+    8. When answering question, say what if the information is from what regiion. So, if a region is selected by user and specified in the question, then only answer based on that region and say so. 
+
+
+    Question: {{question}}
+    """
+
+    general_user_template = "Question:```{question}```"
+    messages_formatted = [
+            SystemMessagePromptTemplate.from_template(general_system_template),
+            HumanMessagePromptTemplate.from_template(general_user_template)
+    ]
+    qa_prompt = ChatPromptTemplate.from_messages( messages_formatted)
+#    print(qa_prompt)
+    return qa_prompt
+
+
+    
+
+
+# qa_prompt=get_prompt_with_vetted_info_from_specie_name("Papaipema nebris", "Researcher")
+# print("First prompt is intialized as: " , qa_prompt, "\n\n")
+
+
+memory = ConversationBufferMemory(memory_key="chat_history",output_key='answer', return_messages=True) # https://github.com/langchain-ai/langchain/issues/9394#issuecomment-1683538834
+
+
+if model_name==4:
+    llm_openai = ChatOpenAI(model_name="gpt-4o-2024-08-06" , temperature=0, max_tokens=max_tokens)  # TODO: NEW MODEL VERSION AVAILABLE 
+else:
+    llm_openai = ChatOpenAI(model_name="gpt-3.5-turbo-0125" , temperature=0, max_tokens=max_tokens)
+
+specie_selector="Papaipema nebris"
+filter = {
+    "$or": [
+        {"matched_specie_0": specie_selector},
+        {"matched_specie_1": specie_selector},
+        {"matched_specie_2": specie_selector},
+    ]
+}
+# retriever = vectordb.as_retriever(search_kwargs={'k':retriever_k_value, 'filter': filter})
+
+# qa_chain = ConversationalRetrievalChain.from_llm(
+#     llm_openai, retriever, memory=memory, verbose=False, return_source_documents=True,\
+#           combine_docs_chain_kwargs={'prompt': qa_prompt}
+
+# )
+# 
+
+def initialize_qa_chain(specie_selector, application_mode, model_name, region, database_persistent_directory=default_persist_directory_insects, domain_name="Insects"):
+    # Add helper function for India info (kept for potential future use, but removed from RAG prompt)
+    def read_and_format_filtered_csv_better(dataframe_given, insect_specie):
+            filtered_data = dataframe_given[dataframe_given['species'] == insect_specie]
+            formatted_data = ""
+            # Format the filtered data
+            for index, row in filtered_data.iterrows():
+                    row_data = [f"{col}: {row[col]}" for col in filtered_data.columns]
+                    formatted_row = "\n".join(row_data)
+                    formatted_data += f"{formatted_row}\n"
+
+            return formatted_data
+    
+    # Get India info (potentially useful if not using RAG or for specific logic)
+    india_info = read_and_format_filtered_csv_better(df_india, specie_selector)
+    # db_region = get_db_region(region) # Map UI region to DB region - REMOVED
+
+    if model_name=="GPT-4":
+        chosen_llm=ChatOpenAI(model_name="gpt-4o-2024-08-06" , temperature=0, max_tokens=max_tokens)
+    elif model_name=="GPT-3.5":
+        chosen_llm=ChatOpenAI(model_name="gpt-3.5-turbo-0125" , temperature=0, max_tokens=max_tokens)
+    elif model_name=="Llama-3 70B":
+        chosen_llm = ChatOpenRouter(model_name="meta-llama/llama-3-70b-instruct", temperature=0,max_tokens=max_tokens )
+    elif model_name=="Llama-3 8B":
+        chosen_llm = ChatOpenRouter(model_name="meta-llama/llama-3-8b-instruct", temperature=0, max_tokens=max_tokens)
+    elif model_name=="Gemini-1.5 Pro":
+        chosen_llm = ChatOpenRouter(model_name="google/gemini-pro-1.5", temperature=0, max_tokens=max_tokens)
+    elif model_name=="Claude 3 Opus":
+        chosen_llm = ChatAnthropic(model_name='claude-3-opus-20240229', temperature=0, max_tokens=max_tokens)
+    elif model_name=="Claude 3.5 Sonnet":
+        chosen_llm = ChatAnthropic(model_name='claude-3-5-sonnet-20240620', temperature=0, max_tokens=max_tokens)
+    else:
+        print("No appropriate llm was selected")  
+        exit()
+    
+    if application_mode == "Farmer":
+        language_constraint = "The language should be customized for Farmers. The given question is likely to be asked by a farmer in the field and will help to make decisions which are immediate and practical."
+    elif application_mode == "Researcher":
+        language_constraint = "The language should be customized for a researcher. The given question is likely to be asked by a scientist and should be comprehensive, aimed at exploring new knowledge or refining existing methodologies."
+    else:
+        print("No valid mode provided. Exiting") 
+        exit()
+
+    # RAG is always ON now
+    memory = ConversationBufferMemory(memory_key="chat_history", output_key='answer', return_messages=True)
+    
+    # Construct the species filter part
+    species_filter = {
+        "$or": [
+            {"matched_specie_" + str(i): specie_selector} for i in range(11) # Generate dynamically up to 10
+        ]
+    }
+    
+    embedding = OpenAIEmbeddings()
+    vectordb = Chroma(persist_directory=database_persistent_directory,
+                     embedding_function=embedding)
+    
+    # --- Find all available regions for this species --- #
+    availability_message = f"Checking region availability for {specie_selector}..."
+    available_regions = set()
+    
+    try:
+        # Query ChromaDB just for metadata based on species
+        species_docs = vectordb.get(where=species_filter, include=['metadatas'])
+        if species_docs and species_docs.get('metadatas'):
+            for meta in species_docs['metadatas']:
+                if 'region' in meta:
+                    available_regions.add(meta['region'])
+        
+        if available_regions:
+            available_regions_list = sorted(list(available_regions))
+            availability_message = f"Information for **{specie_selector}** is available in region(s): **{', '.join(available_regions_list)}**."
+        else:
+            available_regions_list = []
+            availability_message = f"No regional information found for **{specie_selector}** in the {domain_name} database."
+    except Exception as e:
+        print(f"Error checking region availability: {e}")
+        available_regions_list = []
+        availability_message = f"Could not determine region availability for {specie_selector}."
+    
+    # --- Prepare context sections by region --- #
+    # Dictionary to hold context documents for each region
+    # region_contexts = {} # Unused variable, removing
+    
+    # First check if selected region has information
+    selected_region_has_info = region in available_regions
+    
+    # Create list of other available regions (excluding selected region)
+    other_regions = [r for r in available_regions_list if r != region]
+    
+    # --- Create multi-region retrieval chain --- #
+    class MultiRegionRetriever:
+        def __init__(self, vectordb, species_filter, selected_region, other_regions, k=3):
+            self.vectordb = vectordb
+            self.species_filter = species_filter
+            self.selected_region = selected_region
+            self.other_regions = other_regions
+            self.k = k
+        
+        def get_relevant_documents(self, query):
+            all_docs = []
+            region_docs = {}
+            
+            # First get documents for selected region
+            # Fix illogical condition: self.selected_region == self.selected_region is always True
+            # Replace with a check if selected_region exists
+            if self.selected_region:
+                selected_filter = {"$and": [self.species_filter, {"region": self.selected_region}]}
+                selected_retriever = self.vectordb.as_retriever(search_kwargs={'k': self.k, 'filter': selected_filter})
+                try:
+                    selected_docs = selected_retriever.get_relevant_documents(query)
+                    if selected_docs:
+                        all_docs.extend(selected_docs)
+                        region_docs[self.selected_region] = selected_docs
+                except Exception as e:
+                    print(f"Error retrieving docs for selected region {self.selected_region}: {e}")
+            
+            # Then get documents for each other region
+            for other_region in self.other_regions:
+                if other_region != self.selected_region:  # Skip if same as selected region
+                    other_filter = {"$and": [self.species_filter, {"region": other_region}]}
+                    other_retriever = self.vectordb.as_retriever(search_kwargs={'k': self.k, 'filter': other_filter})
+                    try:
+                        other_docs = other_retriever.get_relevant_documents(query)
+                        if other_docs:
+                            all_docs.extend(other_docs)
+                            region_docs[other_region] = other_docs
+                    except Exception as e:
+                        print(f"Error retrieving docs for region {other_region}: {e}")
+            
+            # Store the region-specific documents for formatting in the prompt
+            self.last_region_docs = region_docs
+            return all_docs
+    
+    # Initialize the multi-region retriever
+    multi_region_retriever = MultiRegionRetriever(
+        vectordb=vectordb,
+        species_filter=species_filter,
+        selected_region=region,
+        other_regions=available_regions_list,
+        k=retriever_k_value
+    )
+    
+    # Custom prompt handler that formats context by region
+    # Remove unused imports
+    # from langchain.chains.combine_documents import create_stuff_documents_chain
+    # from langchain.chains import create_retrieval_chain
+    
+    # Updated prompt template for multi-part response with region-specific contexts
+    general_system_template = f"""
+You are an AI assistant specialized in providing information about {domain_name.lower()} ({specie_selector}). The user is primarily interested in the '{region}' region.
+
+The following context has been retrieved from a database organized by region:
+
+{{context}}
+
+Instructions:
+1. Analyze the user's question in relation to {specie_selector}.
+2. Structure your answer in the following multi-part format:
+
+   **Part 1: Selected Region Information ({region})**
+   If relevant information exists in the context for the selected region that answers the user's query:
+   Based on your selected region ({region}), for {specie_selector}, [summary of information for selected region] [1].
+   
+   If no relevant information exists for the selected region:
+   "Based on the provided documents, there is no specific information for {specie_selector} in your selected region ({region}) regarding your question."
+
+   **Part 2: Other Regions Information** (Only include if information from other regions is available AND relevant to the query)
+   If you found relevant information from other regions that answers the user's query, include:
+   
+   Additionally, information was found for other regions:
+   - In [Other Region Name]: [summary of information that directly answers the user's query] [next reference number].
+   - In [Another Region Name]: [summary of information that directly answers the user's query] [next reference number].
+   
+   Only include regions where the information directly addresses the user's question.
+   Use consecutive reference numbers starting from where Part 1 left off.
+   If no other regions have relevant information, omit this part entirely.
+
+   **Part 3: General Knowledge** (Only include if context information is insufficient or incomplete)
+   If the available context does not fully address the query, add:
+   
+   Based on my general knowledge as {model_name}: [Your general knowledge insights that directly address the query] [next reference number].
+   
+   If the context information is sufficient, omit this part entirely.
+
+3. After providing all parts of your answer, include a References section ONLY for information you actually used:
+
+References:
+[1] Based on Expert Curated information about {specie_selector} in {region}
+[2] Based on Expert Curated information about {specie_selector} in [Other Region Name]
+[3] Based on Expert Curated information about {specie_selector} in [Another Region Name]
+[x] {model_name}'s inherent knowledge
+
+IMPORTANT:
+- Only include reference numbers that correspond to information you actually used in your answer.
+- Reference numbers should be sequential (1, 2, 3...) based on the order they appear in your answer.
+- If you don't use information from a particular region, don't include a reference for it.
+- If you don't use general knowledge, don't include a reference for it.
+- Every claim with a reference marker [x] must have a corresponding entry in the References section.
+
+4. Apply this language constraint: {language_constraint}
+5. Keep your summaries concise and directly related to the user's question.
+
+User Question about {specie_selector} ({domain_name}): {{question}}
+"""
+
+    class RegionFormattingLLMChain:
+        def __init__(self, llm, prompt, retriever):
+            self.llm = llm
+            self.prompt = prompt
+            self.retriever = retriever
+        
+        def __call__(self, inputs):
+            # Get documents using the multi-region retriever
+            docs = self.retriever.get_relevant_documents(inputs["question"])
+            
+            # Get the region-specific document organization
+            region_docs = getattr(self.retriever, "last_region_docs", {})
+            
+            # Format context with clear region sections
+            formatted_context = ""
+            
+            # First add context for selected region if available
+            if region in region_docs:
+                formatted_context += f"--- CONTEXT FROM SELECTED REGION: {region} ---\n"
+                for i, doc in enumerate(region_docs[region]):
+                    formatted_context += f"Document {i+1} from {region}:\n{doc.page_content}\n\n"
+            
+            # Then add context for each other region
+            for other_region in [r for r in region_docs.keys() if r != region]:
+                formatted_context += f"--- CONTEXT FROM OTHER REGION: {other_region} ---\n"
+                for i, doc in enumerate(region_docs[other_region]):
+                    formatted_context += f"Document {i+1} from {other_region}:\n{doc.page_content}\n\n"
+            
+            # Replace the context placeholder with our formatted context
+            formatted_prompt = self.prompt.format(
+                context=formatted_context,
+                question=inputs["question"]
+            )
+            
+            # Call the LLM with our formatted prompt
+            result = self.llm.invoke(formatted_prompt)
+            
+            # Return the result in the expected format
+            return {"answer": result.content, "source_documents": docs}
+    
+    # Create the custom chain
+    qa_chain = RegionFormattingLLMChain(
+        llm=chosen_llm,
+        prompt=general_system_template,
+        retriever=multi_region_retriever
+    )
+    
+    return qa_chain, availability_message
+# result = qa_chain.invoke({"question": "where are stalk borer eggs laid?"})
+# print("Got the first LLM task working: ", result)
+
+
+#Application Interface: 
+with gr.Blocks(theme=gr.themes.Soft())  as demo:
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Markdown(
+                """
+                ![Logo](file/logo1.png)
+                """
+            )
+        with gr.Column(scale=1):
+            gr.Markdown(
+                """
+                ![Logo](file/logo2.png)
+                """
+            )
+
+    # Configure UI layout
+    chatbot = gr.Chatbot(height=600, label="AgLLM")
+    with gr.Row():
+        with gr.Column(scale=1):
+            with gr.Row():
+                    domain_name = gr.Dropdown(
+                        list(["Insects", "Weeds"]),
+                        value="Insects",
+                        label="Domain",
+                        info="Select Domain",
+                        interactive=True,
+                        scale=1,
+                        visible=True
+                    )
+                    region_selector = gr.Dropdown(
+                        list(["United States", "India", "Africa"]), # Updated regions
+                        value="United States", # Updated default
+                        label="Region",
+                        info="Select the Region",
+                        interactive=True,
+                        scale=1,
+                        visible=True
+                    )
+
+                    # Model selection
+                    specie_selector = gr.Dropdown(
+                        list(set(species_list_insects)),
+                        value=species_list_insects[0],
+                        label="Species",
+                        info="Select the Species",
+                        interactive=True,
+                        scale=1,
+                        visible=True
+                    )
+            with gr.Row():
+                    model_name = gr.Dropdown(
+                        list(["GPT-4", "GPT-3.5", "Llama-3 70B", "Llama-3 8B", "Gemini-1.5 Pro", "Claude 3 Opus", "Claude 3.5 Sonnet"]),
+                        value="Llama-3 70B",
+                        label="LLM",
+                        info="Select the LLM",
+                        interactive=True,
+                        scale=1,
+                        visible=True
+                    )
+                    application_mode = gr.Dropdown(
+                        list(["Farmer", "Researcher"]),
+                        value="Researcher",
+                        label="Mode",
+                        info="Select the Mode",
+                        interactive=True,
+                        scale=1,
+                        visible=True
+                    )
+            region_availability_display = gr.Markdown(value="Select species/domain to see region availability.") # Added display area
+
+        with gr.Column(scale=2):
+            # User input prompt text field
+            user_prompt_message = gr.Textbox(placeholder="Please add user prompt here", label="User prompt")
+            with gr.Row():
+                # clear = gr.Button("Clear Conversation", scale=2)
+                submitBtn = gr.Button("Submit", scale=8)
+
+    state = gr.State([])
+    qa_chain_state = gr.State(value=None)
+
+    # Handle user message
+    def user(user_prompt_message, history):
+        # print("HISTORY IS: ", history)  # TODO: REMOVE IT LATER
+        if user_prompt_message != "":
+            return history + [[user_prompt_message, None]]
+        else:
+            return history + [["Invalid prompts - user prompt cannot be empty", None]]
+
+    # Chatbot logic for configuration, sending the prompts, rendering the streamed back generations, etc.
+    def bot(model_name, application_mode, user_prompt_message, history, messages_history, qa_chain, domain_name, region): # Removed use_rag
+        if qa_chain == None:
+            # Initial QA chain setup if not already done (uses default species for the selected domain)
+            initial_species = species_list_insects[0] if domain_name == "Insects" else species_list_weeds[0]
+            # Need to handle the tuple returned by init_qa_chain now
+            # Use the currently selected region for initialization if qa_chain is None
+            qa_chain, _ = init_qa_chain(initial_species, application_mode, model_name, domain_name, region) # Pass region
+
+        history[-1][1] = ""  # Placeholder for the answer
+
+        # RAG is always ON now
+        result = qa_chain({"question": user_prompt_message, "chat_history": messages_history})
+        answer = result["answer"]
+        # source_documents = result.get("source_documents", []) # Keep source_documents if needed for debugging or future refinement
+        # formatted_response = format_response_with_source(answer, source_documents, domain_name, region) # REMOVED: Rely on LLM prompt now
+
+        history[-1][1] = answer # Assign raw LLM answer directly
+        return [history, messages_history]
+
+    # Helper function to format the response with source information
+    # def format_response_with_source(answer, source_documents, domain_name, region): # Pass region # FUNCTION NO LONGER USED
+    #     try:
+    #         answer_start = answer.find("Answer:")
+    #         source_start = answer.find("Source:")
+    #         ... (rest of the function commented out or removed) ...
+    #     except Exception as e:
+    #         print(f"Error parsing output or formatting source: {e}")
+    #         formatted_response = answer # Return raw answer on error
+    #
+    #     return formatted_response
+
+    # Initialize the chat history with default system message
+    def init_history(messages_history):
+        messages_history = []
+        messages_history += [system_message]
+        return messages_history
+
+    # Clean up the user input text field
+    def input_cleanup():
+        return ""
+
+    def init_qa_chain(specie_selector, application_mode, model_name, domain_name, region): # Removed use_rag
+        print(f"--- init_qa_chain wrapper called with domain: '{domain_name}' ---") # DIAGNOSTIC PRINT
+        qa_chain_instance = None
+        availability_msg = "Error initializing QA chain."
+        try:
+            if domain_name=="Insects":
+                qa_chain_instance, availability_msg = initialize_qa_chain(specie_selector, application_mode, model_name, region, default_persist_directory_insects, domain_name) # Removed use_rag
+            elif domain_name=="Weeds":
+                qa_chain_instance, availability_msg = initialize_qa_chain(specie_selector, application_mode, model_name, region, default_persist_directory_weeds, domain_name) # Removed use_rag
+            else:
+                print("No Appropriate Chain Selected")
+                availability_msg = "Invalid domain selected."
+                # Return None for chain and the message
+        except Exception as e:
+            print(f"Error in init_qa_chain wrapper: {e}")
+            availability_msg = f"Error initializing: {e}"
+        
+        return qa_chain_instance, availability_msg # Return both chain and message
+
+    # Update QA chain AND availability message when relevant inputs change
+    inputs_for_qa_chain = [specie_selector, application_mode, model_name, domain_name, region_selector] # CORRECT ORDER
+    outputs_for_qa_chain = [qa_chain_state, region_availability_display]
+
+    # specie_selector.change(init_qa_chain, inputs=inputs_for_qa_chain, outputs=outputs_for_qa_chain)
+    # model_name.change(init_qa_chain, inputs=inputs_for_qa_chain, outputs=outputs_for_qa_chain)
+    # region_selector.change(init_qa_chain, inputs=inputs_for_qa_chain, outputs=outputs_for_qa_chain)
+    # application_mode.change(init_qa_chain, inputs=inputs_for_qa_chain, outputs=outputs_for_qa_chain)
+    # domain_name.change(init_qa_chain, inputs=inputs_for_qa_chain, outputs=outputs_for_qa_chain)
+    specie_selector.change(init_qa_chain, inputs=inputs_for_qa_chain, outputs=outputs_for_qa_chain)
+    model_name.change(init_qa_chain, inputs=inputs_for_qa_chain, outputs=outputs_for_qa_chain)
+    region_selector.change(init_qa_chain, inputs=inputs_for_qa_chain, outputs=outputs_for_qa_chain)
+    application_mode.change(init_qa_chain, inputs=inputs_for_qa_chain, outputs=outputs_for_qa_chain)
+    domain_name.change(init_qa_chain, inputs=inputs_for_qa_chain, outputs=outputs_for_qa_chain)
+
+    #####
+    def update_species_list(domain):
+        if domain == "Insects":
+            return gr.Dropdown( species_list_insects, value=species_list_insects[0], label="Species", info="Select the Species", interactive=True, scale=1, visible=True )
+        elif domain == "Weeds":
+            return gr.Dropdown( species_list_weeds, value=species_list_weeds[0], label="Species", info="Select the Species", interactive=True, scale=1, visible=True )
+
+    domain_name.change(
+        update_species_list,
+        inputs=[domain_name],
+        outputs=[specie_selector]
+    )
+
+    # When the user clicks Enter and the user message is submitted
+    user_prompt_message.submit(
+        user,
+        [user_prompt_message, chatbot],
+        [chatbot],
+        queue=False
+    ).then(
+        bot,
+        [model_name, application_mode, user_prompt_message, chatbot, state, qa_chain_state, domain_name, region_selector], # Removed use_rag
+        [chatbot, state]
+    ).then(input_cleanup,
+        [],
+        [user_prompt_message],
+        queue=False
+    )
+
+    # When the user clicks the submit button
+    submitBtn.click(
+        user,
+        [user_prompt_message, chatbot],
+        [chatbot],
+        queue=False
+    ).then(
+        bot,
+        [model_name, application_mode, user_prompt_message, chatbot, state, qa_chain_state, domain_name, region_selector], # Removed use_rag
+        [chatbot, state]
+    ).then(
+        input_cleanup,
+        [],
+        [user_prompt_message],
+        queue=False
+    )
+
+    # When the user clicks the clear button
+    # clear.click(lambda: None, None, chatbot, queue=False).success(init_history, [state], [state])
+    if __name__ == "__main__":
+    # demo.launch()
+        demo.queue().launch(allowed_paths=["/"], share=False, show_error=True)

app_backup.py

@@ -0,0 +1,490 @@
+import os
+# https://stackoverflow.com/questions/76175046/how-to-add-prompt-to-langchain-conversationalretrievalchain-chat-over-docs-with
+# again from: 
+# https://python.langchain.com/docs/integrations/providers/vectara/vectara_chat
+from langchain.document_loaders import PyPDFDirectoryLoader
+import pandas as pd
+import langchain
+from queue import Queue
+from typing import Any
+from langchain.llms.huggingface_text_gen_inference import HuggingFaceTextGenInference
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from langchain.schema import LLMResult
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.prompts.prompt import PromptTemplate
+from anyio.from_thread import start_blocking_portal #For model callback streaming
+
+
+from langchain.prompts import SystemMessagePromptTemplate, HumanMessagePromptTemplate, ChatPromptTemplate
+import os
+from dotenv import load_dotenv
+
+import streamlit as st
+import json
+from langchain.document_loaders import PyPDFLoader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.chains.question_answering import load_qa_chain
+from langchain.chat_models import ChatOpenAI
+# from langchain.chat_models import ChatAnthropic
+from langchain_anthropic import ChatAnthropic
+from langchain.vectorstores import Chroma
+import chromadb
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.llms import OpenAI
+from langchain.chains import RetrievalQA
+from langchain.document_loaders import TextLoader
+from langchain.document_loaders import DirectoryLoader
+from langchain_community.document_loaders import PyMuPDFLoader
+from langchain.schema import Document
+
+from langchain.memory import ConversationBufferMemory
+
+from langchain.chains.conversational_retrieval.prompts import CONDENSE_QUESTION_PROMPT
+from langchain.chains.conversational_retrieval.prompts import QA_PROMPT
+import gradio as gr
+from langchain.memory import ConversationBufferMemory
+from langchain.chains import ConversationalRetrievalChain
+print("Started")
+
+def get_species_list_from_db(db_name):
+    embedding = OpenAIEmbeddings()
+    vectordb_temp = Chroma(persist_directory=db_name, 
+                  embedding_function=embedding)
+    species_list=[]
+    for meta in vectordb_temp.get()["metadatas"] :
+        try:
+            matched_first_species = meta['matched_specie_0']
+        except KeyError:
+            continue
+        # Since each document is considered as a single chunk, the chunk_index is 0 for all
+        species_list.append( matched_first_species)
+
+    return species_list
+
+
+
+# default_persist_directory = './db5' # For deployement
+default_persist_directory_insects='./vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species'
+default_persist_directory_weeds='./vector-databases-deployed/db5-agllm-data-isu-field-weeds-all-species'
+
+species_list_insects=get_species_list_from_db(default_persist_directory_insects)
+species_list_weeds=get_species_list_from_db(default_persist_directory_weeds)
+# default_persist_directory = 'vector-databases/db5-pre-completion' # For Development
+csv_filepath1 = "./agllm-data/corrected/Corrected_supplemented-insect_data-2500-sorted.xlsx"
+csv_filepath2 = "./agllm-data/corrected/Corrected_supplemented-insect_data-remaining.xlsx"
+model_name=4
+max_tokens=400
+system_message = {"role": "system", "content": "You are a helpful assistant."} # TODO: double check how this plays out later. 
+langchain.debug=False # TODO: DOUBLE CHECK
+from langchain import globals
+globals.set_debug(False)
+
+retriever_k_value=3
+embedding = OpenAIEmbeddings()
+print("Started....")
+class ChatOpenRouter(ChatOpenAI):
+    openai_api_base: str
+    openai_api_key: str
+    model_name: str
+
+    def __init__(self,
+                 model_name: str,
+                 openai_api_key: [str] = None,
+                 openai_api_base: str = "https://openrouter.ai/api/v1",
+                 **kwargs):
+        openai_api_key = openai_api_key or os.getenv('OPENROUTER_API_KEY')
+        super().__init__(openai_api_base=openai_api_base,
+                         openai_api_key=openai_api_key,
+                         model_name=model_name, **kwargs)
+
+
+######### todo: skipping the first step
+
+
+
+# print(# Single example
+# vectordb.as_retriever(k=2, search_kwargs={"filter": {"matched_specie_0": "Hypagyrtis unipunctata"}, 'k':1}).get_relevant_documents(
+#     "Checking if retriever is correctly initalized?"
+# ))
+
+columns = ['species', 'common name', 'order', 'family',
+           'genus', 'Updated role in ecosystem', 'Proof',
+           'ipm strategies', 'size of insect', 'geographical spread',
+           'life cycle specifics', 'pest for plant species', 'species status',
+           'distribution area', 'appearance', 'identification']
+
+df1 = pd.read_excel(csv_filepath1, usecols=columns)
+df2 = pd.read_excel(csv_filepath2, usecols=columns)
+
+all_insects_data = pd.concat([df1, df2], ignore_index=True)
+
+def get_prompt_with_vetted_info_from_specie_name(search_for_specie, mode):
+
+    def read_and_format_filtered_csv_better(insect_specie):
+            filtered_data = all_insects_data[all_insects_data['species'] == insect_specie]
+            formatted_data = ""
+            # Format the filtered data
+            for index, row in filtered_data.iterrows():
+                    row_data = [f"{col}: {row[col]}" for col in filtered_data.columns]
+                    formatted_row = "\n".join(row_data)
+                    formatted_data += f"{formatted_row}\n"
+
+            return formatted_data
+
+            # Use the path to your CSV file here
+            
+    vetted_info=read_and_format_filtered_csv_better(search_for_specie)
+
+    if mode=="Farmer":
+            language_constraint="The language should be acustomed to the Farmers. Given question is likely to be asked by a farmer in the field will ask which will help to make decisions which are immediate and practical."
+    elif mode=="Researcher":
+            language_constraint="The language should be acustomed to a researcher. Given question is likely to be asked by a scientist which are comprehensive and aimed at exploring new knowledge or refining existing methodologies"
+    else:
+            print("No  valid mode provided. Exiting")
+            exit()
+#    general_system_template = """ 
+#    In every question you are provided information about the insect/weed. Two types of information are: First,  Vetted Information (which is same in every questinon) and Second, some context from external documents about an insect/weed species and a question by the user. answer the question according to these two types of informations. 
+#    ---- 
+#    Vetted info is as follows:
+#    {vetted_info}
+#    ----
+#    The context retrieved for documents about this particular question is as follows:
+#    {context}
+#    ----
+#    Additional Instruction:
+#    1. Reference Constraint
+#    At the end of each answer provide the source/reference for the given data in following format: 
+#    \n\n[enter two new lines before writing below] References:
+#    Vetted Information Used: Write what was used from the document for coming up with the answer above. Write exact part of lines. If nothing, write 'Nothing'. 
+#    Documents Used: Write what was used from the document for coming up with the answer above. If nothing, write 'Nothing'. Write exact part of lines and document used. 
+#    2. Information Constraint: 
+#    Only answer the question from information provided otherwise say you dont know. You have to answer in 50 words including references. Prioritize information in documents/context over vetted information. And first mention the warnings/things to be careful about. 
+#    3. Language constraint:
+#    {language_constraint}
+
+#    ----
+#    """.format(vetted_info=vetted_info, language_constraint=language_constraint,context="{context}", )
+
+    general_system_template = f""" 
+    You are an AI assistant specialized in providing information about insects/weeds. Answer the user's question based on the available information or your general knowledge.
+
+    The context retrieved for this question is as follows:
+    {{context}}
+
+    Instructions:
+    1. Evaluate the relevance of the provided context to the question.
+    2. If the context contains relevant information, use it to answer the question.
+    3. If the context does not contain relevant information, use your general knowledge to answer the question.
+    4. Format your response as follows:
+       Answer: Provide a concise answer in less than 50 words.
+       Reference: If you used the provided context, cite the specific information used. If you used your general knowledge, state "Based on general knowledge".
+
+    5. Language constraint:
+       {language_constraint}
+
+    Question: {{question}}
+    """
+
+    general_user_template = "Question:```{question}```"
+    messages_formatted = [
+            SystemMessagePromptTemplate.from_template(general_system_template),
+            HumanMessagePromptTemplate.from_template(general_user_template)
+    ]
+    qa_prompt = ChatPromptTemplate.from_messages( messages_formatted )
+#    print(qa_prompt)
+    return qa_prompt
+
+
+    
+
+
+qa_prompt=get_prompt_with_vetted_info_from_specie_name("Papaipema nebris", "Researcher")
+# print("First prompt is intialized as: " , qa_prompt, "\n\n")
+
+
+memory = ConversationBufferMemory(memory_key="chat_history",output_key='answer', return_messages=True) # https://github.com/langchain-ai/langchain/issues/9394#issuecomment-1683538834
+
+
+if model_name==4:
+    llm_openai = ChatOpenAI(model_name="gpt-4-1106-preview" , temperature=0, max_tokens=max_tokens)  # TODO: NEW MODEL VERSION AVAILABLE 
+else:
+    llm_openai = ChatOpenAI(model_name="gpt-3.5-turbo-0125" , temperature=0, max_tokens=max_tokens)
+
+specie_selector="Papaipema nebris"
+filter = {
+    "$or": [
+        {"matched_specie_0": specie_selector},
+        {"matched_specie_1": specie_selector},
+        {"matched_specie_2": specie_selector},
+    ]
+}
+# retriever = vectordb.as_retriever(search_kwargs={'k':retriever_k_value, 'filter': filter})
+
+# qa_chain = ConversationalRetrievalChain.from_llm(
+#     llm_openai, retriever, memory=memory, verbose=False, return_source_documents=True,\
+#           combine_docs_chain_kwargs={'prompt': qa_prompt}
+
+# )
+# 
+
+def initialize_qa_chain(specie_selector, application_mode, model_name="GPT-4", database_persistent_directory=default_persist_directory_insects):
+    if model_name=="GPT-4":
+        chosen_llm=ChatOpenAI(model_name="gpt-4-1106-preview" , temperature=0, max_tokens=max_tokens)
+    elif model_name=="GPT-3.5":
+        chosen_llm=ChatOpenAI(model_name="gpt-3.5-turbo-0125" , temperature=0, max_tokens=max_tokens)
+    elif model_name=="Llama-3 70B":
+        chosen_llm = ChatOpenRouter(model_name="meta-llama/llama-3-70b-instruct", temperature=0,max_tokens=max_tokens )
+    elif model_name=="Llama-3 8B":
+        chosen_llm = ChatOpenRouter(model_name="meta-llama/llama-3-8b-instruct", temperature=0, max_tokens=max_tokens)
+    elif model_name=="Gemini-1.5 Pro":
+        chosen_llm = ChatOpenRouter(model_name="google/gemini-pro-1.5", temperature=0, max_tokens=max_tokens)
+    elif model_name=="Claude 3 Opus":
+        chosen_llm = ChatAnthropic(model_name='claude-3-opus-20240229', temperature=0, max_tokens=max_tokens)
+ 
+    else:
+        print("No appropriate llm was selected")
+        exit()
+    
+     
+
+    filter = {
+        "$or": [
+            {"matched_specie_0": specie_selector},
+            {"matched_specie_1": specie_selector},
+            {"matched_specie_2": specie_selector},
+            {"matched_specie_3": specie_selector},
+            {"matched_specie_4": specie_selector},
+            {"matched_specie_5": specie_selector},
+            {"matched_specie_6": specie_selector},
+            {"matched_specie_7": specie_selector},
+            {"matched_specie_8": specie_selector},
+            {"matched_specie_9": specie_selector},
+            {"matched_specie_10": specie_selector}
+        ]
+    }
+
+    embedding = OpenAIEmbeddings()
+    vectordb = Chroma(persist_directory=database_persistent_directory, 
+                    embedding_function=embedding)
+
+    print("got updated retriever without metadata filtering")
+    retriever = vectordb.as_retriever(search_kwargs={'k':retriever_k_value, 'filter': filter})
+    
+    memory = ConversationBufferMemory(memory_key="chat_history", output_key='answer', return_messages=True)
+    qa_prompt=get_prompt_with_vetted_info_from_specie_name(specie_selector, application_mode)
+    qa_chain = ConversationalRetrievalChain.from_llm(
+        chosen_llm, retriever, memory=memory, verbose=False, return_source_documents=True,
+        combine_docs_chain_kwargs={'prompt': qa_prompt}
+    )
+    
+    return qa_chain
+# result = qa_chain.invoke({"question": "where are stalk borer eggs laid?"})
+# print("Got the first LLM task working: ", result)
+
+
+#Application Interface: 
+with gr.Blocks(theme=gr.themes.Soft())  as demo:
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Markdown(
+                """
+                ![Logo](file/logo1.png)
+                """
+            )
+        with gr.Column(scale=1):
+            gr.Markdown(
+                """
+                ![Logo](file/logo2.png)
+                """
+            )
+
+    # Configure UI layout
+    chatbot = gr.Chatbot(height=600, label="AgLLM")
+    with gr.Row():
+        with gr.Column(scale=1):
+            with gr.Row():
+                    domain_name = gr.Dropdown(
+                        list(["Insects", "Weeds"]),
+                        value="Insects",
+                        label="Domain",
+                        info="Select Domain",
+                        interactive=True,
+                        scale=1,
+                        visible=True
+                    )
+
+                    # Model selection
+                    specie_selector = gr.Dropdown(
+                        species_list_insects,
+                        value=species_list_insects[0],
+                        label="Species",
+                        info="Select the Species",
+                        interactive=True,
+                        scale=1,
+                        visible=True
+                    )
+            with gr.Row():
+                    model_name = gr.Dropdown(
+                        list(["GPT-4", "GPT-3.5", "Llama-3 70B", "Llama-3 8B", "Gemini-1.5 Pro", "Claude 3 Opus"]),
+                        value="Llama-3 70B",
+                        label="LLM",
+                        info="Select the LLM",
+                        interactive=True,
+                        scale=1,
+                        visible=True
+                    )
+                    application_mode = gr.Dropdown(
+                        list(["Farmer", "Researcher"]),
+                        value="Researcher",
+                        label="Mode",
+                        info="Select the Mode",
+                        interactive=True,
+                        scale=1,
+                        visible=True
+                    )
+
+
+        with gr.Column(scale=2):
+            # User input prompt text field
+            user_prompt_message = gr.Textbox(placeholder="Please add user prompt here", label="User prompt")
+            with gr.Row():
+                # clear = gr.Button("Clear Conversation", scale=2)
+                submitBtn = gr.Button("Submit", scale=8)
+
+    state = gr.State([])
+    qa_chain_state = gr.State(value=None)
+
+    # Handle user message
+    def user(user_prompt_message, history):
+        # print("HISTORY IS: ", history)  # TODO: REMOVE IT LATER
+        if user_prompt_message != "":
+            return history + [[user_prompt_message, None]]
+        else:
+            return history + [["Invalid prompts - user prompt cannot be empty", None]]
+
+    # Chatbot logic for configuration, sending the prompts, rendering the streamed back generations, etc.
+    def bot(model_name, application_mode, user_prompt_message, history, messages_history, qa_chain, domain_name):
+        if qa_chain == None:
+            qa_chain=init_qa_chain(species_list_insects[0], application_mode, model_name, domain_name)
+
+        dialog = []
+        bot_message = ""
+        history[-1][1] = ""  # Placeholder for the answer
+        
+        dialog = [
+            {"role": "user", "content": user_prompt_message},
+        ]
+        messages_history += dialog
+        
+        # Queue for streamed character rendering
+        q = Queue()
+
+        # Async task for streamed chain results wired to callbacks we previously defined, so we don't block the UI
+
+        def task(user_prompt_message):
+            result = qa_chain.invoke({"question": user_prompt_message})
+            answer = result["answer"]
+
+            try:
+                answer_start = answer.find("Answer:")
+                reference_start = answer.find("Reference:")
+                
+                if answer_start != -1 and reference_start != -1:
+                    model_answer = answer[answer_start + len("Answer:"):reference_start].strip()
+                    reference = answer[reference_start + len("Reference:"):].strip()
+                    formatted_response = f"Answer:\n{model_answer}\n\nReferences:\n{reference}"
+                else:
+                    formatted_response = answer
+            except:
+                print(f"Error parsing so displaying the raw output")
+                formatted_response = answer
+            
+            return formatted_response
+                
+        history[-1][1] = task(user_prompt_message)
+        return [history, messages_history]
+
+    # Initialize the chat history with default system message
+    def init_history(messages_history):
+        messages_history = []
+        messages_history += [system_message]
+        return messages_history
+
+    # Clean up the user input text field
+    def input_cleanup():
+        return ""
+
+    def init_qa_chain(specie_selector, application_mode, model_name, domain_name):
+        if domain_name=="Insects":
+            qa_chain = initialize_qa_chain(specie_selector, application_mode, model_name, default_persist_directory_insects)
+        elif domain_name=="Weeds":
+            qa_chain = initialize_qa_chain(specie_selector, application_mode, model_name, default_persist_directory_weeds)
+        else:
+            print("No Appropriate Chain Selected")
+        return qa_chain
+
+    specie_selector.change(
+        init_qa_chain,
+        inputs=[specie_selector, application_mode,model_name, domain_name ],
+        outputs=[qa_chain_state]
+    )
+    model_name.change(
+        init_qa_chain,
+        inputs=[specie_selector, application_mode,model_name, domain_name ],
+        outputs=[qa_chain_state]
+    )
+
+    #####
+    def update_species_list(domain):
+        if domain == "Insects":
+            return gr.Dropdown( species_list_insects, value=species_list_insects[0], label="Species", info="Select the Species", interactive=True, scale=1, visible=True )
+        elif domain == "Weeds":
+            return gr.Dropdown( species_list_weeds, value=species_list_weeds[0], label="Species", info="Select the Species", interactive=True, scale=1, visible=True )
+
+    domain_name.change(
+        update_species_list,
+        inputs=[domain_name],
+        outputs=[specie_selector]
+    )
+
+    # When the user clicks Enter and the user message is submitted
+    user_prompt_message.submit(
+        user,
+        [user_prompt_message, chatbot],
+        [chatbot],
+        queue=False
+    ).then(
+        bot,
+        [model_name, application_mode, user_prompt_message, chatbot, state, qa_chain_state, domain_name],
+        [chatbot, state]
+    ).then(input_cleanup,
+        [],
+        [user_prompt_message],
+        queue=False
+    )
+
+    # When the user clicks the submit button
+    submitBtn.click(
+        user,
+        [user_prompt_message, chatbot],
+        [chatbot],
+        queue=False
+    ).then(
+        bot,
+        [model_name, application_mode, user_prompt_message, chatbot, state, qa_chain_state, domain_name],
+        [chatbot, state]
+    ).then(
+        input_cleanup,
+        [],
+        [user_prompt_message],
+        queue=False
+    )
+
+    # When the user clicks the clear button
+    # clear.click(lambda: None, None, chatbot, queue=False).success(init_history, [state], [state])
+    if __name__ == "__main__":
+    # demo.launch()
+        demo.queue().launch(allowed_paths=["/"], share=False, show_error=True)

app_backup_2.py

@@ -0,0 +1,510 @@
+import os
+# https://stackoverflow.com/questions/76175046/how-to-add-prompt-to-langchain-conversationalretrievalchain-chat-over-docs-with
+# again from: 
+# https://python.langchain.com/docs/integrations/providers/vectara/vectara_chat
+from langchain.document_loaders import PyPDFDirectoryLoader
+import pandas as pd
+import langchain
+from queue import Queue
+from typing import Any
+from langchain.llms.huggingface_text_gen_inference import HuggingFaceTextGenInference
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from langchain.schema import LLMResult
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.prompts.prompt import PromptTemplate
+from anyio.from_thread import start_blocking_portal #For model callback streaming
+
+
+from langchain.prompts import SystemMessagePromptTemplate, HumanMessagePromptTemplate, ChatPromptTemplate
+import os
+from dotenv import load_dotenv
+
+import streamlit as st
+import json
+from langchain.document_loaders import PyPDFLoader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.chains.question_answering import load_qa_chain
+from langchain.chat_models import ChatOpenAI
+# from langchain.chat_models import ChatAnthropic
+from langchain_anthropic import ChatAnthropic
+from langchain.vectorstores import Chroma
+import chromadb
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.llms import OpenAI
+from langchain.chains import RetrievalQA
+from langchain.document_loaders import TextLoader
+from langchain.document_loaders import DirectoryLoader
+from langchain_community.document_loaders import PyMuPDFLoader
+from langchain.schema import Document
+
+from langchain.memory import ConversationBufferMemory
+
+from langchain.chains.conversational_retrieval.prompts import CONDENSE_QUESTION_PROMPT
+from langchain.chains.conversational_retrieval.prompts import QA_PROMPT
+import gradio as gr
+from langchain.memory import ConversationBufferMemory
+from langchain.chains import ConversationalRetrievalChain
+print("Started")
+
+def get_species_list_from_db(db_name):
+    embedding = OpenAIEmbeddings()
+    vectordb_temp = Chroma(persist_directory=db_name, 
+                  embedding_function=embedding)
+    species_list=[]
+    for meta in vectordb_temp.get()["metadatas"] :
+        try:
+            matched_first_species = meta['matched_specie_0']
+        except KeyError:
+            continue
+        # Since each document is considered as a single chunk, the chunk_index is 0 for all
+        species_list.append( matched_first_species)
+
+    return species_list
+
+
+
+# default_persist_directory = './db5' # For deployement
+default_persist_directory_insects='./vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species'
+default_persist_directory_weeds='./vector-databases-deployed/db5-agllm-data-isu-field-weeds-all-species'
+
+species_list_insects=get_species_list_from_db(default_persist_directory_insects)
+species_list_weeds=get_species_list_from_db(default_persist_directory_weeds)
+# default_persist_directory = 'vector-databases/db5-pre-completion' # For Development
+csv_filepath1 = "./agllm-data/corrected/Corrected_supplemented-insect_data-2500-sorted.xlsx"
+csv_filepath2 = "./agllm-data/corrected/Corrected_supplemented-insect_data-remaining.xlsx"
+model_name=4
+max_tokens=400
+system_message = {"role": "system", "content": "You are a helpful assistant."} # TODO: double check how this plays out later. 
+langchain.debug=False # TODO: DOUBLE CHECK
+from langchain import globals
+globals.set_debug(False)
+
+retriever_k_value=3
+embedding = OpenAIEmbeddings()
+print("Started....")
+class ChatOpenRouter(ChatOpenAI):
+    openai_api_base: str
+    openai_api_key: str
+    model_name: str
+
+    def __init__(self,
+                 model_name: str,
+                 openai_api_key: [str] = None,
+                 openai_api_base: str = "https://openrouter.ai/api/v1",
+                 **kwargs):
+        openai_api_key = openai_api_key or os.getenv('OPENROUTER_API_KEY')
+        super().__init__(openai_api_base=openai_api_base,
+                         openai_api_key=openai_api_key,
+                         model_name=model_name, **kwargs)
+
+
+######### todo: skipping the first step
+
+
+
+# print(# Single example
+# vectordb.as_retriever(k=2, search_kwargs={"filter": {"matched_specie_0": "Hypagyrtis unipunctata"}, 'k':1}).get_relevant_documents(
+#     "Checking if retriever is correctly initalized?"
+# ))
+
+columns = ['species', 'common name', 'order', 'family',
+           'genus', 'Updated role in ecosystem', 'Proof',
+           'ipm strategies', 'size of insect', 'geographical spread',
+           'life cycle specifics', 'pest for plant species', 'species status',
+           'distribution area', 'appearance', 'identification']
+
+df1 = pd.read_excel(csv_filepath1, usecols=columns)
+df2 = pd.read_excel(csv_filepath2, usecols=columns)
+
+all_insects_data = pd.concat([df1, df2], ignore_index=True)
+
+def get_prompt_with_vetted_info_from_specie_name(search_for_specie, mode):
+
+    def read_and_format_filtered_csv_better(insect_specie):
+            filtered_data = all_insects_data[all_insects_data['species'] == insect_specie]
+            formatted_data = ""
+            # Format the filtered data
+            for index, row in filtered_data.iterrows():
+                    row_data = [f"{col}: {row[col]}" for col in filtered_data.columns]
+                    formatted_row = "\n".join(row_data)
+                    formatted_data += f"{formatted_row}\n"
+
+            return formatted_data
+
+            # Use the path to your CSV file here
+            
+    vetted_info=read_and_format_filtered_csv_better(search_for_specie)
+
+    if mode=="Farmer":
+            language_constraint="The language should be acustomed to the Farmers. Given question is likely to be asked by a farmer in the field will ask which will help to make decisions which are immediate and practical."
+    elif mode=="Researcher":
+            language_constraint="The language should be acustomed to a researcher. Given question is likely to be asked by a scientist which are comprehensive and aimed at exploring new knowledge or refining existing methodologies"
+    else:
+            print("No  valid mode provided. Exiting")
+            exit()
+#    general_system_template = """ 
+#    In every question you are provided information about the insect/weed. Two types of information are: First,  Vetted Information (which is same in every questinon) and Second, some context from external documents about an insect/weed species and a question by the user. answer the question according to these two types of informations. 
+#    ---- 
+#    Vetted info is as follows:
+#    {vetted_info}
+#    ----
+#    The context retrieved for documents about this particular question is as follows:
+#    {context}
+#    ----
+#    Additional Instruction:
+#    1. Reference Constraint
+#    At the end of each answer provide the source/reference for the given data in following format: 
+#    \n\n[enter two new lines before writing below] References:
+#    Vetted Information Used: Write what was used from the document for coming up with the answer above. Write exact part of lines. If nothing, write 'Nothing'. 
+#    Documents Used: Write what was used from the document for coming up with the answer above. If nothing, write 'Nothing'. Write exact part of lines and document used. 
+#    2. Information Constraint: 
+#    Only answer the question from information provided otherwise say you dont know. You have to answer in 50 words including references. Prioritize information in documents/context over vetted information. And first mention the warnings/things to be careful about. 
+#    3. Language constraint:
+#    {language_constraint}
+
+#    ----
+#    """.format(vetted_info=vetted_info, language_constraint=language_constraint,context="{context}", )
+
+    general_system_template = f""" 
+    You are an AI assistant specialized in providing information about insects/weeds. Answer the user's question based on the available information or your general knowledge.
+
+    The context retrieved for this question is as follows:
+    {{context}}
+
+    Instructions:
+    1. Evaluate the relevance of the provided context to the question.
+    2. If the context contains relevant information, use it to answer the question and explicitly mention "Based on provided information" in your source.
+    3. If the context does not contain relevant information, use your general knowledge to answer the question and state "Based on general knowledge" as the source.
+    4. Format your response as follows:
+       Answer: Provide a concise answer in less than 50 words.
+       Source: State either "Based on provided information" or "Based on general knowledge".
+
+    5. Language constraint:
+       {language_constraint}
+
+    Question: {{question}}
+    """
+
+    general_user_template = "Question:```{question}```"
+    messages_formatted = [
+            SystemMessagePromptTemplate.from_template(general_system_template),
+            HumanMessagePromptTemplate.from_template(general_user_template)
+    ]
+    qa_prompt = ChatPromptTemplate.from_messages( messages_formatted )
+#    print(qa_prompt)
+    return qa_prompt
+
+
+    
+
+
+qa_prompt=get_prompt_with_vetted_info_from_specie_name("Papaipema nebris", "Researcher")
+# print("First prompt is intialized as: " , qa_prompt, "\n\n")
+
+
+memory = ConversationBufferMemory(memory_key="chat_history",output_key='answer', return_messages=True) # https://github.com/langchain-ai/langchain/issues/9394#issuecomment-1683538834
+
+
+if model_name==4:
+    llm_openai = ChatOpenAI(model_name="gpt-4-1106-preview" , temperature=0, max_tokens=max_tokens)  # TODO: NEW MODEL VERSION AVAILABLE 
+else:
+    llm_openai = ChatOpenAI(model_name="gpt-3.5-turbo-0125" , temperature=0, max_tokens=max_tokens)
+
+specie_selector="Papaipema nebris"
+filter = {
+    "$or": [
+        {"matched_specie_0": specie_selector},
+        {"matched_specie_1": specie_selector},
+        {"matched_specie_2": specie_selector},
+    ]
+}
+# retriever = vectordb.as_retriever(search_kwargs={'k':retriever_k_value, 'filter': filter})
+
+# qa_chain = ConversationalRetrievalChain.from_llm(
+#     llm_openai, retriever, memory=memory, verbose=False, return_source_documents=True,\
+#           combine_docs_chain_kwargs={'prompt': qa_prompt}
+
+# )
+# 
+
+def initialize_qa_chain(specie_selector, application_mode, model_name="GPT-4", database_persistent_directory=default_persist_directory_insects):
+    if model_name=="GPT-4":
+        chosen_llm=ChatOpenAI(model_name="gpt-4-1106-preview" , temperature=0, max_tokens=max_tokens)
+    elif model_name=="GPT-3.5":
+        chosen_llm=ChatOpenAI(model_name="gpt-3.5-turbo-0125" , temperature=0, max_tokens=max_tokens)
+    elif model_name=="Llama-3 70B":
+        chosen_llm = ChatOpenRouter(model_name="meta-llama/llama-3-70b-instruct", temperature=0,max_tokens=max_tokens )
+    elif model_name=="Llama-3 8B":
+        chosen_llm = ChatOpenRouter(model_name="meta-llama/llama-3-8b-instruct", temperature=0, max_tokens=max_tokens)
+    elif model_name=="Gemini-1.5 Pro":
+        chosen_llm = ChatOpenRouter(model_name="google/gemini-pro-1.5", temperature=0, max_tokens=max_tokens)
+    elif model_name=="Claude 3 Opus":
+        chosen_llm = ChatAnthropic(model_name='claude-3-opus-20240229', temperature=0, max_tokens=max_tokens)
+ 
+    else:
+        print("No appropriate llm was selected")
+        exit()
+    
+     
+
+    filter = {
+        "$or": [
+            {"matched_specie_0": specie_selector},
+            {"matched_specie_1": specie_selector},
+            {"matched_specie_2": specie_selector},
+            {"matched_specie_3": specie_selector},
+            {"matched_specie_4": specie_selector},
+            {"matched_specie_5": specie_selector},
+            {"matched_specie_6": specie_selector},
+            {"matched_specie_7": specie_selector},
+            {"matched_specie_8": specie_selector},
+            {"matched_specie_9": specie_selector},
+            {"matched_specie_10": specie_selector}
+        ]
+    }
+
+    embedding = OpenAIEmbeddings()
+    vectordb = Chroma(persist_directory=database_persistent_directory, 
+                    embedding_function=embedding)
+
+    # print("got updated retriever without metadata filtering")
+    retriever = vectordb.as_retriever(search_kwargs={'k':retriever_k_value, 'filter': filter})
+    
+    memory = ConversationBufferMemory(memory_key="chat_history", output_key='answer', return_messages=True)
+    qa_prompt=get_prompt_with_vetted_info_from_specie_name(specie_selector, application_mode)
+    qa_chain = ConversationalRetrievalChain.from_llm(
+        chosen_llm, retriever, memory=memory, verbose=False, return_source_documents=True,
+        combine_docs_chain_kwargs={'prompt': qa_prompt}
+    )
+    
+    return qa_chain
+# result = qa_chain.invoke({"question": "where are stalk borer eggs laid?"})
+# print("Got the first LLM task working: ", result)
+
+
+#Application Interface: 
+with gr.Blocks(theme=gr.themes.Soft())  as demo:
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Markdown(
+                """
+                ![Logo](file/logo1.png)
+                """
+            )
+        with gr.Column(scale=1):
+            gr.Markdown(
+                """
+                ![Logo](file/logo2.png)
+                """
+            )
+
+    # Configure UI layout
+    chatbot = gr.Chatbot(height=600, label="AgLLM")
+    with gr.Row():
+        with gr.Column(scale=1):
+            with gr.Row():
+                    domain_name = gr.Dropdown(
+                        list(["Insects", "Weeds"]),
+                        value="Insects",
+                        label="Domain",
+                        info="Select Domain",
+                        interactive=True,
+                        scale=1,
+                        visible=True
+                    )
+
+                    # Model selection
+                    specie_selector = gr.Dropdown(
+                        species_list_insects,
+                        value=species_list_insects[0],
+                        label="Species",
+                        info="Select the Species",
+                        interactive=True,
+                        scale=1,
+                        visible=True
+                    )
+            with gr.Row():
+                    model_name = gr.Dropdown(
+                        list(["GPT-4", "GPT-3.5", "Llama-3 70B", "Llama-3 8B", "Gemini-1.5 Pro", "Claude 3 Opus"]),
+                        value="Llama-3 70B",
+                        label="LLM",
+                        info="Select the LLM",
+                        interactive=True,
+                        scale=1,
+                        visible=True
+                    )
+                    application_mode = gr.Dropdown(
+                        list(["Farmer", "Researcher"]),
+                        value="Researcher",
+                        label="Mode",
+                        info="Select the Mode",
+                        interactive=True,
+                        scale=1,
+                        visible=True
+                    )
+
+
+        with gr.Column(scale=2):
+            # User input prompt text field
+            user_prompt_message = gr.Textbox(placeholder="Please add user prompt here", label="User prompt")
+            with gr.Row():
+                # clear = gr.Button("Clear Conversation", scale=2)
+                submitBtn = gr.Button("Submit", scale=8)
+
+    state = gr.State([])
+    qa_chain_state = gr.State(value=None)
+
+    # Handle user message
+    def user(user_prompt_message, history):
+        # print("HISTORY IS: ", history)  # TODO: REMOVE IT LATER
+        if user_prompt_message != "":
+            return history + [[user_prompt_message, None]]
+        else:
+            return history + [["Invalid prompts - user prompt cannot be empty", None]]
+
+    # Chatbot logic for configuration, sending the prompts, rendering the streamed back generations, etc.
+    def bot(model_name, application_mode, user_prompt_message, history, messages_history, qa_chain, domain_name):
+        if qa_chain == None:
+            qa_chain=init_qa_chain(species_list_insects[0], application_mode, model_name, domain_name)
+
+        dialog = []
+        bot_message = ""
+        history[-1][1] = ""  # Placeholder for the answer
+        
+        dialog = [
+            {"role": "user", "content": user_prompt_message},
+        ]
+        messages_history += dialog
+        
+        # Queue for streamed character rendering
+        q = Queue()
+
+        # Async task for streamed chain results wired to callbacks we previously defined, so we don't block the UI
+
+        def task(user_prompt_message):
+            result = qa_chain.invoke({"question": user_prompt_message})
+            answer = result["answer"]
+            source_documents = result.get("source_documents", [])
+            print("SOURCE DOCUMENTS: ", source_documents)
+
+            try:
+                answer_start = answer.find("Answer:")
+                source_start = answer.find("Source:")
+                
+                if answer_start != -1 and source_start != -1:
+                    model_answer = answer[answer_start + len("Answer:"):source_start].strip()
+                    source = answer[source_start + len("Source:"):].strip()
+                    
+                    if "Based on provided information" in source:
+                        # Extract the most relevant source document
+                        if source_documents:
+                            doc = source_documents[0]
+                            species_name = doc.metadata.get('matched_specie_0', 'Unspecified species')
+                            
+                            if domain_name == "Insects":
+                                formatted_source = f'Iowa State University Extension and Outreach. "Field Crop Insects." Iowa State University Extension Store, June 26, 2023. https://store.extension.iastate.edu/product/13725. Information about {species_name}.'
+                            elif domain_name == "Weeds":
+                                formatted_source = f'Iowa State University Extension and Outreach. "Weed Identification Field Guide 2nd Edition." Iowa State University Extension Store, August 2015. https://store.extension.iastate.edu/product/13358. Information about {species_name}.'
+                            else:
+                                formatted_source = f"Based on provided information about {species_name}, but domain is unspecified."
+                        else:
+                            formatted_source = "Based on provided information, but source details are unavailable."
+                    else:
+                        formatted_source = source
+                    
+                    formatted_response = f"Answer:\n{model_answer}\n\nSource:\n{formatted_source}"
+                else:
+                    formatted_response = answer
+            except Exception as e:
+                print(f"Error parsing output: {e}")
+                formatted_response = answer
+            
+            return formatted_response
+                
+        history[-1][1] = task(user_prompt_message)
+        return [history, messages_history]
+
+    # Initialize the chat history with default system message
+    def init_history(messages_history):
+        messages_history = []
+        messages_history += [system_message]
+        return messages_history
+
+    # Clean up the user input text field
+    def input_cleanup():
+        return ""
+
+    def init_qa_chain(specie_selector, application_mode, model_name, domain_name):
+        if domain_name=="Insects":
+            qa_chain = initialize_qa_chain(specie_selector, application_mode, model_name, default_persist_directory_insects)
+        elif domain_name=="Weeds":
+            qa_chain = initialize_qa_chain(specie_selector, application_mode, model_name, default_persist_directory_weeds)
+        else:
+            print("No Appropriate Chain Selected")
+        return qa_chain
+
+    specie_selector.change(
+        init_qa_chain,
+        inputs=[specie_selector, application_mode,model_name, domain_name ],
+        outputs=[qa_chain_state]
+    )
+    model_name.change(
+        init_qa_chain,
+        inputs=[specie_selector, application_mode,model_name, domain_name ],
+        outputs=[qa_chain_state]
+    )
+
+    #####
+    def update_species_list(domain):
+        if domain == "Insects":
+            return gr.Dropdown( species_list_insects, value=species_list_insects[0], label="Species", info="Select the Species", interactive=True, scale=1, visible=True )
+        elif domain == "Weeds":
+            return gr.Dropdown( species_list_weeds, value=species_list_weeds[0], label="Species", info="Select the Species", interactive=True, scale=1, visible=True )
+
+    domain_name.change(
+        update_species_list,
+        inputs=[domain_name],
+        outputs=[specie_selector]
+    )
+
+    # When the user clicks Enter and the user message is submitted
+    user_prompt_message.submit(
+        user,
+        [user_prompt_message, chatbot],
+        [chatbot],
+        queue=False
+    ).then(
+        bot,
+        [model_name, application_mode, user_prompt_message, chatbot, state, qa_chain_state, domain_name],
+        [chatbot, state]
+    ).then(input_cleanup,
+        [],
+        [user_prompt_message],
+        queue=False
+    )
+
+    # When the user clicks the submit button
+    submitBtn.click(
+        user,
+        [user_prompt_message, chatbot],
+        [chatbot],
+        queue=False
+    ).then(
+        bot,
+        [model_name, application_mode, user_prompt_message, chatbot, state, qa_chain_state, domain_name],
+        [chatbot, state]
+    ).then(
+        input_cleanup,
+        [],
+        [user_prompt_message],
+        queue=False
+    )
+
+    # When the user clicks the clear button
+    # clear.click(lambda: None, None, chatbot, queue=False).success(init_history, [state], [state])
+    if __name__ == "__main__":
+    # demo.launch()
+        demo.queue().launch(allowed_paths=["/"], share=False, show_error=True)

app_database_prep.py

@@ -0,0 +1,234 @@
+import os
+import shutil
+
+from langchain.document_loaders import PyPDFDirectoryLoader
+import pandas as pd
+import langchain
+from queue import Queue
+from typing import Any, List
+from langchain.llms.huggingface_text_gen_inference import HuggingFaceTextGenInference
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from langchain.schema import LLMResult
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.prompts.prompt import PromptTemplate
+from anyio.from_thread import start_blocking_portal #For model callback streaming
+
+langchain.debug=True # TODO: DOUBLE CHECK
+
+system_message = {"role": "system", "content": "You are a helpful assistant."} # TODO: double check how this plays out later. 
+import os
+from dotenv import load_dotenv
+
+import streamlit as st
+
+from langchain.document_loaders import PyPDFLoader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.chains.question_answering import load_qa_chain
+from langchain.chat_models import ChatOpenAI
+from langchain.vectorstores import Chroma
+import chromadb
+
+
+## added information in metadata: 
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.llms import OpenAI
+from langchain.chains import RetrievalQA
+from langchain.document_loaders import TextLoader
+from langchain.document_loaders import DirectoryLoader
+from langchain_community.document_loaders import PyMuPDFLoader
+from langchain.schema import Document
+
+
+# Function to process a sheet from the Excel file
+def process_excel_sheet(
+    excel_path: str, 
+    sheet_name: str, 
+    region: str, 
+    splitter: RecursiveCharacterTextSplitter
+) -> List[Document]:
+    """Loads data from an Excel sheet, creates Documents, splits them, and adds metadata."""
+    print(f"--- Processing Excel Sheet: {sheet_name} (Region: {region}) ---")
+    try:
+        df = pd.read_excel(excel_path, sheet_name=sheet_name)
+        print(f"Excel Data Head ({sheet_name}):\\n", df.head())
+    except Exception as e:
+        print(f"Error loading sheet '{sheet_name}' from {excel_path}: {e}")
+        return []
+
+    initial_documents = []
+    for index, row in df.iterrows():
+        ipm_info = str(row['IPM Info']) if pd.notna(row['IPM Info']) else ""
+        # Check if essential columns exist and are not empty (removed accuracy check)
+        if pd.isna(row['Common Name']) or pd.isna(row['Species']):
+             print(f"Skipping row {index+2} in sheet '{sheet_name}' due to missing essential data (Common Name or Species).")
+             continue
+
+        doc = Document(
+            page_content=ipm_info,
+            metadata={
+                "source": f"{excel_path}#sheet={sheet_name}#row={index+2}",
+                "common_name": row['Common Name'],
+                "species": row['Species'],
+                "matched_specie_0": row['Species'],
+                "region": region
+            }
+        )
+        initial_documents.append(doc)
+
+    if initial_documents:
+        print(f"First Document from {sheet_name} (before splitting):\\n", initial_documents[0])
+    else:
+        print(f"No documents created from sheet: {sheet_name}")
+        return [] # Return empty list if no documents were created
+
+    split_documents = []
+    for doc in initial_documents:
+        splits = splitter.split_documents([doc])
+        for i, split_doc in enumerate(splits, start=1):
+            metadata = split_doc.metadata.copy()
+            metadata["source"] = f"{metadata['source']}#chunk{i}"
+            split_doc.metadata = metadata
+            split_documents.append(split_doc)
+
+    if split_documents:
+        print(f"First Document chunk from {sheet_name}:\\n", split_documents[0])
+    
+    print(f"Finished processing sheet: {sheet_name}. Found {len(split_documents)} chunks.")
+    print("---------------------------------------------------")
+    return split_documents
+
+# --- Main Script Logic ---
+
+# loader = DirectoryLoader('./agllm-data/', glob="./*.pdf", loader_cls=PyMuPDFLoader)
+# loader = DirectoryLoader('/u/marshad/data/agllm-data/', glob='**/*.pdf', loader_cls=PyMuPDFLoader)
+data_domain_identifier="agllm-data-isu-field-insects-all-species"
+persist_directory = f'vector-databases-deployed/db5-{data_domain_identifier}' # was full
+loader = DirectoryLoader(f'agllm-data/{data_domain_identifier}', glob='**/*.pdf', loader_cls=PyMuPDFLoader)#,# was full, loader_kwargs={'chunk_size':512})
+chunk_size_input=512 
+metadata_raw = pd.read_csv(f"./agllm-data/{data_domain_identifier}/matched_species_results_v2.csv")
+documents = loader.load()
+
+## Load Excel File Path (Define once)
+excel_file_path = "agllm-data/PestID Species.xlsx"
+
+
+## Process PDF documents and add metadata
+print("--- Processing PDF Documents ---")
+pdf_documents_for_splitting = [] # Prepare list to hold docs with added metadata
+for doc in documents:
+    # Add region for PDF docs
+    doc.metadata["region"] = "United States" 
+    
+    # Add species metadata (existing logic)
+    file_name_associated_with_this_doc = doc.metadata["source"].split('/')[-1]
+    matching_species_for_this_file_name = metadata_raw[metadata_raw["File Name"].str.lower() == file_name_associated_with_this_doc.lower()]["Species"]
+    # Ensure matching_species_for_this_file_name is iterable and not empty
+    if not matching_species_for_this_file_name.empty:
+        for specie_index in range(len(matching_species_for_this_file_name)):
+             # Check if specie_index is within bounds (although range should handle this)
+            if specie_index < len(matching_species_for_this_file_name):
+                specie_name = matching_species_for_this_file_name.iloc[specie_index]
+                doc.metadata["matched_specie_" + str(specie_index)] = specie_name
+            else:
+                # This case should ideally not happen with range(len(...))
+                print(f"Warning: Specie index {specie_index} out of bounds for file {file_name_associated_with_this_doc}")
+    else:
+         print(f"Warning: No matching species found in CSV for PDF: {file_name_associated_with_this_doc}")
+
+    pdf_documents_for_splitting.append(doc) # Add modified doc to new list
+
+
+# Initialize Text Splitter
+text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size_input, chunk_overlap=10)
+
+
+# Split PDF documents
+pdf_splitted_documents = []
+for doc in pdf_documents_for_splitting: # Use the list with added metadata
+    splits = text_splitter.split_documents([doc])
+    for i, split_doc in enumerate(splits, start=1):
+        metadata = split_doc.metadata.copy()
+        # Update source for PDF chunks (existing logic)
+        source_base = metadata.get('source', 'unknown_source')
+        page_num = metadata.get('page', 'unknown_page')
+        metadata["source"] = f"{source_base}#page{page_num}#chunk{i}"
+        # Remove the raw page number if desired, as it's now in the source string
+        # metadata.pop('page', None) 
+        split_doc.metadata = metadata
+        pdf_splitted_documents.append(split_doc)
+
+print("First PDF Document chunk:\\n", pdf_splitted_documents[0] if pdf_splitted_documents else "No PDF documents processed")
+print(f"Count after PDF processing: {len(pdf_splitted_documents)}")
+print("---------------------------------------------------")
+
+
+# Process Excel Sheets using the function
+india_splitted_documents = process_excel_sheet(
+    excel_path=excel_file_path, 
+    sheet_name="India", 
+    region="India", 
+    splitter=text_splitter
+)
+
+africa_splitted_documents = process_excel_sheet(
+    excel_path=excel_file_path, 
+    sheet_name="Africa", 
+    region="Africa", 
+    splitter=text_splitter
+)
+
+
+# Combine lists from all sources
+splitted_documents = pdf_splitted_documents + india_splitted_documents + africa_splitted_documents
+
+
+# print(splitted_documents[0]) # Original print statement - commented out as we print chunks above
+print("=== Combined Processing Done ===") # Adjusted print statement
+print(f"Total documents after combining PDF, India, and Africa sources: {len(splitted_documents)}")
+print("=============================")
+
+
+
+# ONLY FOR THE FIRST TIME
+
+# Check if the persist directory exists and delete it to ensure a fresh start
+if os.path.exists(persist_directory):
+    print(f"Deleting existing vector database directory: {persist_directory}")
+    shutil.rmtree(persist_directory)
+    print(f"Directory deleted.")
+else:
+    print(f"Vector database directory not found, creating a new one: {persist_directory}")
+
+embedding = OpenAIEmbeddings()
+vectordb = Chroma.from_documents(documents=splitted_documents, 
+                                 embedding=embedding,
+                                 persist_directory=persist_directory)
+
+# persiste the db to disk
+vectordb.persist()
+vectordb = None
+
+
+# Now we can load the persisted database from disk, and use it as normal. 
+
+vectordb = Chroma(persist_directory=persist_directory, 
+                  embedding_function=embedding)
+
+
+print(vectordb.get())
+
+#just a test script: 
+
+specie_selector="Aphis spiraecola"
+filter = {
+    "$or": [
+        {"matched_specie_0": specie_selector},
+        {"matched_specie_1": specie_selector},
+        {"matched_specie_2": specie_selector},
+    ]
+}
+answer = vectordb.as_retriever(search_kwargs={'k':10, 'filter': filter}).get_relevant_documents(
+    "anything else.?")
+print(answer)

outdated-files/agllm-data.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e644edb3d6ae6153f2411ff768ba5047314a9fdd92f97b3d4fa25b8e28e98a5
+size 4658799

outdated-files/app-basic.py

@@ -0,0 +1,19 @@
+# import gradio as gr
+
+# def greet(name):
+#     return "Hello " + name + "!"
+
+# demo = gr.Interface(fn=greet, inputs="textbox", outputs="textbox")
+    
+# demo.launch()  # Share your demo with just 1 extra parameter 
+
+
+import gradio as gr
+
+def greet(name):
+    return "Hello " + name + "!"
+
+demo = gr.Interface(fn=greet, inputs="textbox", outputs="textbox")
+
+if __name__ == "__main__":
+    demo.launch()

outdated-files/app-old.py

@@ -0,0 +1,334 @@
+import os
+# https://stackoverflow.com/questions/76175046/how-to-add-prompt-to-langchain-conversationalretrievalchain-chat-over-docs-with
+# again from: 
+# https://python.langchain.com/docs/integrations/providers/vectara/vectara_chat
+from langchain.document_loaders import PyPDFDirectoryLoader
+import pandas as pd
+import langchain
+from queue import Queue
+from typing import Any
+from langchain.llms.huggingface_text_gen_inference import HuggingFaceTextGenInference
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from langchain.schema import LLMResult
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.prompts.prompt import PromptTemplate
+from anyio.from_thread import start_blocking_portal #For model callback streaming
+
+from langchain.prompts import SystemMessagePromptTemplate, HumanMessagePromptTemplate, ChatPromptTemplate
+import os
+from dotenv import load_dotenv
+
+import streamlit as st
+
+from langchain.document_loaders import PyPDFLoader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.chains.question_answering import load_qa_chain
+from langchain.chat_models import ChatOpenAI
+from langchain.vectorstores import Chroma
+import chromadb
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.llms import OpenAI
+from langchain.chains import RetrievalQA
+from langchain.document_loaders import TextLoader
+from langchain.document_loaders import DirectoryLoader
+from langchain_community.document_loaders import PyMuPDFLoader
+from langchain.schema import Document
+
+from langchain.memory import ConversationBufferMemory
+
+from langchain.chains import ConversationalRetrievalChain
+from langchain.chains.conversational_retrieval.prompts import CONDENSE_QUESTION_PROMPT
+from langchain.chains.conversational_retrieval.prompts import QA_PROMPT
+import gradio as gr
+from langchain.memory import ConversationBufferMemory
+from langchain.chains import ConversationalRetrievalChain
+
+persist_directory = '/projects/bcjp/marshad/agllm/db5'
+csv_filepath1 = "./agllm-data/corrected/Corrected_supplemented-insect_data-2500-sorted.xlsx"
+csv_filepath2 = "./agllm-data/corrected/Corrected_supplemented-insect_data-remaining.xlsx"
+model_name=4
+max_tokens=400
+system_message = {"role": "system", "content": "You are a helpful assistant."} # TODO: double check how this plays out later. 
+langchain.debug=True # TODO: DOUBLE CHECK
+retriever_k_value=2
+embedding = OpenAIEmbeddings()
+
+######### todo: skipping the first step
+
+embedding = OpenAIEmbeddings()
+vectordb = Chroma(persist_directory=persist_directory, 
+                  embedding_function=embedding)
+
+retriever = vectordb.as_retriever()
+
+print(# Single example
+vectordb.as_retriever(k=2, search_kwargs={"filter": {"matched_specie_0": "Hypagyrtis unipunctata"}, 'k':1}).get_relevant_documents(
+    "Checking if retriever is correctly initalized?"
+))
+
+columns = ['species', 'common name', 'order', 'family',
+           'genus', 'Updated role in ecosystem', 'Proof',
+           'ipm strategies', 'size of insect', 'geographical spread',
+           'life cycle specifics', 'pest for plant species', 'species status',
+           'distribution area', 'appearance', 'identification']
+
+df1 = pd.read_excel(csv_filepath1, usecols=columns)
+df2 = pd.read_excel(csv_filepath2, usecols=columns)
+
+all_insects_data = pd.concat([df1, df2], ignore_index=True)
+
+def get_prompt_with_vetted_info_from_specie_name(search_for_specie, mode):
+
+       def read_and_format_filtered_csv_better(insect_specie):
+              filtered_data = all_insects_data[all_insects_data['species'] == insect_specie]
+              formatted_data = ""
+              # Format the filtered data
+              for index, row in filtered_data.iterrows():
+                     row_data = [f"{col}: {row[col]}" for col in filtered_data.columns]
+                     formatted_row = "\n".join(row_data)
+                     formatted_data += f"{formatted_row}\n"
+
+              return formatted_data
+
+              # Use the path to your CSV file here
+              
+       vetted_info=read_and_format_filtered_csv_better(search_for_specie)
+       if mode=="user":
+              language_constraint="The language should be acustomed to the end user. This question is likely asked by a farmer. So, answer things in their language. Bur for referencing information, you can use the original content. This is only for the main answer to be provided by you."
+       elif mode=="researcher":
+              language_constraint="The language should be acustomed to a researcher. This question is likely asked by an academic researcher. So you can use all the technical terms freely. And for referencing information, you can use the original content. This is only for the main answer to be provided by you."
+       else:
+              print("No  valid model provided. Exiting")
+              exit()
+       general_system_template = """ 
+       In every question you are provided information about the insect. Two types of information are: First,  Vetted Information (which is same in every questinon) and Second, some context from external documents about an insect specie and a question by the user. answer the question according to these two types of informations. 
+       ---- 
+       Vetted info is as follows:
+       {vetted_info}
+       ----
+       The context retrieved for documents about this particular question is a as follows:
+       {context}
+       ----
+       Additional Instruction:
+       1. Reference Constraint
+       At the end of each answer provide the source/reference for the given data in following format: 
+       \n\n[enter two new lines before writing below] References:
+       Vetted Information Used: Write what was used from the document for coming up with the answer above. Write exact part of lines. If nothing, write 'Nothing'. 
+       Documents Used: Write what was used from the document for coming up with the answer above. If nothing, write 'Nothing'. Write exact part of lines and document used. 
+       2. Information Constraint: 
+       Only answer the question from information provided otherwise say you dont know. You have to answer in 150 words including references. Prioritize information in documents/context over vetted information. And first mention the warnings/things to be careful about. 
+       3. Language constraint:
+       {language_constraint}
+
+       ----
+       """.format(vetted_info=vetted_info, language_constraint=language_constraint,context="{context}", )
+
+
+       general_user_template = "Question:```{question}```"
+       messages_formatted = [
+              SystemMessagePromptTemplate.from_template(general_system_template),
+              # HumanMessagePromptTemplate.from_template(general_system_template),
+              HumanMessagePromptTemplate.from_template(general_user_template)
+       ]
+       qa_prompt = ChatPromptTemplate.from_messages( messages_formatted )
+       print(qa_prompt)
+       return qa_prompt
+qa_prompt=get_prompt_with_vetted_info_from_specie_name("Papaipema nebris", "researcher")
+print("First prompt is intialized as: " , qa_prompt, "\n\n")
+
+
+memory = ConversationBufferMemory(memory_key="chat_history",output_key='answer', return_messages=True) # https://github.com/langchain-ai/langchain/issues/9394#issuecomment-1683538834
+
+
+if model_name==4:
+    llm_openai = ChatOpenAI(model_name="gpt-4-1106-preview" , temperature=0, max_tokens=max_tokens)  # TODO: NEW MODEL VERSION AVAILABLE 
+else:
+    llm_openai = ChatOpenAI(model_name="gpt-3.5-turbo-0125" , temperature=0, max_tokens=max_tokens)
+
+specie_selector="Papaipema nebris"
+filter = {
+    "$or": [
+        {"matched_specie_0": specie_selector},
+        {"matched_specie_1": specie_selector},
+        {"matched_specie_2": specie_selector},
+    ]
+}
+retriever = vectordb.as_retriever(search_kwargs={'k':retriever_k_value, 'filter': filter})
+
+qa_chain = ConversationalRetrievalChain.from_llm(
+    llm_openai, retriever, memory=memory, verbose=False, return_source_documents=True,\
+          combine_docs_chain_kwargs={'prompt': qa_prompt}
+
+)
+# 
+
+def initialize_qa_chain(specie_selector, application_mode):
+
+    filter = {
+        "$or": [
+            {"matched_specie_0": specie_selector},
+            {"matched_specie_1": specie_selector},
+            {"matched_specie_2": specie_selector},
+        ]
+    }
+    retriever = vectordb.as_retriever(search_kwargs={'k':2, 'filter': filter})
+    
+    memory = ConversationBufferMemory(memory_key="chat_history", output_key='answer', return_messages=True)
+    qa_prompt=get_prompt_with_vetted_info_from_specie_name(specie_selector, application_mode)
+    qa_chain = ConversationalRetrievalChain.from_llm(
+        llm_openai, retriever, memory=memory, verbose=False, return_source_documents=True,
+        combine_docs_chain_kwargs={'prompt': qa_prompt}
+    )
+    
+    return qa_chain
+result = qa_chain.invoke({"question": "where are stalk borer eggs laid?"})
+print("Got the first LLM task working: ", result)
+
+
+#Application Interface: 
+with gr.Blocks(theme=gr.themes.Soft())  as demo:
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Markdown(
+                """
+                ![Logo](file/logo1.png)
+                """
+            )
+        with gr.Column(scale=1):
+            gr.Markdown(
+                """
+                ![Logo](file/logo2.png)
+                """
+            )
+
+    # Configure UI layout
+    chatbot = gr.Chatbot(height=600, label="AgLLM22")
+    with gr.Row():
+        with gr.Column(scale=1):
+            with gr.Row():
+                # Model selection
+                specie_selector = gr.Dropdown(
+                    list(["Papaipema nebris", "Nomophila nearctica"]),
+                    value="Papaipema nebris",
+                    label="Species",
+                    info="Select the Species",
+                    interactive=True,
+                    scale=2,
+                    visible=True
+                )
+            with gr.Row():
+                application_mode = gr.Dropdown(
+                    list(["user", "researcher"]),
+                    value="researcher",
+                    label="Mode",
+                    info="Select the Mode",
+                    interactive=True,
+                    scale=2,
+                    visible=True
+                )
+            with gr.Row():
+                pass
+        with gr.Column(scale=2):
+            # User input prompt text field
+            user_prompt_message = gr.Textbox(placeholder="Please add user prompt here", label="User prompt")
+            with gr.Row():
+                # clear = gr.Button("Clear Conversation", scale=2)
+                submitBtn = gr.Button("Submit", scale=8)
+
+    state = gr.State([])
+    qa_chain_state = gr.State(value=None)
+
+    # Handle user message
+    def user(user_prompt_message, history):
+        print("HISTORY IS: ", history)  # TODO: REMOVE IT LATER
+        if user_prompt_message != "":
+            return history + [[user_prompt_message, None]]
+        else:
+            return history + [["Invalid prompts - user prompt cannot be empty", None]]
+
+    # Chatbot logic for configuration, sending the prompts, rendering the streamed back generations, etc.
+    def bot(application_mode, user_prompt_message, history, messages_history, qa_chain):
+        if qa_chain == None:
+            qa_chain=init_qa_chain("Papaipema nebris", application_mode)
+
+        dialog = []
+        bot_message = ""
+        history[-1][1] = ""  # Placeholder for the answer
+        
+        dialog = [
+            {"role": "user", "content": user_prompt_message},
+        ]
+        messages_history += dialog
+        
+        # Queue for streamed character rendering
+        q = Queue()
+
+        # Async task for streamed chain results wired to callbacks we previously defined, so we don't block the UI
+        def task(user_prompt_message):
+            ret = qa_chain.invoke({"question": user_prompt_message})["answer"]
+            return ret
+        
+        history[-1][1] = task(user_prompt_message)
+        return [history, messages_history]
+
+    # Initialize the chat history with default system message
+    def init_history(messages_history):
+        messages_history = []
+        messages_history += [system_message]
+        return messages_history
+
+    # Clean up the user input text field
+    def input_cleanup():
+        return ""
+
+    def init_qa_chain(specie_selector, application_mode):
+        qa_chain = initialize_qa_chain(specie_selector, application_mode)
+        return qa_chain
+
+    specie_selector.change(
+        init_qa_chain,
+        inputs=[specie_selector, application_mode],
+        outputs=[qa_chain_state]
+    )
+
+    # When the user clicks Enter and the user message is submitted
+    user_prompt_message.submit(
+        user,
+        [user_prompt_message, chatbot],
+        [chatbot],
+        queue=False
+    ).then(
+        bot,
+        [application_mode, user_prompt_message, chatbot, state, qa_chain_state],
+        [chatbot, state]
+    ).then(input_cleanup,
+        [],
+        [user_prompt_message],
+        queue=False
+    )
+
+    # When the user clicks the submit button
+    submitBtn.click(
+        user,
+        [user_prompt_message, chatbot],
+        [chatbot],
+        queue=False
+    ).then(
+        bot,
+        [application_mode, user_prompt_message, chatbot, state, qa_chain_state],
+        [chatbot, state]
+    ).then(
+        input_cleanup,
+        [],
+        [user_prompt_message],
+        queue=False
+    )
+
+    # When the user clicks the clear button
+    # clear.click(lambda: None, None, chatbot, queue=False).success(init_history, [state], [state])
+    if __name__ == "__main__":
+    # demo.launch()
+        demo.queue().launch(allowed_paths=["/"], server_name="0.0.0.0", share=True, debug=True)

outdated-files/dd.txt

@@ -0,0 +1 @@
+input_variables=['context', 'question'] messages=[SystemMessagePromptTemplate(prompt=PromptTemplate(input_variables=['context'], template=" \n       In every question you are provided information about the insect. Two types of information are: First,  Vetted Information (which is same in every questinon) and Second, some context from external documents about an insect specie and a question by the user. answer the question according to these two types of informations. \n       ---- \n       Vetted info is as follows:\n       species: Papaipema nebris\ncommon name: stalk borer\norder: Lepidoptera\nfamily: Noctuidae\ngenus: Papaipema\nUpdated role in ecosystem: Pest\nProof: The stalk borer, Papaipema nebris, is correctly classified as a pest because it damages a variety of crops by boring into their stems. This can significantly reduce crop yields and impact agricultural production. Its larvae feed inside the stalks causing the plants to wilt and often die, which characterizes its detrimental role in the ecosystem predominantly affecting agricultural systems.\nipm strategies: IPM strategies for stalk borer include crop rotation, planting trap crops, timely planting to avoid peak egg-laying periods, mechanical control through cultivation, and chemical controls with insecticides if necessary.\nsize of insect: 25-35 mm\ngeographical spread: United States, Canada\nlife cycle specifics: Stalk borers undergo complete metamorphosis, including egg, larva, pupa, and adult stages. Eggs are laid on grasses in fall, hatch in spring, and the larvae bore into stems of a variety of host plants. Pupation takes place within the host plant or in the soil.\npest for plant species: Corn, sorghum, and other grasses\nspecies status: It is native to North America.\ndistribution area: Most widespread in the corn belt of the United States.\nappearance: The adult stalk borer is a medium-sized moth with a wingspan of about 25-35 mm with distinct, dark, and light gray or brown patterns on its wings.\nidentification: Eggs are white and difficult to find, larvae are purplish or pinkish with a distinct white stripe and darker line above this, and adults are gray or brown moths with a characteristic wing pattern.\n\n       ----\n       The context retrieved for documents about this particular question is a as follows:\n       {context}\n       ----\n       Additional Instruction:\n       1. Reference Constraint\n       At the end of each answer provide the source/reference for the given data in following format: \n       \n\n[enter two new lines before writing below] References:\n       Vetted Information Used: Write what was used from the document for coming up with the answer above. Write exact part of lines. If nothing, write 'Nothing'. \n       Documents Used: Write what was used from the document for coming up with the answer above. If nothing, write 'Nothing'. Write exact part of lines and document used. \n       2. Information Constraint: \n       Only answer the question from information provided otherwise say you dont know. You have to answer in 150 words including references. Prioritize information in documents/context over vetted information. And first mention the warnings/things to be careful about. \n       3. Language constraint:\n       The language should be acustomed to a researcher. This question is likely asked by an academic researcher. So you can use all the technical terms freely. And for referencing information, you can use the original content. This is only for the main answer to be provided by you.\n\n       ----\n       ")), HumanMessagePromptTemplate(prompt=PromptTemplate(input_variables=['question'], template='Question:```{question}```'))]

outdated-files/rag-evaluation (outdated).ipynb

@@ -0,0 +1,896 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "42084110-295b-493a-9b3e-5d8d29ff78b3",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "source": [
+    "# LLM RAG Evaluation with MLflow Example Notebook\n",
+    "\n",
+    "In this notebook, we will demonstrate how to evaluate various a RAG system with MLflow."
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    "<a href=\"https://raw.githubusercontent.com/mlflow/mlflow/master/docs/source/llms/llm-evaluate/notebooks/rag-evaluation.ipynb\" class=\"notebook-download-btn\"><i class=\"fas fa-download\"></i>Download this Notebook</a>"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We need to set our OpenAI API key.\n",
+    "\n",
+    "In order to set your private key safely, please be sure to either export your key through a command-line terminal for your current instance, or, for a permanent addition to all user-based sessions, configure your favored environment management configuration file (i.e., .bashrc, .zshrc) to have the following entry:\n",
+    "\n",
+    "`OPENAI_API_KEY=<your openai API key>`\n",
+    "\n",
+    "If using Azure OpenAI, you will instead need to set\n",
+    "\n",
+    "`OPENAI_API_TYPE=\"azure\"`\n",
+    "\n",
+    "`OPENAI_API_VERSION=<YYYY-MM-DD>`\n",
+    "\n",
+    "`OPENAI_API_KEY=<https://<>.<>.<>.com>`\n",
+    "\n",
+    "`OPENAI_API_DEPLOYMENT_NAME=<deployment name>`\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "fb946228-62fb-4d68-9732-75935c9cb401",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "import mlflow\n",
+    "import os\n",
+    "os.environ[\"OPENAI_API_KEY\"] =\"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\"\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "273d1345-95d7-435a-a7b6-a5f3dbb3f073",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "source": [
+    "## Create a RAG system\n",
+    "\n",
+    "Use Langchain and Chroma to create a RAG system that answers questions based on the MLflow documentation."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "2c28d0ad-f469-46ab-a2b4-c5e8db50a729",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.chains import RetrievalQA\n",
+    "from langchain.document_loaders import WebBaseLoader\n",
+    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
+    "from langchain.llms import OpenAI\n",
+    "from langchain.text_splitter import CharacterTextSplitter\n",
+    "from langchain.vectorstores import Chroma\n",
+    "from langchain.chat_models import ChatOpenAI\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "83a7e77e-6717-472a-86dc-02e2c356ddef",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "loader = WebBaseLoader(\"https://mlflow.org/docs/latest/index.html\")\n",
+    "\n",
+    "documents = loader.load()\n",
+    "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
+    "texts = text_splitter.split_documents(documents)\n",
+    "\n",
+    "embeddings = OpenAIEmbeddings()\n",
+    "docsearch = Chroma.from_documents(texts, embeddings)\n",
+    "\n",
+    "qa = RetrievalQA.from_chain_type(\n",
+    "    llm=ChatOpenAI(model_name=\"gpt-3.5-turbo-0125\" , temperature=0),\n",
+    "    chain_type=\"stuff\",\n",
+    "    retriever=docsearch.as_retriever(),\n",
+    "    return_source_documents=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "fd70bcf6-7c44-44d3-9435-567b82611e1c",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "source": [
+    "## Evaluate the RAG system using `mlflow.evaluate()`"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "de1bc359-2e40-459c-bea4-bed35a117988",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "source": [
+    "Create a simple function that runs each input through the RAG chain"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "667ec809-2bb5-4170-9937-6804386b41ec",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def model(input_df):\n",
+    "    answer = []\n",
+    "    for index, row in input_df.iterrows():\n",
+    "        answer.append(qa(row[\"questions\"]))\n",
+    "\n",
+    "    return answer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "d1064306-b7f3-4b3e-825c-4353d808f21d",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "source": [
+    "Create an eval dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "a5481491-e4a9-42ea-8a3f-f527faffd04d",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "eval_df = pd.DataFrame(\n",
+    "    {\n",
+    "        \"questions\": [\n",
+    "            \"What is MLflow?\",\n",
+    "            \"How to run mlflow.evaluate()?\",\n",
+    "            \"How to log_table()?\",\n",
+    "            \"How to load_table()?\",\n",
+    "        ],\n",
+    "    }\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "9c3c8023-8feb-427a-b36d-34cd1853a5dc",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "source": [
+    "Create a faithfulness metric"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "3882b940-9c25-41ce-a301-72d8c0c90aaa",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "EvaluationMetric(name=faithfulness, greater_is_better=True, long_name=faithfulness, version=v1, metric_details=\n",
+      "Task:\n",
+      "You must return the following fields in your response in two lines, one below the other:\n",
+      "score: Your numerical score for the model's faithfulness based on the rubric\n",
+      "justification: Your reasoning about the model's faithfulness score\n",
+      "\n",
+      "You are an impartial judge. You will be given an input that was sent to a machine\n",
+      "learning model, and you will be given an output that the model produced. You\n",
+      "may also be given additional information that was used by the model to generate the output.\n",
+      "\n",
+      "Your task is to determine a numerical score called faithfulness based on the input and output.\n",
+      "A definition of faithfulness and a grading rubric are provided below.\n",
+      "You must use the grading rubric to determine your score. You must also justify your score.\n",
+      "\n",
+      "Examples could be included below for reference. Make sure to use them as references and to\n",
+      "understand them before completing the task.\n",
+      "\n",
+      "Input:\n",
+      "{input}\n",
+      "\n",
+      "Output:\n",
+      "{output}\n",
+      "\n",
+      "{grading_context_columns}\n",
+      "\n",
+      "Metric definition:\n",
+      "Faithfulness is only evaluated with the provided output and provided context, please ignore the provided input entirely when scoring faithfulness. Faithfulness assesses how much of the provided output is factually consistent with the provided context. A higher score indicates that a higher proportion of claims present in the output can be derived from the provided context. Faithfulness does not consider how much extra information from the context is not present in the output.\n",
+      "\n",
+      "Grading rubric:\n",
+      "Faithfulness: Below are the details for different scores:\n",
+      "- Score 1: None of the claims in the output can be inferred from the provided context.\n",
+      "- Score 2: Some of the claims in the output can be inferred from the provided context, but the majority of the output is missing from, inconsistent with, or contradictory to the provided context.\n",
+      "- Score 3: Half or more of the claims in the output can be inferred from the provided context.\n",
+      "- Score 4: Most of the claims in the output can be inferred from the provided context, with very little information that is not directly supported by the provided context.\n",
+      "- Score 5: All of the claims in the output are directly supported by the provided context, demonstrating high faithfulness to the provided context.\n",
+      "\n",
+      "Examples:\n",
+      "\n",
+      "Example Input:\n",
+      "How do I disable MLflow autologging?\n",
+      "\n",
+      "Example Output:\n",
+      "mlflow.autolog(disable=True) will disable autologging for all functions. In Databricks, autologging is enabled by default. \n",
+      "\n",
+      "Additional information used by the model:\n",
+      "key: context\n",
+      "value:\n",
+      "mlflow.autolog(log_input_examples: bool = False, log_model_signatures: bool = True, log_models: bool = True, log_datasets: bool = True, disable: bool = False, exclusive: bool = False, disable_for_unsupported_versions: bool = False, silent: bool = False, extra_tags: Optional[Dict[str, str]] = None) → None[source] Enables (or disables) and configures autologging for all supported integrations. The parameters are passed to any autologging integrations that support them. See the tracking docs for a list of supported autologging integrations. Note that framework-specific configurations set at any point will take precedence over any configurations set by this function.\n",
+      "\n",
+      "Example score: 2\n",
+      "Example justification: The output provides a working solution, using the mlflow.autolog() function that is provided in the context.\n",
+      "        \n",
+      "\n",
+      "Example Input:\n",
+      "How do I disable MLflow autologging?\n",
+      "\n",
+      "Example Output:\n",
+      "mlflow.autolog(disable=True) will disable autologging for all functions.\n",
+      "\n",
+      "Additional information used by the model:\n",
+      "key: context\n",
+      "value:\n",
+      "mlflow.autolog(log_input_examples: bool = False, log_model_signatures: bool = True, log_models: bool = True, log_datasets: bool = True, disable: bool = False, exclusive: bool = False, disable_for_unsupported_versions: bool = False, silent: bool = False, extra_tags: Optional[Dict[str, str]] = None) → None[source] Enables (or disables) and configures autologging for all supported integrations. The parameters are passed to any autologging integrations that support them. See the tracking docs for a list of supported autologging integrations. Note that framework-specific configurations set at any point will take precedence over any configurations set by this function.\n",
+      "\n",
+      "Example score: 5\n",
+      "Example justification: The output provides a solution that is using the mlflow.autolog() function that is provided in the context.\n",
+      "        \n",
+      "\n",
+      "You must return the following fields in your response in two lines, one below the other:\n",
+      "score: Your numerical score for the model's faithfulness based on the rubric\n",
+      "justification: Your reasoning about the model's faithfulness score\n",
+      "\n",
+      "Do not add additional new lines. Do not add any other fields.\n",
+      "    )\n"
+     ]
+    }
+   ],
+   "source": [
+    "from mlflow.metrics.genai import EvaluationExample, faithfulness\n",
+    "\n",
+    "# Create a good and bad example for faithfulness in the context of this problem\n",
+    "faithfulness_examples = [\n",
+    "    EvaluationExample(\n",
+    "        input=\"How do I disable MLflow autologging?\",\n",
+    "        output=\"mlflow.autolog(disable=True) will disable autologging for all functions. In Databricks, autologging is enabled by default. \",\n",
+    "        score=2,\n",
+    "        justification=\"The output provides a working solution, using the mlflow.autolog() function that is provided in the context.\",\n",
+    "        grading_context={\n",
+    "            \"context\": \"mlflow.autolog(log_input_examples: bool = False, log_model_signatures: bool = True, log_models: bool = True, log_datasets: bool = True, disable: bool = False, exclusive: bool = False, disable_for_unsupported_versions: bool = False, silent: bool = False, extra_tags: Optional[Dict[str, str]] = None) → None[source] Enables (or disables) and configures autologging for all supported integrations. The parameters are passed to any autologging integrations that support them. See the tracking docs for a list of supported autologging integrations. Note that framework-specific configurations set at any point will take precedence over any configurations set by this function.\"\n",
+    "        },\n",
+    "    ),\n",
+    "    EvaluationExample(\n",
+    "        input=\"How do I disable MLflow autologging?\",\n",
+    "        output=\"mlflow.autolog(disable=True) will disable autologging for all functions.\",\n",
+    "        score=5,\n",
+    "        justification=\"The output provides a solution that is using the mlflow.autolog() function that is provided in the context.\",\n",
+    "        grading_context={\n",
+    "            \"context\": \"mlflow.autolog(log_input_examples: bool = False, log_model_signatures: bool = True, log_models: bool = True, log_datasets: bool = True, disable: bool = False, exclusive: bool = False, disable_for_unsupported_versions: bool = False, silent: bool = False, extra_tags: Optional[Dict[str, str]] = None) → None[source] Enables (or disables) and configures autologging for all supported integrations. The parameters are passed to any autologging integrations that support them. See the tracking docs for a list of supported autologging integrations. Note that framework-specific configurations set at any point will take precedence over any configurations set by this function.\"\n",
+    "        },\n",
+    "    ),\n",
+    "]\n",
+    "\n",
+    "faithfulness_metric = faithfulness(model=\"openai:/gpt-4\", examples=faithfulness_examples)\n",
+    "print(faithfulness_metric)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Create a relevance metric. You can see the full grading prompt by printing the metric or by accessing the `metric_details` attribute of the metric."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "EvaluationMetric(name=relevance, greater_is_better=True, long_name=relevance, version=v1, metric_details=\n",
+      "Task:\n",
+      "You must return the following fields in your response in two lines, one below the other:\n",
+      "score: Your numerical score for the model's relevance based on the rubric\n",
+      "justification: Your reasoning about the model's relevance score\n",
+      "\n",
+      "You are an impartial judge. You will be given an input that was sent to a machine\n",
+      "learning model, and you will be given an output that the model produced. You\n",
+      "may also be given additional information that was used by the model to generate the output.\n",
+      "\n",
+      "Your task is to determine a numerical score called relevance based on the input and output.\n",
+      "A definition of relevance and a grading rubric are provided below.\n",
+      "You must use the grading rubric to determine your score. You must also justify your score.\n",
+      "\n",
+      "Examples could be included below for reference. Make sure to use them as references and to\n",
+      "understand them before completing the task.\n",
+      "\n",
+      "Input:\n",
+      "{input}\n",
+      "\n",
+      "Output:\n",
+      "{output}\n",
+      "\n",
+      "{grading_context_columns}\n",
+      "\n",
+      "Metric definition:\n",
+      "Relevance encompasses the appropriateness, significance, and applicability of the output with respect to both the input and context. Scores should reflect the extent to which the output directly addresses the question provided in the input, given the provided context.\n",
+      "\n",
+      "Grading rubric:\n",
+      "Relevance: Below are the details for different scores:- Score 1: The output doesn't mention anything about the question or is completely irrelevant to the provided context.\n",
+      "- Score 2: The output provides some relevance to the question and is somehow related to the provided context.\n",
+      "- Score 3: The output mostly answers the question and is largely consistent with the provided context.\n",
+      "- Score 4: The output answers the question and is consistent with the provided context.\n",
+      "- Score 5: The output answers the question comprehensively using the provided context.\n",
+      "\n",
+      "Examples:\n",
+      "\n",
+      "Example Input:\n",
+      "How is MLflow related to Databricks?\n",
+      "\n",
+      "Example Output:\n",
+      "Databricks is a data engineering and analytics platform designed to help organizations process and analyze large amounts of data. Databricks is a company specializing in big data and machine learning solutions.\n",
+      "\n",
+      "Additional information used by the model:\n",
+      "key: context\n",
+      "value:\n",
+      "MLflow is an open-source platform for managing the end-to-end machine learning (ML) lifecycle. It was developed by Databricks, a company that specializes in big data and machine learning solutions. MLflow is designed to address the challenges that data scientists and machine learning engineers face when developing, training, and deploying machine learning models.\n",
+      "\n",
+      "Example score: 2\n",
+      "Example justification: The output provides relevant information about Databricks, mentioning it as a company specializing in big data and machine learning solutions. However, it doesn't directly address how MLflow is related to Databricks, which is the specific question asked in the input. Therefore, the output is only somewhat related to the provided context.\n",
+      "        \n",
+      "\n",
+      "Example Input:\n",
+      "How is MLflow related to Databricks?\n",
+      "\n",
+      "Example Output:\n",
+      "MLflow is a product created by Databricks to enhance the efficiency of machine learning processes.\n",
+      "\n",
+      "Additional information used by the model:\n",
+      "key: context\n",
+      "value:\n",
+      "MLflow is an open-source platform for managing the end-to-end machine learning (ML) lifecycle. It was developed by Databricks, a company that specializes in big data and machine learning solutions. MLflow is designed to address the challenges that data scientists and machine learning engineers face when developing, training, and deploying machine learning models.\n",
+      "\n",
+      "Example score: 4\n",
+      "Example justification: The output provides a relevant and accurate statement about the relationship between MLflow and Databricks. While it doesn't provide extensive detail, it still offers a substantial and meaningful response. To achieve a score of 5, the response could be further improved by providing additional context or details about how MLflow specifically functions within the Databricks ecosystem.\n",
+      "        \n",
+      "\n",
+      "You must return the following fields in your response in two lines, one below the other:\n",
+      "score: Your numerical score for the model's relevance based on the rubric\n",
+      "justification: Your reasoning about the model's relevance score\n",
+      "\n",
+      "Do not add additional new lines. Do not add any other fields.\n",
+      "    )\n"
+     ]
+    }
+   ],
+   "source": [
+    "from mlflow.metrics.genai import EvaluationExample, relevance\n",
+    "\n",
+    "relevance_metric = relevance(model=\"openai:/gpt-4\")\n",
+    "print(relevance_metric)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eval_df_final=eval_df.copy(deep=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "aa=model(eval_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'query': 'What is MLflow?',\n",
+       "  'result': 'MLflow is an open-source platform designed to help machine learning practitioners and teams manage the complexities of the machine learning process. It focuses on the full lifecycle of machine learning projects, making sure that each phase is manageable, traceable, and reproducible.',\n",
+       "  'source_documents': [Document(page_content='MLflow: A Tool for Managing the Machine Learning Lifecycle \\nMLflow is an open-source platform, purpose-built to assist machine learning practitioners and teams in\\nhandling the complexities of the machine learning process. MLflow focuses on the full lifecycle for\\nmachine learning projects, ensuring that each phase is manageable, traceable, and reproducible.\\nIn each of the sections below, you will find overviews, guides, and step-by-step tutorials to walk you through\\nthe features of MLflow and how they can be leveraged to solve real-world MLOps problems.\\n\\nGetting Started with MLflow \\nIf this is your first time exploring MLflow, the tutorials and guides here are a great place to start. The emphasis in each of these is\\ngetting you up to speed as quickly as possible with the basic functionality, terms, APIs, and general best practices of using MLflow in order to\\nenhance your learning in area-specific guides and tutorials.\\n\\nGetting Started Guides and Quickstarts', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='MLflow: A Tool for Managing the Machine Learning Lifecycle \\nMLflow is an open-source platform, purpose-built to assist machine learning practitioners and teams in\\nhandling the complexities of the machine learning process. MLflow focuses on the full lifecycle for\\nmachine learning projects, ensuring that each phase is manageable, traceable, and reproducible.\\nIn each of the sections below, you will find overviews, guides, and step-by-step tutorials to walk you through\\nthe features of MLflow and how they can be leveraged to solve real-world MLOps problems.\\n\\nGetting Started with MLflow \\nIf this is your first time exploring MLflow, the tutorials and guides here are a great place to start. The emphasis in each of these is\\ngetting you up to speed as quickly as possible with the basic functionality, terms, APIs, and general best practices of using MLflow in order to\\nenhance your learning in area-specific guides and tutorials.\\n\\nGetting Started Guides and Quickstarts', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='MLflow: A Tool for Managing the Machine Learning Lifecycle \\nMLflow is an open-source platform, purpose-built to assist machine learning practitioners and teams in\\nhandling the complexities of the machine learning process. MLflow focuses on the full lifecycle for\\nmachine learning projects, ensuring that each phase is manageable, traceable, and reproducible.\\nIn each of the sections below, you will find overviews, guides, and step-by-step tutorials to walk you through\\nthe features of MLflow and how they can be leveraged to solve real-world MLOps problems.\\n\\nGetting Started with MLflow \\nIf this is your first time exploring MLflow, the tutorials and guides here are a great place to start. The emphasis in each of these is\\ngetting you up to speed as quickly as possible with the basic functionality, terms, APIs, and general best practices of using MLflow in order to\\nenhance your learning in area-specific guides and tutorials.\\n\\nGetting Started Guides and Quickstarts', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation\\n\\n2.12.1\\n\\n\\n MLflow\\n\\nWhat is MLflow?\\nGetting Started with MLflow\\nNew Features\\nLLMs\\nModel Evaluation\\nDeep Learning\\nTraditional ML\\nDeployment\\nMLflow Tracking\\nSystem Metrics\\nMLflow Projects\\nMLflow Models\\nMLflow Model Registry\\nMLflow Recipes\\nMLflow Plugins\\nMLflow Authentication\\nCommand-Line Interface\\nSearch Runs\\nSearch Experiments\\nPython API\\nR API\\nJava API\\nREST API\\nOfficial MLflow Docker Image\\nCommunity Model Flavors\\nTutorials and Examples\\n\\n\\nContribute\\n\\n\\nDocumentation \\nMLflow: A Tool for Managing the Machine Learning Lifecycle', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'})]},\n",
+       " {'query': 'How to run mlflow.evaluate()?',\n",
+       "  'result': 'To run `mlflow.evaluate()`, you need to follow these steps:\\n\\n1. Ensure you have MLflow installed in your environment.\\n2. Import the necessary libraries, including `mlflow`.\\n3. Load your model and data.\\n4. Use the `mlflow.evaluate()` function, passing in your model, data, and any other necessary parameters.\\n5. Review the evaluation results and metrics provided by the function.\\n\\nIf you need more specific details or code examples, please refer to the MLflow documentation or tutorials for a step-by-step guide on running `mlflow.evaluate()`.',\n",
+       "  'source_documents': [Document(page_content='Model Evaluation \\nDive into MLflow’s robust framework for evaluating the performance of your ML models.\\nWith support for traditional ML evaluation (classification and regression tasks), as well as support for evaluating large language models (LLMs),\\nthis suite of APIs offers a simple but powerful automated approach to evaluating the quality of the model development work that you’re doing.\\nIn particular, for LLM evaluation, the mlflow.evaluate() API allows you to validate not only models, but providers and prompts.\\nBy leveraging your own datasets and using the provided default evaluation criteria for tasks such as text summarization and question answering, you can\\nget reliable metrics that allow you to focus on improving the quality of your solution, rather than spending time writing scoring code.\\nVisual insights are also available through the MLflow UI, showcasing logged outputs, auto-generated plots, and model comparison artifacts.', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='Model Evaluation \\nDive into MLflow’s robust framework for evaluating the performance of your ML models.\\nWith support for traditional ML evaluation (classification and regression tasks), as well as support for evaluating large language models (LLMs),\\nthis suite of APIs offers a simple but powerful automated approach to evaluating the quality of the model development work that you’re doing.\\nIn particular, for LLM evaluation, the mlflow.evaluate() API allows you to validate not only models, but providers and prompts.\\nBy leveraging your own datasets and using the provided default evaluation criteria for tasks such as text summarization and question answering, you can\\nget reliable metrics that allow you to focus on improving the quality of your solution, rather than spending time writing scoring code.\\nVisual insights are also available through the MLflow UI, showcasing logged outputs, auto-generated plots, and model comparison artifacts.', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='Model Evaluation \\nDive into MLflow’s robust framework for evaluating the performance of your ML models.\\nWith support for traditional ML evaluation (classification and regression tasks), as well as support for evaluating large language models (LLMs),\\nthis suite of APIs offers a simple but powerful automated approach to evaluating the quality of the model development work that you’re doing.\\nIn particular, for LLM evaluation, the mlflow.evaluate() API allows you to validate not only models, but providers and prompts.\\nBy leveraging your own datasets and using the provided default evaluation criteria for tasks such as text summarization and question answering, you can\\nget reliable metrics that allow you to focus on improving the quality of your solution, rather than spending time writing scoring code.\\nVisual insights are also available through the MLflow UI, showcasing logged outputs, auto-generated plots, and model comparison artifacts.', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='Learn how to evaluate LLMs and LLM-powered solutions with MLflow Evaluate.\\n                \\n\\n                    Using Custom PyFunc with LLMs\\n                \\n\\n                    Explore the nuances of packaging and deploying advanced LLMs in MLflow using custom PyFuncs. This guide delves deep\\n                    into managing intricate model behaviors, ensuring seamless and efficient LLM deployments.\\n                \\n\\n                    Evaluation for RAG\\n                \\n\\n                    Learn how to evaluate Retrieval Augmented Generation applications by leveraging LLMs to generate a evaluation dataset and evaluate it using the built-in metrics in the MLflow Evaluate API.\\n                \\n\\n                    LLM Tracking with MLflow', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'})]},\n",
+       " {'query': 'How to log_table()?',\n",
+       "  'result': \"I don't have information on a specific function called `log_table()` in the context provided. It's possible that it might be a custom function or a feature not explicitly mentioned in the provided context. If you can provide more details or context, I may be able to assist you further.\",\n",
+       "  'source_documents': [Document(page_content='MLflow Tracking Quickstart\\n                \\n\\n                A great place to start to learn the fundamentals of MLflow Tracking! Learn in 5 minutes how to log, register, and load a model for inference.\\n                \\n\\n                    Intro to MLflow Tutorial\\n                \\n\\n                    Learn how to get started with the basics of MLflow in a step-by-step instructional tutorial that shows the critical\\n                    path to logging your first model\\n                \\n\\n                    Autologging Quickstart\\n                \\n\\n                    Short on time? This is a no-frills quickstart that shows how to leverage autologging during training and how to\\n                    load a model for inference\\n                \\n\\n                    Deployment Quickstart', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='MLflow Tracking Quickstart\\n                \\n\\n                A great place to start to learn the fundamentals of MLflow Tracking! Learn in 5 minutes how to log, register, and load a model for inference.\\n                \\n\\n                    Intro to MLflow Tutorial\\n                \\n\\n                    Learn how to get started with the basics of MLflow in a step-by-step instructional tutorial that shows the critical\\n                    path to logging your first model\\n                \\n\\n                    Autologging Quickstart\\n                \\n\\n                    Short on time? This is a no-frills quickstart that shows how to leverage autologging during training and how to\\n                    load a model for inference\\n                \\n\\n                    Deployment Quickstart', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='MLflow Tracking Quickstart\\n                \\n\\n                A great place to start to learn the fundamentals of MLflow Tracking! Learn in 5 minutes how to log, register, and load a model for inference.\\n                \\n\\n                    Intro to MLflow Tutorial\\n                \\n\\n                    Learn how to get started with the basics of MLflow in a step-by-step instructional tutorial that shows the critical\\n                    path to logging your first model\\n                \\n\\n                    Autologging Quickstart\\n                \\n\\n                    Short on time? This is a no-frills quickstart that shows how to leverage autologging during training and how to\\n                    load a model for inference\\n                \\n\\n                    Deployment Quickstart', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content=\"Dive into the intricacies of MLflow's LLM Tracking system. From capturing prompts to monitoring generated outputs,\\n                    discover how MLflow provides a holistic solution for managing LLM interactions.\", metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'})]},\n",
+       " {'query': 'How to load_table()?',\n",
+       "  'result': \"I'm not sure what `load_table()` refers to in this context. If you can provide more information or context, I might be able to help you better.\",\n",
+       "  'source_documents': [Document(page_content='MLflow Tracking Quickstart\\n                \\n\\n                A great place to start to learn the fundamentals of MLflow Tracking! Learn in 5 minutes how to log, register, and load a model for inference.\\n                \\n\\n                    Intro to MLflow Tutorial\\n                \\n\\n                    Learn how to get started with the basics of MLflow in a step-by-step instructional tutorial that shows the critical\\n                    path to logging your first model\\n                \\n\\n                    Autologging Quickstart\\n                \\n\\n                    Short on time? This is a no-frills quickstart that shows how to leverage autologging during training and how to\\n                    load a model for inference\\n                \\n\\n                    Deployment Quickstart', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='MLflow Tracking Quickstart\\n                \\n\\n                A great place to start to learn the fundamentals of MLflow Tracking! Learn in 5 minutes how to log, register, and load a model for inference.\\n                \\n\\n                    Intro to MLflow Tutorial\\n                \\n\\n                    Learn how to get started with the basics of MLflow in a step-by-step instructional tutorial that shows the critical\\n                    path to logging your first model\\n                \\n\\n                    Autologging Quickstart\\n                \\n\\n                    Short on time? This is a no-frills quickstart that shows how to leverage autologging during training and how to\\n                    load a model for inference\\n                \\n\\n                    Deployment Quickstart', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='MLflow Tracking Quickstart\\n                \\n\\n                A great place to start to learn the fundamentals of MLflow Tracking! Learn in 5 minutes how to log, register, and load a model for inference.\\n                \\n\\n                    Intro to MLflow Tutorial\\n                \\n\\n                    Learn how to get started with the basics of MLflow in a step-by-step instructional tutorial that shows the critical\\n                    path to logging your first model\\n                \\n\\n                    Autologging Quickstart\\n                \\n\\n                    Short on time? This is a no-frills quickstart that shows how to leverage autologging during training and how to\\n                    load a model for inference\\n                \\n\\n                    Deployment Quickstart', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='This guide showcases the seamless end-to-end process of training a linear regression model, packaging it in a reproducible format,\\n                    and deploying to a Kubernetes cluster using MLflow. Explore how MLflow simplifies model deployment to production environments.\\n                \\n\\n\\nNext \\n\\n\\n          © MLflow Project, a Series of LF Projects, LLC. All rights reserved.', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'})]}]"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "aa"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'query': 'What is MLflow?',\n",
+       "  'result': 'MLflow is an open-source platform designed to help machine learning practitioners and teams manage the complexities of the machine learning process. It focuses on the full lifecycle of machine learning projects, ensuring that each phase is manageable, traceable, and reproducible. It offers features such as tracking, projects, models, model registry, and more to assist in solving real-world MLOps problems.',\n",
+       "  'source_documents': [Document(page_content='MLflow: A Tool for Managing the Machine Learning Lifecycle \\nMLflow is an open-source platform, purpose-built to assist machine learning practitioners and teams in\\nhandling the complexities of the machine learning process. MLflow focuses on the full lifecycle for\\nmachine learning projects, ensuring that each phase is manageable, traceable, and reproducible.\\nIn each of the sections below, you will find overviews, guides, and step-by-step tutorials to walk you through\\nthe features of MLflow and how they can be leveraged to solve real-world MLOps problems.\\n\\nGetting Started with MLflow \\nIf this is your first time exploring MLflow, the tutorials and guides here are a great place to start. The emphasis in each of these is\\ngetting you up to speed as quickly as possible with the basic functionality, terms, APIs, and general best practices of using MLflow in order to\\nenhance your learning in area-specific guides and tutorials.\\n\\nGetting Started Guides and Quickstarts', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='MLflow: A Tool for Managing the Machine Learning Lifecycle \\nMLflow is an open-source platform, purpose-built to assist machine learning practitioners and teams in\\nhandling the complexities of the machine learning process. MLflow focuses on the full lifecycle for\\nmachine learning projects, ensuring that each phase is manageable, traceable, and reproducible.\\nIn each of the sections below, you will find overviews, guides, and step-by-step tutorials to walk you through\\nthe features of MLflow and how they can be leveraged to solve real-world MLOps problems.\\n\\nGetting Started with MLflow \\nIf this is your first time exploring MLflow, the tutorials and guides here are a great place to start. The emphasis in each of these is\\ngetting you up to speed as quickly as possible with the basic functionality, terms, APIs, and general best practices of using MLflow in order to\\nenhance your learning in area-specific guides and tutorials.\\n\\nGetting Started Guides and Quickstarts', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='MLflow: A Tool for Managing the Machine Learning Lifecycle \\nMLflow is an open-source platform, purpose-built to assist machine learning practitioners and teams in\\nhandling the complexities of the machine learning process. MLflow focuses on the full lifecycle for\\nmachine learning projects, ensuring that each phase is manageable, traceable, and reproducible.\\nIn each of the sections below, you will find overviews, guides, and step-by-step tutorials to walk you through\\nthe features of MLflow and how they can be leveraged to solve real-world MLOps problems.\\n\\nGetting Started with MLflow \\nIf this is your first time exploring MLflow, the tutorials and guides here are a great place to start. The emphasis in each of these is\\ngetting you up to speed as quickly as possible with the basic functionality, terms, APIs, and general best practices of using MLflow in order to\\nenhance your learning in area-specific guides and tutorials.\\n\\nGetting Started Guides and Quickstarts', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation\\n\\n2.12.1\\n\\n\\n MLflow\\n\\nWhat is MLflow?\\nGetting Started with MLflow\\nNew Features\\nLLMs\\nModel Evaluation\\nDeep Learning\\nTraditional ML\\nDeployment\\nMLflow Tracking\\nSystem Metrics\\nMLflow Projects\\nMLflow Models\\nMLflow Model Registry\\nMLflow Recipes\\nMLflow Plugins\\nMLflow Authentication\\nCommand-Line Interface\\nSearch Runs\\nSearch Experiments\\nPython API\\nR API\\nJava API\\nREST API\\nOfficial MLflow Docker Image\\nCommunity Model Flavors\\nTutorials and Examples\\n\\n\\nContribute\\n\\n\\nDocumentation \\nMLflow: A Tool for Managing the Machine Learning Lifecycle', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'})]},\n",
+       " {'query': 'How to run mlflow.evaluate()?',\n",
+       "  'result': 'To run `mlflow.evaluate()`, you need to follow these steps:\\n\\n1. Import the necessary libraries:\\n```python\\nimport mlflow\\n```\\n\\n2. Load your model and data:\\n```python\\nmodel = load_model()  # Load your model\\ndata = load_data()  # Load your evaluation data\\n```\\n\\n3. Use `mlflow.evaluate()` to evaluate your model:\\n```python\\nmlflow.evaluate(model, data)\\n```\\n\\n4. Review the evaluation metrics and visual insights in the MLflow UI.\\n\\nIf you need more specific details or have a particular use case in mind, please provide additional context for a more tailored explanation.',\n",
+       "  'source_documents': [Document(page_content='Model Evaluation \\nDive into MLflow’s robust framework for evaluating the performance of your ML models.\\nWith support for traditional ML evaluation (classification and regression tasks), as well as support for evaluating large language models (LLMs),\\nthis suite of APIs offers a simple but powerful automated approach to evaluating the quality of the model development work that you’re doing.\\nIn particular, for LLM evaluation, the mlflow.evaluate() API allows you to validate not only models, but providers and prompts.\\nBy leveraging your own datasets and using the provided default evaluation criteria for tasks such as text summarization and question answering, you can\\nget reliable metrics that allow you to focus on improving the quality of your solution, rather than spending time writing scoring code.\\nVisual insights are also available through the MLflow UI, showcasing logged outputs, auto-generated plots, and model comparison artifacts.', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='Model Evaluation \\nDive into MLflow’s robust framework for evaluating the performance of your ML models.\\nWith support for traditional ML evaluation (classification and regression tasks), as well as support for evaluating large language models (LLMs),\\nthis suite of APIs offers a simple but powerful automated approach to evaluating the quality of the model development work that you’re doing.\\nIn particular, for LLM evaluation, the mlflow.evaluate() API allows you to validate not only models, but providers and prompts.\\nBy leveraging your own datasets and using the provided default evaluation criteria for tasks such as text summarization and question answering, you can\\nget reliable metrics that allow you to focus on improving the quality of your solution, rather than spending time writing scoring code.\\nVisual insights are also available through the MLflow UI, showcasing logged outputs, auto-generated plots, and model comparison artifacts.', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='Model Evaluation \\nDive into MLflow’s robust framework for evaluating the performance of your ML models.\\nWith support for traditional ML evaluation (classification and regression tasks), as well as support for evaluating large language models (LLMs),\\nthis suite of APIs offers a simple but powerful automated approach to evaluating the quality of the model development work that you’re doing.\\nIn particular, for LLM evaluation, the mlflow.evaluate() API allows you to validate not only models, but providers and prompts.\\nBy leveraging your own datasets and using the provided default evaluation criteria for tasks such as text summarization and question answering, you can\\nget reliable metrics that allow you to focus on improving the quality of your solution, rather than spending time writing scoring code.\\nVisual insights are also available through the MLflow UI, showcasing logged outputs, auto-generated plots, and model comparison artifacts.', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='Learn how to evaluate LLMs and LLM-powered solutions with MLflow Evaluate.\\n                \\n\\n                    Using Custom PyFunc with LLMs\\n                \\n\\n                    Explore the nuances of packaging and deploying advanced LLMs in MLflow using custom PyFuncs. This guide delves deep\\n                    into managing intricate model behaviors, ensuring seamless and efficient LLM deployments.\\n                \\n\\n                    Evaluation for RAG\\n                \\n\\n                    Learn how to evaluate Retrieval Augmented Generation applications by leveraging LLMs to generate a evaluation dataset and evaluate it using the built-in metrics in the MLflow Evaluate API.\\n                \\n\\n                    LLM Tracking with MLflow', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'})]},\n",
+       " {'query': 'How to log_table()?',\n",
+       "  'result': \"I don't have information on a specific function called `log_table()` in the context provided. It's possible that it might be a custom function or a feature not covered in the provided context. If you can provide more details or context, I may be able to assist you further.\",\n",
+       "  'source_documents': [Document(page_content='MLflow Tracking Quickstart\\n                \\n\\n                A great place to start to learn the fundamentals of MLflow Tracking! Learn in 5 minutes how to log, register, and load a model for inference.\\n                \\n\\n                    Intro to MLflow Tutorial\\n                \\n\\n                    Learn how to get started with the basics of MLflow in a step-by-step instructional tutorial that shows the critical\\n                    path to logging your first model\\n                \\n\\n                    Autologging Quickstart\\n                \\n\\n                    Short on time? This is a no-frills quickstart that shows how to leverage autologging during training and how to\\n                    load a model for inference\\n                \\n\\n                    Deployment Quickstart', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='MLflow Tracking Quickstart\\n                \\n\\n                A great place to start to learn the fundamentals of MLflow Tracking! Learn in 5 minutes how to log, register, and load a model for inference.\\n                \\n\\n                    Intro to MLflow Tutorial\\n                \\n\\n                    Learn how to get started with the basics of MLflow in a step-by-step instructional tutorial that shows the critical\\n                    path to logging your first model\\n                \\n\\n                    Autologging Quickstart\\n                \\n\\n                    Short on time? This is a no-frills quickstart that shows how to leverage autologging during training and how to\\n                    load a model for inference\\n                \\n\\n                    Deployment Quickstart', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='MLflow Tracking Quickstart\\n                \\n\\n                A great place to start to learn the fundamentals of MLflow Tracking! Learn in 5 minutes how to log, register, and load a model for inference.\\n                \\n\\n                    Intro to MLflow Tutorial\\n                \\n\\n                    Learn how to get started with the basics of MLflow in a step-by-step instructional tutorial that shows the critical\\n                    path to logging your first model\\n                \\n\\n                    Autologging Quickstart\\n                \\n\\n                    Short on time? This is a no-frills quickstart that shows how to leverage autologging during training and how to\\n                    load a model for inference\\n                \\n\\n                    Deployment Quickstart', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content=\"Dive into the intricacies of MLflow's LLM Tracking system. From capturing prompts to monitoring generated outputs,\\n                    discover how MLflow provides a holistic solution for managing LLM interactions.\", metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'})]},\n",
+       " {'query': 'How to load_table()?',\n",
+       "  'result': \"I don't have information on a function called `load_table()` in the context provided. It seems to be outside the scope of the MLflow Tracking, Autologging, and Deployment Quickstart guides. If you can provide more context or clarify where `load_table()` is from, I may be able to assist you further.\",\n",
+       "  'source_documents': [Document(page_content='MLflow Tracking Quickstart\\n                \\n\\n                A great place to start to learn the fundamentals of MLflow Tracking! Learn in 5 minutes how to log, register, and load a model for inference.\\n                \\n\\n                    Intro to MLflow Tutorial\\n                \\n\\n                    Learn how to get started with the basics of MLflow in a step-by-step instructional tutorial that shows the critical\\n                    path to logging your first model\\n                \\n\\n                    Autologging Quickstart\\n                \\n\\n                    Short on time? This is a no-frills quickstart that shows how to leverage autologging during training and how to\\n                    load a model for inference\\n                \\n\\n                    Deployment Quickstart', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='MLflow Tracking Quickstart\\n                \\n\\n                A great place to start to learn the fundamentals of MLflow Tracking! Learn in 5 minutes how to log, register, and load a model for inference.\\n                \\n\\n                    Intro to MLflow Tutorial\\n                \\n\\n                    Learn how to get started with the basics of MLflow in a step-by-step instructional tutorial that shows the critical\\n                    path to logging your first model\\n                \\n\\n                    Autologging Quickstart\\n                \\n\\n                    Short on time? This is a no-frills quickstart that shows how to leverage autologging during training and how to\\n                    load a model for inference\\n                \\n\\n                    Deployment Quickstart', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='MLflow Tracking Quickstart\\n                \\n\\n                A great place to start to learn the fundamentals of MLflow Tracking! Learn in 5 minutes how to log, register, and load a model for inference.\\n                \\n\\n                    Intro to MLflow Tutorial\\n                \\n\\n                    Learn how to get started with the basics of MLflow in a step-by-step instructional tutorial that shows the critical\\n                    path to logging your first model\\n                \\n\\n                    Autologging Quickstart\\n                \\n\\n                    Short on time? This is a no-frills quickstart that shows how to leverage autologging during training and how to\\n                    load a model for inference\\n                \\n\\n                    Deployment Quickstart', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'}),\n",
+       "   Document(page_content='This guide showcases the seamless end-to-end process of training a linear regression model, packaging it in a reproducible format,\\n                    and deploying to a Kubernetes cluster using MLflow. Explore how MLflow simplifies model deployment to production environments.\\n                \\n\\n\\nNext \\n\\n\\n          © MLflow Project, a Series of LF Projects, LLC. All rights reserved.', metadata={'language': 'en', 'source': 'https://mlflow.org/docs/latest/index.html', 'title': 'MLflow: A Tool for Managing the Machine Learning Lifecycle — MLflow 2.12.1 documentation'})]}]"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "eval_df_final[\"results\"]=model(eval_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "ea40ce52-6ac7-4c20-9669-d24f80a6cebe",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/u/marshad/.conda/envs/agllm-env1/lib/python3.9/site-packages/mlflow/data/digest_utils.py:26: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
+      "  string_columns = trimmed_df.columns[(df.applymap(type) == str).all(0)]\n",
+      "/u/marshad/.conda/envs/agllm-env1/lib/python3.9/site-packages/mlflow/models/evaluation/base.py:414: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
+      "  data = data.applymap(_hash_array_like_element_as_bytes)\n",
+      "2024/04/21 17:23:37 INFO mlflow.models.evaluation.base: Evaluating the model with the default evaluator.\n",
+      "2024/04/21 17:23:37 INFO mlflow.models.evaluation.default_evaluator: Computing model predictions.\n",
+      "2024/04/21 17:23:44 INFO mlflow.models.evaluation.default_evaluator: Testing metrics on first row...\n",
+      "2024/04/21 17:23:44 WARNING mlflow.metrics.metric_definitions: Failed to load 'toxicity' metric (error: ModuleNotFoundError(\"No module named 'evaluate'\")), skipping metric logging.\n",
+      "2024/04/21 17:23:44 WARNING mlflow.metrics.metric_definitions: Failed to load flesch kincaid metric, skipping metric logging.\n",
+      "2024/04/21 17:23:44 WARNING mlflow.metrics.metric_definitions: Failed to load automated readability index metric, skipping metric logging.\n",
+      "100%|██████████| 1/1 [00:03<00:00,  3.53s/it]\n",
+      "100%|██████████| 1/1 [00:03<00:00,  3.72s/it]\n",
+      "2024/04/21 17:23:51 INFO mlflow.models.evaluation.default_evaluator: Evaluating builtin metrics: token_count\n",
+      "2024/04/21 17:23:51 INFO mlflow.models.evaluation.default_evaluator: Evaluating builtin metrics: toxicity\n",
+      "2024/04/21 17:23:51 WARNING mlflow.metrics.metric_definitions: Failed to load 'toxicity' metric (error: ModuleNotFoundError(\"No module named 'evaluate'\")), skipping metric logging.\n",
+      "2024/04/21 17:23:51 INFO mlflow.models.evaluation.default_evaluator: Evaluating builtin metrics: flesch_kincaid_grade_level\n",
+      "2024/04/21 17:23:51 WARNING mlflow.metrics.metric_definitions: Failed to load flesch kincaid metric, skipping metric logging.\n",
+      "2024/04/21 17:23:51 INFO mlflow.models.evaluation.default_evaluator: Evaluating builtin metrics: ari_grade_level\n",
+      "2024/04/21 17:23:51 WARNING mlflow.metrics.metric_definitions: Failed to load automated readability index metric, skipping metric logging.\n",
+      "2024/04/21 17:23:51 INFO mlflow.models.evaluation.default_evaluator: Evaluating builtin metrics: exact_match\n",
+      "2024/04/21 17:23:51 INFO mlflow.models.evaluation.default_evaluator: Evaluating metrics: faithfulness\n",
+      "100%|██████████| 4/4 [00:03<00:00,  1.02it/s]\n",
+      "2024/04/21 17:23:55 INFO mlflow.models.evaluation.default_evaluator: Evaluating metrics: relevance\n",
+      "100%|██████████| 4/4 [00:04<00:00,  1.20s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'latency/mean': 1.7213420271873474, 'latency/variance': 0.04483020464773446, 'latency/p90': 1.942362928390503, 'faithfulness/v1/mean': 4.75, 'faithfulness/v1/variance': 0.1875, 'faithfulness/v1/p90': 5.0, 'relevance/v1/mean': 3.75, 'relevance/v1/variance': 1.6875, 'relevance/v1/p90': 5.0}\n"
+     ]
+    }
+   ],
+   "source": [
+    "results = mlflow.evaluate(\n",
+    "    model,\n",
+    "    eval_df,\n",
+    "    model_type=\"question-answering\",\n",
+    "    evaluators=\"default\",\n",
+    "    predictions=\"result\",\n",
+    "    extra_metrics=[faithfulness_metric, relevance_metric, mlflow.metrics.latency()],\n",
+    "    evaluator_config={\n",
+    "        \"col_mapping\": {\n",
+    "            \"inputs\": \"questions\",\n",
+    "            \"context\": \"source_documents\",\n",
+    "        }\n",
+    "    },\n",
+    ")\n",
+    "print(results.metrics)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "989a0861-5153-44e6-a19d-efcae7fe6cb5",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "747f65b309b94257b396eebffe814fa6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading artifacts:   0%|          | 0/1 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>questions</th>\n",
+       "      <th>outputs</th>\n",
+       "      <th>source_documents</th>\n",
+       "      <th>latency</th>\n",
+       "      <th>token_count</th>\n",
+       "      <th>toxicity/v1/score</th>\n",
+       "      <th>flesch_kincaid_grade_level/v1/score</th>\n",
+       "      <th>ari_grade_level/v1/score</th>\n",
+       "      <th>faithfulness/v1/score</th>\n",
+       "      <th>faithfulness/v1/justification</th>\n",
+       "      <th>relevance/v1/score</th>\n",
+       "      <th>relevance/v1/justification</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>What is MLflow?</td>\n",
+       "      <td>MLflow is an open-source platform, purpose-bu...</td>\n",
+       "      <td>[{'lc_attributes': {}, 'lc_namespace': ['langc...</td>\n",
+       "      <td>1.989822</td>\n",
+       "      <td>53</td>\n",
+       "      <td>0.000137</td>\n",
+       "      <td>12.5</td>\n",
+       "      <td>18.4</td>\n",
+       "      <td>5</td>\n",
+       "      <td>The output provided by the model is a direct e...</td>\n",
+       "      <td>5</td>\n",
+       "      <td>The output provides a comprehensive answer to ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>How to run mlflow.evaluate()?</td>\n",
+       "      <td>The mlflow.evaluate() API allows you to valid...</td>\n",
+       "      <td>[{'lc_attributes': {}, 'lc_namespace': ['langc...</td>\n",
+       "      <td>1.945368</td>\n",
+       "      <td>55</td>\n",
+       "      <td>0.000200</td>\n",
+       "      <td>9.1</td>\n",
+       "      <td>12.6</td>\n",
+       "      <td>5</td>\n",
+       "      <td>The output provided by the model is completely...</td>\n",
+       "      <td>4</td>\n",
+       "      <td>The output provides a relevant and accurate ex...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>How to log_table()?</td>\n",
+       "      <td>You can log a table with MLflow using the log...</td>\n",
+       "      <td>[{'lc_attributes': {}, 'lc_namespace': ['langc...</td>\n",
+       "      <td>1.521511</td>\n",
+       "      <td>32</td>\n",
+       "      <td>0.000289</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>6.8</td>\n",
+       "      <td>1</td>\n",
+       "      <td>The output claims that you can log a table wit...</td>\n",
+       "      <td>5</td>\n",
+       "      <td>The output provides a comprehensive answer to ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>How to load_table()?</td>\n",
+       "      <td>You can't load_table() with MLflow. MLflow is...</td>\n",
+       "      <td>[{'lc_attributes': {}, 'lc_namespace': ['langc...</td>\n",
+       "      <td>1.105279</td>\n",
+       "      <td>27</td>\n",
+       "      <td>0.000279</td>\n",
+       "      <td>5.8</td>\n",
+       "      <td>8.8</td>\n",
+       "      <td>5</td>\n",
+       "      <td>The output claim that \"You can't load_table() ...</td>\n",
+       "      <td>4</td>\n",
+       "      <td>The output provides a relevant and accurate re...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                       questions  \\\n",
+       "0                What is MLflow?   \n",
+       "1  How to run mlflow.evaluate()?   \n",
+       "2            How to log_table()?   \n",
+       "3           How to load_table()?   \n",
+       "\n",
+       "                                             outputs  \\\n",
+       "0   MLflow is an open-source platform, purpose-bu...   \n",
+       "1   The mlflow.evaluate() API allows you to valid...   \n",
+       "2   You can log a table with MLflow using the log...   \n",
+       "3   You can't load_table() with MLflow. MLflow is...   \n",
+       "\n",
+       "                                    source_documents   latency  token_count  \\\n",
+       "0  [{'lc_attributes': {}, 'lc_namespace': ['langc...  1.989822           53   \n",
+       "1  [{'lc_attributes': {}, 'lc_namespace': ['langc...  1.945368           55   \n",
+       "2  [{'lc_attributes': {}, 'lc_namespace': ['langc...  1.521511           32   \n",
+       "3  [{'lc_attributes': {}, 'lc_namespace': ['langc...  1.105279           27   \n",
+       "\n",
+       "   toxicity/v1/score  flesch_kincaid_grade_level/v1/score  \\\n",
+       "0           0.000137                                 12.5   \n",
+       "1           0.000200                                  9.1   \n",
+       "2           0.000289                                  5.0   \n",
+       "3           0.000279                                  5.8   \n",
+       "\n",
+       "   ari_grade_level/v1/score  faithfulness/v1/score  \\\n",
+       "0                      18.4                      5   \n",
+       "1                      12.6                      5   \n",
+       "2                       6.8                      1   \n",
+       "3                       8.8                      5   \n",
+       "\n",
+       "                       faithfulness/v1/justification  relevance/v1/score  \\\n",
+       "0  The output provided by the model is a direct e...                   5   \n",
+       "1  The output provided by the model is completely...                   4   \n",
+       "2  The output claims that you can log a table wit...                   5   \n",
+       "3  The output claim that \"You can't load_table() ...                   4   \n",
+       "\n",
+       "                          relevance/v1/justification  \n",
+       "0  The output provides a comprehensive answer to ...  \n",
+       "1  The output provides a relevant and accurate ex...  \n",
+       "2  The output provides a comprehensive answer to ...  \n",
+       "3  The output provides a relevant and accurate re...  "
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "results.tables[\"eval_results_table\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "application/vnd.databricks.v1+notebook": {
+   "dashboards": [],
+   "language": "python",
+   "notebookMetadata": {
+    "pythonIndentUnit": 2
+   },
+   "notebookName": "LLM Evaluation Examples -- RAG",
+   "widgets": {}
+  },
+  "kernelspec": {
+   "display_name": "mlflow-dev-env",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.19"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}

push_logs.txt

@@ -0,0 +1 @@
+Uploading LFS objects:   0% (0/58), 0 B | 0 B/s, done.

requirements-23feb2025.txt

@@ -0,0 +1,241 @@
+Package                                  Version
+---------------------------------------- ---------
+aiofiles                                 23.2.1
+aiohappyeyeballs                         2.4.0
+aiohttp                                  3.10.5
+aiosignal                                1.3.1
+alembic                                  1.13.2
+altair                                   5.4.0
+aniso8601                                9.0.1
+annotated-types                          0.7.0
+anthropic                                0.34.1
+anyio                                    4.4.0
+appdirs                                  1.4.4
+appnope                                  0.1.4
+asgiref                                  3.8.1
+asttokens                                2.4.1
+async-timeout                            4.0.3
+attrs                                    24.2.0
+backcall                                 0.2.0
+backoff                                  2.2.1
+bcrypt                                   4.2.0
+blinker                                  1.8.2
+Bottleneck                               1.3.7
+build                                    1.2.1
+cachetools                               5.5.0
+cattrs                                   24.1.2
+certifi                                  2024.7.4
+charset-normalizer                       3.3.2
+chroma-hnswlib                           0.7.6
+chromadb                                 0.5.5
+click                                    8.1.7
+click-plugins                            1.1.1
+cligj                                    0.7.2
+cloudpickle                              3.0.0
+coloredlogs                              15.0.1
+comm                                     0.2.2
+contourpy                                1.2.0
+cycler                                   0.12.1
+databricks-sdk                           0.31.1
+dataclasses-json                         0.6.7
+debugpy                                  1.6.7
+decorator                                5.1.1
+defusedxml                               0.7.1
+Deprecated                               1.2.14
+distro                                   1.9.0
+docker                                   7.1.0
+entrypoints                              0.4
+et-xmlfile                               1.1.0
+exceptiongroup                           1.2.2
+executing                                2.0.1
+fastapi                                  0.112.2
+ffmpy                                    0.4.0
+filelock                                 3.15.4
+fiona                                    1.9.5
+Flask                                    3.0.3
+flatbuffers                              24.3.25
+fonttools                                4.25.0
+frozenlist                               1.4.1
+fsspec                                   2024.6.1
+GDAL                                     3.6.2
+geojson-rewind                           1.1.0
+geomet                                   1.1.0
+geopandas                                0.9.0
+gitdb                                    4.0.11
+GitPython                                3.1.43
+google-auth                              2.34.0
+googleapis-common-protos                 1.63.2
+gradio                                   4.42.0
+gradio_client                            1.3.0
+graphene                                 3.3
+graphql-core                             3.2.3
+graphql-relay                            3.2.0
+grpcio                                   1.66.0
+gunicorn                                 23.0.0
+h11                                      0.14.0
+httpcore                                 1.0.5
+httptools                                0.6.1
+httpx                                    0.27.0
+huggingface-hub                          0.24.6
+humanfriendly                            10.0
+idna                                     3.8
+importlib_metadata                       8.0.0
+importlib_resources                      6.4.4
+ipykernel                                6.29.5
+ipython                                  8.12.0
+itsdangerous                             2.2.0
+jedi                                     0.19.1
+Jinja2                                   3.1.4
+jiter                                    0.5.0
+joblib                                   1.4.2
+jsonpatch                                1.33
+jsonpointer                              3.0.0
+jsonschema                               4.23.0
+jsonschema-specifications                2023.12.1
+jupyter-client                           7.3.4
+jupyter_core                             5.7.2
+kiwisolver                               1.4.4
+kubernetes                               30.1.0
+langchain                                0.2.14
+langchain-anthropic                      0.1.23
+langchain-community                      0.2.12
+langchain-core                           0.2.34
+langchain-openai                         0.1.22
+langchain-text-splitters                 0.2.2
+langsmith                                0.1.104
+lightning-utilities                      0.11.7
+Mako                                     1.3.5
+Markdown                                 3.7
+markdown-it-py                           3.0.0
+MarkupSafe                               2.1.5
+marshmallow                              3.22.0
+matplotlib                               3.9.2
+matplotlib-inline                        0.1.7
+mdurl                                    0.1.2
+missingno                                0.5.2
+mlflow                                   2.16.0
+mlflow-skinny                            2.16.0
+mmh3                                     4.1.0
+monotonic                                1.6
+mpmath                                   1.3.0
+multidict                                6.0.5
+munkres                                  1.1.4
+mypy-extensions                          1.0.0
+narwhals                                 1.5.5
+nest_asyncio                             1.6.0
+networkx                                 3.2.1
+nltk                                     3.9.1
+numexpr                                  2.10.1
+numpy                                    1.26.4
+oauthlib                                 3.2.2
+onnxruntime                              1.19.0
+openai                                   1.42.0
+openpyxl                                 3.0.9
+opentelemetry-api                        1.26.0
+opentelemetry-exporter-otlp-proto-common 1.26.0
+opentelemetry-exporter-otlp-proto-grpc   1.26.0
+opentelemetry-instrumentation            0.47b0
+opentelemetry-instrumentation-asgi       0.47b0
+opentelemetry-instrumentation-fastapi    0.47b0
+opentelemetry-proto                      1.26.0
+opentelemetry-sdk                        1.26.0
+opentelemetry-semantic-conventions       0.47b0
+opentelemetry-util-http                  0.47b0
+orjson                                   3.10.7
+overrides                                7.7.0
+packaging                                24.1
+pandas                                   2.0.3
+parso                                    0.8.4
+patsy                                    0.5.6
+pexpect                                  4.9.0
+pickleshare                              0.7.5
+pillow                                   10.4.0
+pip                                      24.2
+platformdirs                             4.2.2
+plotly                                   5.24.1
+posthog                                  3.5.2
+prompt_toolkit                           3.0.47
+protobuf                                 4.25.4
+psutil                                   5.9.0
+ptyprocess                               0.7.0
+pure_eval                                0.2.3
+pyarrow                                  17.0.0
+pyasn1                                   0.6.0
+pyasn1_modules                           0.4.0
+pydantic                                 2.8.2
+pydantic_core                            2.20.1
+pydeck                                   0.9.1
+pydub                                    0.25.1
+pygbif                                   0.6.4
+Pygments                                 2.18.0
+PyMuPDF                                  1.24.9
+PyMuPDFb                                 1.24.9
+pyogrio                                  0.10.0
+pyparsing                                3.1.2
+PyPDF2                                   3.0.1
+PyPika                                   0.48.9
+pyproj                                   3.6.1
+pyproject_hooks                          1.1.0
+python-dateutil                          2.9.0
+python-dotenv                            1.0.1
+python-multipart                         0.0.9
+pytz                                     2024.1
+PyYAML                                   6.0.2
+pyzmq                                    25.1.2
+referencing                              0.35.1
+regex                                    2024.7.24
+requests                                 2.32.3
+requests-cache                           1.2.1
+requests-oauthlib                        2.0.0
+rich                                     13.7.1
+rpds-py                                  0.20.0
+rsa                                      4.9
+Rtree                                    1.0.1
+ruff                                     0.6.2
+scikit-learn                             1.5.1
+scipy                                    1.13.1
+seaborn                                  0.13.2
+semantic-version                         2.10.0
+setuptools                               72.1.0
+shapely                                  2.0.5
+shellingham                              1.5.4
+six                                      1.16.0
+smmap                                    5.0.1
+sniffio                                  1.3.1
+SQLAlchemy                               2.0.32
+sqlparse                                 0.5.1
+stack-data                               0.6.2
+starlette                                0.38.2
+statsmodels                              0.14.2
+streamlit                                1.37.1
+sympy                                    1.13.2
+tenacity                                 8.5.0
+threadpoolctl                            3.5.0
+tiktoken                                 0.7.0
+tokenizers                               0.20.0
+toml                                     0.10.2
+tomli                                    2.0.1
+tomlkit                                  0.12.0
+torch                                    2.4.0
+torchmetrics                             1.4.1
+tornado                                  6.1
+tqdm                                     4.66.5
+traitlets                                5.14.3
+typer                                    0.12.5
+typing_extensions                        4.12.2
+typing-inspect                           0.9.0
+tzdata                                   2024.1
+url-normalize                            1.4.3
+urllib3                                  2.2.2
+uvicorn                                  0.30.6
+uvloop                                   0.20.0
+watchfiles                               0.23.0
+wcwidth                                  0.2.13
+websocket-client                         1.8.0
+websockets                               12.0
+Werkzeug                                 3.0.4
+wget                                     3.2
+wheel                                    0.43.0
+wrapt                                    1.16.0
+yarl                                     1.9.4
+zipp                                     3.20.0

requirements.txt

@@ -0,0 +1,16 @@
+
+chromadb
+gradio
+openai
+langchain
+langchain-anthropic
+langchain_community
+pandas
+python-dotenv
+streamlit
+pypdf2
+tiktoken
+streamlit
+langchain-openai
+pymupdf
+openpyxl