# This file was auto-generated by Fern from our API Definition.

import typing

from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper
from ..core.request_options import RequestOptions
from ..types.evaluation import Evaluation
from ..types.evaluation_scenario import EvaluationScenario
from ..types.llm_run_rate_limit import LlmRunRateLimit
from .raw_client import AsyncRawEvaluationsClient, RawEvaluationsClient

# this is used as the default value for optional parameters
OMIT = typing.cast(typing.Any, ...)


class EvaluationsClient:
    def __init__(self, *, client_wrapper: SyncClientWrapper):
        self._raw_client = RawEvaluationsClient(client_wrapper=client_wrapper)

    @property
    def with_raw_response(self) -> RawEvaluationsClient:
        """
        Retrieves a raw implementation of this client that returns raw responses.

        Returns
        -------
        RawEvaluationsClient
        """
        return self._raw_client

    def fetch_evaluation_ids(
        self,
        *,
        resource_type: str,
        resource_ids: typing.Optional[typing.Union[str, typing.Sequence[str]]] = None,
        request_options: typing.Optional[RequestOptions] = None,
    ) -> typing.List[str]:
        """
        Fetches evaluation ids for a given resource type and id.

        Arguments:
            app_id (str): The ID of the app for which to fetch evaluations.
            resource_type (str): The type of resource for which to fetch evaluations.
            resource_ids List[ObjectId]: The IDs of resource for which to fetch evaluations.

        Raises:
            HTTPException: If the resource_type is invalid or access is denied.

        Returns:
            List[str]: A list of evaluation ids.

        Parameters
        ----------
        resource_type : str

        resource_ids : typing.Optional[typing.Union[str, typing.Sequence[str]]]

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        typing.List[str]
            Successful Response

        Examples
        --------
        from agenta import AgentaApi

        client = AgentaApi(
            api_key="YOUR_API_KEY",
        )
        client.evaluations.fetch_evaluation_ids(
            resource_type="resource_type",
        )
        """
        _response = self._raw_client.fetch_evaluation_ids(
            resource_type=resource_type,
            resource_ids=resource_ids,
            request_options=request_options,
        )
        return _response.data

    def fetch_list_evaluations(
        self, *, app_id: str, request_options: typing.Optional[RequestOptions] = None
    ) -> typing.List[Evaluation]:
        """
        Fetches a list of evaluations, optionally filtered by an app ID.

        Args:
            app_id (Optional[str]): An optional app ID to filter the evaluations.

        Returns:
            List[Evaluation]: A list of evaluations.

        Parameters
        ----------
        app_id : str

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        typing.List[Evaluation]
            Successful Response

        Examples
        --------
        from agenta import AgentaApi

        client = AgentaApi(
            api_key="YOUR_API_KEY",
        )
        client.evaluations.fetch_list_evaluations(
            app_id="app_id",
        )
        """
        _response = self._raw_client.fetch_list_evaluations(
            app_id=app_id, request_options=request_options
        )
        return _response.data

    def create_evaluation(
        self,
        *,
        app_id: str,
        revisions_ids: typing.Sequence[str],
        evaluators_configs: typing.Sequence[str],
        testset_id: str,
        rate_limit: LlmRunRateLimit,
        correct_answer_column: typing.Optional[str] = OMIT,
        request_options: typing.Optional[RequestOptions] = None,
    ) -> typing.List[Evaluation]:
        """
        Creates a new comparison table document
        Raises:
            HTTPException: _description_
        Returns:
            _description_

        Parameters
        ----------
        app_id : str

        revisions_ids : typing.Sequence[str]

        evaluators_configs : typing.Sequence[str]

        testset_id : str

        rate_limit : LlmRunRateLimit

        correct_answer_column : typing.Optional[str]

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        typing.List[Evaluation]
            Successful Response

        Examples
        --------
        from agenta import AgentaApi, LlmRunRateLimit

        client = AgentaApi(
            api_key="YOUR_API_KEY",
        )
        client.evaluations.create_evaluation(
            app_id="app_id",
            revisions_ids=["revisions_ids"],
            evaluators_configs=["evaluators_configs"],
            testset_id="testset_id",
            rate_limit=LlmRunRateLimit(
                batch_size=1,
                max_retries=1,
                retry_delay=1,
                delay_between_batches=1,
            ),
        )
        """
        _response = self._raw_client.create_evaluation(
            app_id=app_id,
            revisions_ids=revisions_ids,
            evaluators_configs=evaluators_configs,
            testset_id=testset_id,
            rate_limit=rate_limit,
            correct_answer_column=correct_answer_column,
            request_options=request_options,
        )
        return _response.data

    def delete_evaluations(
        self,
        *,
        evaluations_ids: typing.Sequence[str],
        request_options: typing.Optional[RequestOptions] = None,
    ) -> typing.List[str]:
        """
        Delete specific comparison tables based on their unique IDs.

        Args:
        delete_evaluations (List[str]): The unique identifiers of the comparison tables to delete.

        Returns:
        A list of the deleted comparison tables' IDs.

        Parameters
        ----------
        evaluations_ids : typing.Sequence[str]

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        typing.List[str]
            Successful Response

        Examples
        --------
        from agenta import AgentaApi

        client = AgentaApi(
            api_key="YOUR_API_KEY",
        )
        client.evaluations.delete_evaluations(
            evaluations_ids=["evaluations_ids"],
        )
        """
        _response = self._raw_client.delete_evaluations(
            evaluations_ids=evaluations_ids, request_options=request_options
        )
        return _response.data

    def fetch_evaluation_status(
        self,
        evaluation_id: str,
        *,
        request_options: typing.Optional[RequestOptions] = None,
    ) -> typing.Optional[typing.Any]:
        """
        Fetches the status of the evaluation.

        Args:
            evaluation_id (str): the evaluation id
            request (Request): the request object

        Returns:
            (str): the evaluation status

        Parameters
        ----------
        evaluation_id : str

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        typing.Optional[typing.Any]
            Successful Response

        Examples
        --------
        from agenta import AgentaApi

        client = AgentaApi(
            api_key="YOUR_API_KEY",
        )
        client.evaluations.fetch_evaluation_status(
            evaluation_id="evaluation_id",
        )
        """
        _response = self._raw_client.fetch_evaluation_status(
            evaluation_id, request_options=request_options
        )
        return _response.data

    def fetch_evaluation_results(
        self,
        evaluation_id: str,
        *,
        request_options: typing.Optional[RequestOptions] = None,
    ) -> typing.Optional[typing.Any]:
        """
        Fetches the results of the evaluation

        Args:
            evaluation_id (str): the evaluation id
            request (Request): the request object

        Returns:
            _type_: _description_

        Parameters
        ----------
        evaluation_id : str

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        typing.Optional[typing.Any]
            Successful Response

        Examples
        --------
        from agenta import AgentaApi

        client = AgentaApi(
            api_key="YOUR_API_KEY",
        )
        client.evaluations.fetch_evaluation_results(
            evaluation_id="evaluation_id",
        )
        """
        _response = self._raw_client.fetch_evaluation_results(
            evaluation_id, request_options=request_options
        )
        return _response.data

    def fetch_evaluation_scenarios(
        self,
        evaluation_id: str,
        *,
        request_options: typing.Optional[RequestOptions] = None,
    ) -> typing.List[EvaluationScenario]:
        """
        Fetches evaluation scenarios for a given evaluation ID.

        Arguments:
            evaluation_id (str): The ID of the evaluation for which to fetch scenarios.

        Raises:
            HTTPException: If the evaluation is not found or access is denied.

        Returns:
            List[EvaluationScenario]: A list of evaluation scenarios.

        Parameters
        ----------
        evaluation_id : str

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        typing.List[EvaluationScenario]
            Successful Response

        Examples
        --------
        from agenta import AgentaApi

        client = AgentaApi(
            api_key="YOUR_API_KEY",
        )
        client.evaluations.fetch_evaluation_scenarios(
            evaluation_id="evaluation_id",
        )
        """
        _response = self._raw_client.fetch_evaluation_scenarios(
            evaluation_id, request_options=request_options
        )
        return _response.data

    def fetch_evaluation(
        self,
        evaluation_id: str,
        *,
        request_options: typing.Optional[RequestOptions] = None,
    ) -> Evaluation:
        """
        Fetches a single evaluation based on its ID.

        Args:
            evaluation_id (str): The ID of the evaluation to fetch.

        Returns:
            Evaluation: The fetched evaluation.

        Parameters
        ----------
        evaluation_id : str

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        Evaluation
            Successful Response

        Examples
        --------
        from agenta import AgentaApi

        client = AgentaApi(
            api_key="YOUR_API_KEY",
        )
        client.evaluations.fetch_evaluation(
            evaluation_id="evaluation_id",
        )
        """
        _response = self._raw_client.fetch_evaluation(
            evaluation_id, request_options=request_options
        )
        return _response.data

    def fetch_evaluation_scenarios_comparison_results(
        self,
        *,
        evaluations_ids: str,
        request_options: typing.Optional[RequestOptions] = None,
    ) -> typing.Optional[typing.Any]:
        """
        Fetches evaluation scenarios for a given evaluation ID.

        Arguments:
            evaluation_id (str): The ID of the evaluation for which to fetch scenarios.

        Raises:
            HTTPException: If the evaluation is not found or access is denied.

        Returns:
            List[EvaluationScenario]: A list of evaluation scenarios.

        Parameters
        ----------
        evaluations_ids : str

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        typing.Optional[typing.Any]
            Successful Response

        Examples
        --------
        from agenta import AgentaApi

        client = AgentaApi(
            api_key="YOUR_API_KEY",
        )
        client.evaluations.fetch_evaluation_scenarios_comparison_results(
            evaluations_ids="evaluations_ids",
        )
        """
        _response = self._raw_client.fetch_evaluation_scenarios_comparison_results(
            evaluations_ids=evaluations_ids, request_options=request_options
        )
        return _response.data


class AsyncEvaluationsClient:
    def __init__(self, *, client_wrapper: AsyncClientWrapper):
        self._raw_client = AsyncRawEvaluationsClient(client_wrapper=client_wrapper)

    @property
    def with_raw_response(self) -> AsyncRawEvaluationsClient:
        """
        Retrieves a raw implementation of this client that returns raw responses.

        Returns
        -------
        AsyncRawEvaluationsClient
        """
        return self._raw_client

    async def fetch_evaluation_ids(
        self,
        *,
        resource_type: str,
        resource_ids: typing.Optional[typing.Union[str, typing.Sequence[str]]] = None,
        request_options: typing.Optional[RequestOptions] = None,
    ) -> typing.List[str]:
        """
        Fetches evaluation ids for a given resource type and id.

        Arguments:
            app_id (str): The ID of the app for which to fetch evaluations.
            resource_type (str): The type of resource for which to fetch evaluations.
            resource_ids List[ObjectId]: The IDs of resource for which to fetch evaluations.

        Raises:
            HTTPException: If the resource_type is invalid or access is denied.

        Returns:
            List[str]: A list of evaluation ids.

        Parameters
        ----------
        resource_type : str

        resource_ids : typing.Optional[typing.Union[str, typing.Sequence[str]]]

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        typing.List[str]
            Successful Response

        Examples
        --------
        import asyncio

        from agenta import AsyncAgentaApi

        client = AsyncAgentaApi(
            api_key="YOUR_API_KEY",
        )


        async def main() -> None:
            await client.evaluations.fetch_evaluation_ids(
                resource_type="resource_type",
            )


        asyncio.run(main())
        """
        _response = await self._raw_client.fetch_evaluation_ids(
            resource_type=resource_type,
            resource_ids=resource_ids,
            request_options=request_options,
        )
        return _response.data

    async def fetch_list_evaluations(
        self, *, app_id: str, request_options: typing.Optional[RequestOptions] = None
    ) -> typing.List[Evaluation]:
        """
        Fetches a list of evaluations, optionally filtered by an app ID.

        Args:
            app_id (Optional[str]): An optional app ID to filter the evaluations.

        Returns:
            List[Evaluation]: A list of evaluations.

        Parameters
        ----------
        app_id : str

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        typing.List[Evaluation]
            Successful Response

        Examples
        --------
        import asyncio

        from agenta import AsyncAgentaApi

        client = AsyncAgentaApi(
            api_key="YOUR_API_KEY",
        )


        async def main() -> None:
            await client.evaluations.fetch_list_evaluations(
                app_id="app_id",
            )


        asyncio.run(main())
        """
        _response = await self._raw_client.fetch_list_evaluations(
            app_id=app_id, request_options=request_options
        )
        return _response.data

    async def create_evaluation(
        self,
        *,
        app_id: str,
        revisions_ids: typing.Sequence[str],
        evaluators_configs: typing.Sequence[str],
        testset_id: str,
        rate_limit: LlmRunRateLimit,
        correct_answer_column: typing.Optional[str] = OMIT,
        request_options: typing.Optional[RequestOptions] = None,
    ) -> typing.List[Evaluation]:
        """
        Creates a new comparison table document
        Raises:
            HTTPException: _description_
        Returns:
            _description_

        Parameters
        ----------
        app_id : str

        revisions_ids : typing.Sequence[str]

        evaluators_configs : typing.Sequence[str]

        testset_id : str

        rate_limit : LlmRunRateLimit

        correct_answer_column : typing.Optional[str]

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        typing.List[Evaluation]
            Successful Response

        Examples
        --------
        import asyncio

        from agenta import AsyncAgentaApi, LlmRunRateLimit

        client = AsyncAgentaApi(
            api_key="YOUR_API_KEY",
        )


        async def main() -> None:
            await client.evaluations.create_evaluation(
                app_id="app_id",
                revisions_ids=["revisions_ids"],
                evaluators_configs=["evaluators_configs"],
                testset_id="testset_id",
                rate_limit=LlmRunRateLimit(
                    batch_size=1,
                    max_retries=1,
                    retry_delay=1,
                    delay_between_batches=1,
                ),
            )


        asyncio.run(main())
        """
        _response = await self._raw_client.create_evaluation(
            app_id=app_id,
            revisions_ids=revisions_ids,
            evaluators_configs=evaluators_configs,
            testset_id=testset_id,
            rate_limit=rate_limit,
            correct_answer_column=correct_answer_column,
            request_options=request_options,
        )
        return _response.data

    async def delete_evaluations(
        self,
        *,
        evaluations_ids: typing.Sequence[str],
        request_options: typing.Optional[RequestOptions] = None,
    ) -> typing.List[str]:
        """
        Delete specific comparison tables based on their unique IDs.

        Args:
        delete_evaluations (List[str]): The unique identifiers of the comparison tables to delete.

        Returns:
        A list of the deleted comparison tables' IDs.

        Parameters
        ----------
        evaluations_ids : typing.Sequence[str]

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        typing.List[str]
            Successful Response

        Examples
        --------
        import asyncio

        from agenta import AsyncAgentaApi

        client = AsyncAgentaApi(
            api_key="YOUR_API_KEY",
        )


        async def main() -> None:
            await client.evaluations.delete_evaluations(
                evaluations_ids=["evaluations_ids"],
            )


        asyncio.run(main())
        """
        _response = await self._raw_client.delete_evaluations(
            evaluations_ids=evaluations_ids, request_options=request_options
        )
        return _response.data

    async def fetch_evaluation_status(
        self,
        evaluation_id: str,
        *,
        request_options: typing.Optional[RequestOptions] = None,
    ) -> typing.Optional[typing.Any]:
        """
        Fetches the status of the evaluation.

        Args:
            evaluation_id (str): the evaluation id
            request (Request): the request object

        Returns:
            (str): the evaluation status

        Parameters
        ----------
        evaluation_id : str

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        typing.Optional[typing.Any]
            Successful Response

        Examples
        --------
        import asyncio

        from agenta import AsyncAgentaApi

        client = AsyncAgentaApi(
            api_key="YOUR_API_KEY",
        )


        async def main() -> None:
            await client.evaluations.fetch_evaluation_status(
                evaluation_id="evaluation_id",
            )


        asyncio.run(main())
        """
        _response = await self._raw_client.fetch_evaluation_status(
            evaluation_id, request_options=request_options
        )
        return _response.data

    async def fetch_evaluation_results(
        self,
        evaluation_id: str,
        *,
        request_options: typing.Optional[RequestOptions] = None,
    ) -> typing.Optional[typing.Any]:
        """
        Fetches the results of the evaluation

        Args:
            evaluation_id (str): the evaluation id
            request (Request): the request object

        Returns:
            _type_: _description_

        Parameters
        ----------
        evaluation_id : str

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        typing.Optional[typing.Any]
            Successful Response

        Examples
        --------
        import asyncio

        from agenta import AsyncAgentaApi

        client = AsyncAgentaApi(
            api_key="YOUR_API_KEY",
        )


        async def main() -> None:
            await client.evaluations.fetch_evaluation_results(
                evaluation_id="evaluation_id",
            )


        asyncio.run(main())
        """
        _response = await self._raw_client.fetch_evaluation_results(
            evaluation_id, request_options=request_options
        )
        return _response.data

    async def fetch_evaluation_scenarios(
        self,
        evaluation_id: str,
        *,
        request_options: typing.Optional[RequestOptions] = None,
    ) -> typing.List[EvaluationScenario]:
        """
        Fetches evaluation scenarios for a given evaluation ID.

        Arguments:
            evaluation_id (str): The ID of the evaluation for which to fetch scenarios.

        Raises:
            HTTPException: If the evaluation is not found or access is denied.

        Returns:
            List[EvaluationScenario]: A list of evaluation scenarios.

        Parameters
        ----------
        evaluation_id : str

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        typing.List[EvaluationScenario]
            Successful Response

        Examples
        --------
        import asyncio

        from agenta import AsyncAgentaApi

        client = AsyncAgentaApi(
            api_key="YOUR_API_KEY",
        )


        async def main() -> None:
            await client.evaluations.fetch_evaluation_scenarios(
                evaluation_id="evaluation_id",
            )


        asyncio.run(main())
        """
        _response = await self._raw_client.fetch_evaluation_scenarios(
            evaluation_id, request_options=request_options
        )
        return _response.data

    async def fetch_evaluation(
        self,
        evaluation_id: str,
        *,
        request_options: typing.Optional[RequestOptions] = None,
    ) -> Evaluation:
        """
        Fetches a single evaluation based on its ID.

        Args:
            evaluation_id (str): The ID of the evaluation to fetch.

        Returns:
            Evaluation: The fetched evaluation.

        Parameters
        ----------
        evaluation_id : str

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        Evaluation
            Successful Response

        Examples
        --------
        import asyncio

        from agenta import AsyncAgentaApi

        client = AsyncAgentaApi(
            api_key="YOUR_API_KEY",
        )


        async def main() -> None:
            await client.evaluations.fetch_evaluation(
                evaluation_id="evaluation_id",
            )


        asyncio.run(main())
        """
        _response = await self._raw_client.fetch_evaluation(
            evaluation_id, request_options=request_options
        )
        return _response.data

    async def fetch_evaluation_scenarios_comparison_results(
        self,
        *,
        evaluations_ids: str,
        request_options: typing.Optional[RequestOptions] = None,
    ) -> typing.Optional[typing.Any]:
        """
        Fetches evaluation scenarios for a given evaluation ID.

        Arguments:
            evaluation_id (str): The ID of the evaluation for which to fetch scenarios.

        Raises:
            HTTPException: If the evaluation is not found or access is denied.

        Returns:
            List[EvaluationScenario]: A list of evaluation scenarios.

        Parameters
        ----------
        evaluations_ids : str

        request_options : typing.Optional[RequestOptions]
            Request-specific configuration.

        Returns
        -------
        typing.Optional[typing.Any]
            Successful Response

        Examples
        --------
        import asyncio

        from agenta import AsyncAgentaApi

        client = AsyncAgentaApi(
            api_key="YOUR_API_KEY",
        )


        async def main() -> None:
            await client.evaluations.fetch_evaluation_scenarios_comparison_results(
                evaluations_ids="evaluations_ids",
            )


        asyncio.run(main())
        """
        _response = (
            await self._raw_client.fetch_evaluation_scenarios_comparison_results(
                evaluations_ids=evaluations_ids, request_options=request_options
            )
        )
        return _response.data
