public-apps/groq.livemd

<!-- livebook:{"app_settings":{"access_type":"public","show_source":true,"slug":"groq-agent"}} -->

# Elixir groq

```elixir
Mix.install([
  {:groq, "~> 0.1"},
  {:hackney, "~> 1.18"},
  {:jason, "~> 1.4"},
  {:phoenix_live_view, "~> 0.18.3"},
  {:kino, "~> 0.9.0"}
])

# Check if Jason is available
json_library = if Code.ensure_loaded?(Jason), do: Jason, else: :error

# Configure Groq to use the available JSON library
Application.put_env(:groq, :json_library, json_library)

System.put_env("GROQ_API_KEY", System.fetch_env!("LB_GROQ_API_KEY"))

# Start the Groq application
case Application.ensure_all_started(:groq) do
  {:ok, _} -> IO.puts("Groq application started successfully")
  {:error, error} -> IO.puts("Failed to start Groq application: #{inspect(error)}")
end
```

## Using groq and tool call in liveview

We are gonna use the client from https://github.com/connorjacobsen/groq-elixir/tree/main

Basically is Groq's endpoint wrapper

```elixir
# Ensure all dependencies are started
Application.ensure_all_started(:hackney)
Application.ensure_all_started(:groq)
```

```elixir
response = Groq.ChatCompletion.create(%{
  "model" => "mixtral-8x7b-32768",
  "messages" => [
    %{
      "role" => "user",
      "content" => "Explain the importance of fast language models"
    }
  ]
})

case response do
  {:ok, result} ->
    IO.puts("Response content:")
    case result do
      %{"choices" => [%{"message" => %{"content" => content}} | _]} ->
        IO.puts(content)
      _ ->
        IO.puts("Unexpected response structure: #{inspect(result)}")
    end
  {:error, error} ->
    IO.puts("Error: #{inspect(error)}")
end
```

### Create a custom function to use as a tool

```elixir
# Define a simple function to get the current time
defmodule TimeFunctions do
  def get_current_time do
    {{year, month, day}, {hour, minute, second}} = :calendar.local_time()
    "Current time: #{year}-#{month}-#{day} #{hour}:#{minute}:#{second}"
  end
end

# Define the function's properties
function_properties = [
  %{
    "name" => "get_current_time",
    "description" => "Get the current time",
    "parameters" => %{
      "type" => "object",
      "properties" => %{},
      "required" => []
    }
  }
]
```

### Let's call the function if the user need it

```elixir
# Create a chat completion request with function calling
response = Groq.ChatCompletion.create(%{
  "model" => "mixtral-8x7b-32768",
  "messages" => [
    %{
      "role" => "user",
      "content" => "What time is it?"
    }
  ],
  "tools" => [
    %{
      "type" => "function",
      "function" => Enum.at(function_properties, 0)
    }
  ],
  "tool_choice" => "auto"
})

# Handle the response
case response do
  {:ok, result} ->
    case result do
      %{"choices" => [%{"message" => message} | _]} ->
        case message do
          %{"tool_calls" => [%{"function" => %{"name" => "get_current_time"}} | _]} ->
            time = TimeFunctions.get_current_time()
            IO.puts("Assistant requested the current time. #{time}")
          _ ->
            IO.puts("Assistant response: #{message["content"]}")
        end
      _ ->
        IO.puts("Unexpected response structure: #{inspect(result)}")
    end
  {:error, error} ->
    IO.puts("Error: #{inspect(error)}")
end


```

### Let's build a Math Agent.

Since the models can operate mathematical operations we are going to create an Ange to understand user query and if there any mathematical expresion we are going to use a 'calculate' function to process it.

```elixir
defmodule MathAgent do
  def calculate(expression) do
    try do
      result = Code.eval_string(expression) |> elem(0)
      Jason.encode!(%{result: result})
    rescue
      _ -> Jason.encode!(%{error: "Invalid expression"})
    end
  end

  def function_properties do
    [
      %{
        "type" => "function",
        "function" => %{
          "name" => "calculate",
          "description" => "Evaluate a mathematical expression",
          "parameters" => %{
            "type" => "object",
            "properties" => %{
              "expression" => %{
                "type" => "string",
                "description" => "The mathematical expression to evaluate"
              }
            },
            "required" => ["expression"]
          }
        }
      }
    ]
  end

  def create_chat_completion(messages) do
    Groq.ChatCompletion.create(%{
      "model" => "llama3-groq-70b-8192-tool-use-preview",
      "messages" => messages,
      "tools" => function_properties(),
      "tool_choice" => "auto",
      "max_tokens" => 4096
    })
  end

  def handle_response({:ok, result}) do
    case result do
      %{"choices" => choices} when is_list(choices) and length(choices) > 0 ->
        first_choice = Enum.at(choices, 0)
        handle_message(first_choice["message"])
      _ ->
        {:error, "Unexpected response structure: #{inspect(result)}"}
    end
  end

  def handle_response({:error, error}) do
    {:error, "Error: #{inspect(error)}"}
  end

  defp handle_message(%{"tool_calls" => [tool_call | _]} = message) do
    IO.puts("\nModel is using a tool:")
    IO.inspect(message, label: "Full message")
    
    %{"function" => %{"name" => function_name, "arguments" => arguments}} = tool_call
    case function_name do
      "calculate" ->
        args = Jason.decode!(arguments)
        IO.puts("Calling calculate function with expression: #{args["expression"]}")
        result = calculate(args["expression"])
        IO.puts("Calculate function result: #{result}")
        {:tool_call, tool_call["id"], function_name, result}
      _ ->
        {:error, "Unknown function: #{function_name}"}
    end
  end

  defp handle_message(message) do
    IO.puts("\nModel is not using a tool:")
    IO.inspect(message, label: "Full message")
    {:ok, message["content"]}
  end

  def run_conversation(user_prompt) do
    IO.puts("Starting conversation with user prompt: #{user_prompt}")
    
    initial_messages = [
      %{
        "role" => "system",
        "content" => "You are a calculator assistant. 
        Use the calculate function to perform mathematical operations and provide the results. 
        If the answer is not about Math, you will respond: Eee locoo, aguante Cristinaaaa 🇦🇷!"
      },
      %{
        "role" => "user",
        "content" => user_prompt
      }
    ]

    case create_chat_completion(initial_messages) do
      {:ok, result} ->
        IO.puts("\nReceived initial response from Groq API")
        case handle_response({:ok, result}) do
          {:tool_call, id, name, content} ->
            IO.puts("\nProcessing tool call result")
            tool_message = %{
              "tool_call_id" => id,
              "role" => "tool",
              "name" => name,
              "content" => content
            }
            first_choice = Enum.at(result["choices"], 0)
            new_messages = initial_messages ++ [first_choice["message"], tool_message]
            IO.puts("\nSending follow-up request to Groq API")
            case create_chat_completion(new_messages) do
              {:ok, final_result} -> 
                IO.puts("\nReceived final response from Groq API")
                handle_response({:ok, final_result})
              error -> error
            end
          other -> other
        end
      error -> error
    end
  end
end
```

### Let's try with user input

Trye

```elixir
user_input = "What is 25 * 4 + 10?"

IO.puts("Running conversation with input: #{user_input}\n")
case MathAgent.run_conversation(user_input) do
  {:ok, response} -> IO.puts("\nFinal Response: #{response}")
  {:error, error} -> IO.puts("\nError: #{error}")
end
```

#### If we query some question that is not related about math we are going to receive according of what we have in our system prompt on how the model have to behave

```elixir
# Example of a query that might not use the tool
user_input_no_tool = "What's the capital of France?"

IO.puts("\n\nRunning conversation with input that might not use the tool: #{user_input_no_tool}\n")
case MathAgent.run_conversation(user_input_no_tool) do
  {:ok, response} -> IO.puts("\nFinal Response: #{response}")
  {:error, error} -> IO.puts("\nError: #{error}")
end
```

### Use Case: Retrieve information from "database" using natural lenguage as a query

<!-- livebook:{"break_markdown":true} -->

Let's create a fake 50.000 user database

```elixir
fake_database = Enum.map(1..50_000, fn _ ->
  user_id = :crypto.strong_rand_bytes(16) |> Base.encode16(case: :lower)
  minutes = :rand.uniform(1000)
  {user_id, minutes}
end)

IO.puts("Sample of fake data (first 10 entries):")
fake_database |> Enum.take(10) |> IO.inspect()

defmodule DataStore do
  def get_fake_database do
    unquote(Macro.escape(fake_database))
  end
end

IO.puts("Fake database generated and stored in DataStore module.")
```

###### Le'ts create a GroqChat module and create a function as a tool with the properties and call the client.

###### We are going to use Kino to create a funny interface :)

```elixir
defmodule GroqChat do
  # Use the fake database from the DataStore module
  @fake_database DataStore.get_fake_database()

  def get_top_users(n) do
    @fake_database
    |> Enum.sort_by(fn {_, minutes} -> minutes end, :desc)
    |> Enum.take(n)
    |> Enum.map(fn {user_id, minutes} -> %{user_id: user_id, minutes: minutes} end)
    |> Jason.encode!()
  end

  def function_properties do
    [
      %{
        "type" => "function",
        "function" => %{
          "name" => "get_top_users",
          "description" => "Get the top N users with the most system usage time",
          "parameters" => %{
            "type" => "object",
            "properties" => %{
              "n" => %{
                "type" => "integer",
                "description" => "Number of top users to retrieve"
              }
            },
            "required" => ["n"]
          }
        }
      }
    ]
  end

  def create_chat_completion(messages) do
    Groq.ChatCompletion.create(
      %{
        "model" => "llama3-groq-70b-8192-tool-use-preview",
        "messages" => messages,
        "tools" => function_properties(),
        "tool_choice" => "auto",
        "max_tokens" => 4096
      },
      recv_timeout: 30_000  # Increase timeout to 30 seconds
    )
  end

  def handle_response({:ok, result}) do
    case result do
      %{"choices" => choices} when is_list(choices) and length(choices) > 0 ->
        first_choice = Enum.at(choices, 0)
        handle_message(first_choice["message"])
      _ ->
        {:error, "Unexpected response structure: #{inspect(result)}"}
    end
  end

  def handle_response({:error, error}) do
    {:error, "Error: #{inspect(error)}"}
  end

  defp handle_message(%{"tool_calls" => [tool_call | _]} = message) do
    IO.puts("\nModel is using a tool:")
    IO.inspect(message, label: "Full message")
    
    %{"function" => %{"name" => function_name, "arguments" => arguments}} = tool_call
    case function_name do
      "get_top_users" ->
        args = Jason.decode!(arguments)
        IO.puts("Calling get_top_users function with n: #{args["n"]}")
        result = get_top_users(args["n"])
        IO.puts("Get top users function result: #{result}")
        {:tool_call, tool_call["id"], function_name, result}
      _ ->
        {:error, "Unknown function: #{function_name}"}
    end
  end

  defp handle_message(message) do
    IO.puts("\nModel is not using a tool:")
    IO.inspect(message, label: "Full message")
    {:ok, message["content"]}
  end

  def run_conversation(user_prompt) do
    IO.puts("Starting conversation with user prompt: #{user_prompt}")
    
    initial_messages = [
      %{
        "role" => "system",
        "content" => "You are an assistant capable of retrieving information about top system users. 
        Use the get_top_users function to retrieve information about users in database with 
        the most system usage time. If question is not about user or mintues info respond: Eyy guachoo, esto solo para database"
      },
      %{
        "role" => "user",
        "content" => user_prompt
      }
    ]

    case create_chat_completion(initial_messages) do
      {:ok, result} ->
        IO.puts("\nReceived initial response from Groq API")
        case handle_response({:ok, result}) do
          {:tool_call, id, name, content} ->
            IO.puts("\nProcessing tool call result")
            tool_message = %{
              "tool_call_id" => id,
              "role" => "tool",
              "name" => name,
              "content" => content
            }
            first_choice = Enum.at(result["choices"], 0)
            new_messages = initial_messages ++ [first_choice["message"], tool_message]
            IO.puts("\nSending follow-up request to Groq API")
            case create_chat_completion(new_messages) do
              {:ok, final_result} -> 
                IO.puts("\nReceived final response from Groq API")
                handle_response({:ok, final_result})
              error -> error
            end
          other -> other
        end
      error -> error
    end
  end
end

# Create an input form with a submit option
form = Kino.Control.form(
  [
    query: Kino.Input.text("Enter your query. Example: list the 5 user with most used the system?")
  ],
  submit: "Send"
)

# Create a frame to display the response
frame = Kino.Frame.new()

# Set up the event stream for form submission
Kino.Control.stream(form)
|> Kino.listen(fn %{data: %{query: query}} ->
  Kino.Frame.render(frame, Kino.Markdown.new("**Loading...**"))
  
  # Make the API call
  case GroqChat.run_conversation(query) do
    {:ok, response} ->
      Kino.Frame.render(frame, Kino.Markdown.new("**Response:**\n\n#{response}"))
    {:error, error} ->
      Kino.Frame.render(frame, Kino.Markdown.new("**Error:**\n\n#{error}"))
  end
end)

# Render the UI elements
Kino.Layout.grid([
  form,
  frame
])
```

<!-- livebook:{"offset":13926,"stamp":{"token":"XCP.uif9L-BcwPrb46PT8dWaX-PK1L_knFhTLHIWcWPSMch2yDch8FA8yfOYF8Se-7uQ0JK-TJ0aFK7cW-wXCB9Y_hnMp5M3nT0n1HXBTRz3HNLcOqbTUN30","version":2}} -->