# Create a virtual environment
mkdir llm-benchmark && cd llm-benchmark
python3 -m venv llm-benchmark-env
source llm-benchmark-env/bin/activate

# Install dependencies
pip3 install ipykernel jupyter
pip3 install python-dotenv
pip3 install --upgrade together
pip3 install openai
pip3 install langchain
pip3 install litellm click tqdm tabulate termcolor
pip3 install pandas
# Clone LLM-Bench repository
git clone https://github.com/BerriAI/litellm
# Optionally, fire up VSCode or your favorite IDE and let's get rolling!
code .

TOGETHER_API_KEY=<Your API Key>
OPENAI_API_KEY=<Your API KEY>

import os
import time
import json
import logging
import pandas as pd
from datetime import datetime

import together
import openai
from langchain.llms.base import LLM
from langchain import PromptTemplate,  LLMChain
# The dotenv library's load_dotenv function reads a .env file 
# to load environment variables into the process environment. 
# This is a common method to handle configuration settings securely.
from dotenv import load_dotenv 
# Load env variables
load_dotenv()
# Set up logging
logging.basicConfig(level=logging.INFO)

model_list = together.Models.list()
print(f"There are {len(model_list)} models to choose from!")
[model['name'] for model in model_list][:20]

['Austism/chronos-hermes-13b',
 'DiscoResearch/DiscoLM-mixtral-8x7b-v2',
 'EleutherAI/llemma_7b',
 'Gryphe/MythoMax-L2-13b',
 'Meta-Llama/Llama-Guard-7b',
 'Nexusflow/NexusRaven-V2-13B',
 'NousResearch/Nous-Capybara-7B-V1p9',
 'NousResearch/Nous-Hermes-Llama2-13b',
 'NousResearch/Nous-Hermes-Llama2-70b',
 'NousResearch/Nous-Hermes-llama-2-7b',
 'NumbersStation/nsql-llama-2-7B',
 'Open-Orca/Mistral-7B-OpenOrca',
 'Phind/Phind-CodeLlama-34B-Python-v1',
 'Phind/Phind-CodeLlama-34B-v2',
 'SG161222/Realistic_Vision_V3.0_VAE',
 'Undi95/ReMM-SLERP-L2-13B',
 'Undi95/Toppy-M-7B',
 'WizardLM/WizardCoder-15B-V1.0',
 'WizardLM/WizardCoder-Python-34B-V1.0',
 'WizardLM/WizardLM-13B-V1.2']

client = openai.OpenAI(
    api_key=os.environ.get("TOGETHER_API_KEY"),
    base_url='https://api.together.xyz',
)

chat_completion = client.chat.completions.create(
messages=[
    {
    "role": "user",
    "content": "Tell me about Mixture of Experts",
    }
],
    model="mistralai/Mixtral-8x7B-Instruct-v0.1",
    max_tokens=1024
)
print(chat_completion.choices[0].message.content)

INFO:httpx:HTTP Request: POST https://api.together.xyz/chat/completions "HTTP/1.1 200 OK"

Mixture of Experts (MoE) is a type of machine learning model that combines 
the predictions of multiple simpler models, or "experts," to make more 
accurate and flexible predictions. The idea behind MoE is to divide the 
input space into regions, or "expertise areas," and to have each expert 
specialize in making predictions for a particular region.

At a high level, a MoE model consists of two main components: a gating 
network and a set of expert networks. The gating network is responsible 
for determining which expert should be used to make a prediction for a 
given input. It does this by computing a set of weights, or "gating scores," 
that indicate the relative expertise of each expert for the input. The expert 
networks, on the other hand, are responsible for making the actual predictions. Each expert network is trained to specialize in making predictions for a particular region of the input space.

During the training process, the MoE model is presented with a set of 
input-output pairs and learns to adjust the gating scores and the parameters 
of the expert networks to minimize the overall prediction error. This allows 
the model to learn a set of experts that are well-suited to making 
predictions for different regions of the input space.

One of the key benefits of MoE models is that they can be more accurate 
and flexible than a single, monolithic model. This is because they are 
able to adapt to the structure of the data and make more specialized 
predictions for different regions of the input space. Additionally, MoE 
models can be more efficient to train and use than a single, large model, 
because the gating network can select a small number of experts to use for 
a given input.

MoE models have been applied to a wide range of machine learning tasks, 
including regression, classification, and density estimation. They have 
been shown to be particularly effective in settings where the data is 
high-dimensional, noisy, or has a complex structure.

prompt = "<|im_start|>user\nTell me about Mixture of Experts<|im_end|>\n<|im_start|>assistant"
model = "mistralai/Mixtral-8x7B-Instruct-v0.1"

output = together.Complete.create(
  prompt = prompt,
  model = model, 
  max_tokens = 512,
  temperature = 0.7,
  top_k = 50,
  top_p = 0.7,
  repetition_penalty = 1,
  stop = ["<|im_end|>", "<|im_start|>"]
)
print(json.dumps(output, indent = 4))

{
    "id": "8354ffbaa9560b85-AMS",
    "status": "finished",
    "prompt": [
        "<|im_start|>user\nTell me about Mixture of Experts<|im_end|>\n<|im_start|>assistant"
    ],
    "model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
    "model_owner": "",
    "num_returns": 1,
    "args": {
        "model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
        "prompt": "<|im_start|>user\nTell me about Mixture of Experts<|im_end|>\n<|im_start|>assistant",
        "top_p": 0.7,
        "top_k": 50,
        "temperature": 0.7,
        "max_tokens": 512,
        "stop": [
            "<|im_end|>",
            "<|im_start|>"
        ],
        "repetition_penalty": 1,
        "logprobs": null,
        "safety_model": null
    },
    "subjobs": [],
    "output": {
        "usage": {
            "prompt_tokens": 35,
            "completion_tokens": 182,
            "total_tokens": 217
        },
        "result_type": "language-model-inference",
        "choices": [
            {
                "text": "\nMixture of Experts (MoE) is a type of machine learning model that combines the predictions of multiple expert models to make a final prediction. The idea is to have a group of models that are each good at different things, and then use a gating network to decide which model to use for a given input.\n\nThis approach can be particularly useful in situations where the data is complex and no single model can effectively handle all aspects of it. By using a mixture of experts, the model can adapt to the different parts of the data, improving its overall performance.\n\nIn practice, Mixture of Experts models can be quite complex, involving many different models and a sophisticated gating network. However, they have been shown to be very effective in a variety of applications, including natural language processing, speech recognition, and computer vision.\n"
            }
        ]
    }
}

print(output['output']['choices'][0]['text'])
# Mixture of Experts (MoE) is a type of machine learning model that ...

for token in together.Complete.create_streaming(
  prompt = prompt,
  model = model, 
  max_tokens = 512,
  temperature = 0.7,
  top_k = 50,
  top_p = 0.7,
  repetition_penalty = 1,
  stop = ["<|im_end|>", "<|im_start|>"]
):
    print(token, end="", flush=True)

questions = [
# Reasoning
"A bat and a ball cost $1.10 in total. The bat costs $1.00 more than the ball. How much does the ball cost?",
"If it takes 5 machines 5 minutes to make 5 widgets, how long would it take 100 machines to make 100 widgets?",
"In a lake, there is a patch of lily pads. Every day, the patch doubles in size. If it takes 48 days for the patch to cover the entire lake, how long would it take for the patch to cover half of the lake?",
"If you're running a race and you pass the person in 2nd place, what place are you in now?",
"Two fathers and two sons go fishing. Each of them catches one fish. So why do they bring home only three fish?",
# Logic
"All roses are flowers. Some flowers fade quickly. Therefore, some roses fade quickly. Is this statement true, false, or uncertain?",
"If some A are B, and some B are C, does it follow that some A are C?",
"No birds are dogs. All dogs are mammals. Therefore, no birds are mammals. Is this logical conclusion correct?",
"If it is true that all men are mortal and Socrates is a man, what can be concluded?",
"Whenever it rains, the grass gets wet. It's not raining. Can we conclude that the grass is not wet?",
# Coding
"Write a function in Python that checks if a number is prime.",
"How would you reverse a string in JavaScript?",
"Write a SQL query to find the second highest salary from a `Employee` table.",
"How can you swap two variables `x` and `y` in Python without using a third variable?",
"Write a recursive function in any programming language to compute the factorial of a number."
]

togetherai_client = openai.OpenAI(
    api_key=os.environ.get("TOGETHER_API_KEY"),
    base_url='https://api.together.xyz',
)

openai_client = openai.OpenAI()

models = [
    { "client": openai_client, "model": "gpt-4", "answers": []}, 
    { "client": openai_client, "model": "gpt-3.5-turbo", "answers": []}, 
    { "client": togetherai_client, "model": "mistralai/Mixtral-8x7B-Instruct-v0.1", "answers": []},
]

for question in questions:
    for model in models: 
        
        completion = model["client"].chat.completions.create(
                messages=[
                    {
                    "role": "user",
                    "content": question,
                    }
                ],
                    model=model["model"],
                    max_tokens=1024
            )
        
        model["answers"].append(
            {
                "question": question, 
                "answer": completion.choices[0].message.content}
            )
        
        print(f"""
              ------------------
              {model["model"]}
              {question}
              {completion.choices[0].message.content}
              ------------------
              """)

INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK"

              ------------------
              gpt-4
              A bat and a ball cost $1.10 in total...

df = pd.DataFrame(models).explode('answers')
df = pd.concat([df.drop(['answers'], axis=1), df['answers'].apply(pd.Series)], axis=1)
df.to_csv("gpt-35-turbo_vs_mixtral-8x7b-instruct-comparison.csv")
df.head()

 ------------------
gpt-4
Two fathers and two sons go fishing. Each of them catches one fish. So why do they bring home only three fish?
Because the group includes a grandfather, his son, and his grandson. So there are only three people (two fathers and two sons) in total. Each catches one fish, which means they bring home three fish in total.
------------------         

------------------
gpt-3.5-turbo
Two fathers and two sons go fishing. Each of them catches one fish. So why do they bring home only three fish?
The group consists of two fathers and their respective sons. Therefore, there are only three individuals in total: the two fathers and one son. Each person caught one fish, resulting in a total of three fish being brought home.
------------------

------------------
mistralai/Mixtral-8x7B-Instruct-v0.1
Two fathers and two sons go fishing. Each of them catches one fish. So why do they bring home only three fish?
Because there are only three people: a grandfather, his son, and his grandson. The son is both a father and a son.
------------------

To find the second highest salary from an `Employee` table, 
you can use the following SQL query:

```sql
SELECT MAX(salary) AS 'Second Highest Salary'
FROM Employee
WHERE salary < (SELECT MAX(salary) FROM Employee);
```

This query first finds the maximum salary in the `Employee` table using 
the subquery `(SELECT MAX(salary) FROM Employee)`. Then, it selects the 
maximum salary that is less than the overall maximum salary, which will 
be the second highest salary.

Note: If there are multiple employees with the same second highest salary, 
this query will return only one of them. If you want to return all employees 
with the second highest salary, you can use the following query:

```sql
SELECT *
FROM Employee
WHERE salary = (
    SELECT DISTINCT salary
    FROM Employee
    WHERE salary < (SELECT MAX(salary) FROM Employee)
    ORDER BY salary DESC
    LIMIT 1
);
```

This query finds the distinct second highest salary and then selects all employees with that salary.

Here is a simple recursive function in Python to compute the factorial of a 
number:

```python
def factorial(n):
    if n == 0:
        return 1
    else:
        return n * factorial(n-1)
```

This function works by repeatedly calling itself with progressively smaller 
values, until it reaches the base case where `n` equals 0. At this point, it 
returns 1, because the factorial of 0 is defined as 1.

For any other value of `n`, the function computes `n` times the factorial 
of `n-1`. For example, the factorial of 5 (denoted as 5!) is calculated 
as follows:

5! = 5 * 4!
    = 5 * (4 * 3!)
    = 5 * (4 * (3 * 2!))
    = 5 * (4 * (3 * (2 * 1!)))
    = 5 * (4 * (3 * (2 * (1 * 0!))))
    = 5 * (4 * (3 * (2 * (1 * 1))))
    = 5 * (4 * (3 * (2 * 2)))
    = 5 * (4 * (3 * 6))
    = 5 * (4 * 24)
    = 5 * 96
    = 480

So if you call `factorial(5)`, the function will return `5 * factorial(4)`, 
which in turn calls `4 * factorial(3)`, and so on, until `factorial(0)` 
returns 1. Then the recursion unwinds, multiplying all the returned values 
together to get the final result.

class TogetherLLM(LLM):
    """
    Together LLM integration.
    
    Attributes:
        model (str): Model endpoint to use.
        together_api_key (str): Together API key.
        temperature (float): Sampling temperature to use.
        max_tokens (int): Maximum number of tokens to generate.
    """
    
    model: str = "mistralai/Mixtral-8x7B-Instruct-v0.1"
    together_api_key: str = os.environ["TOGETHER_API_KEY"]
    temperature: float = 0.7
    max_tokens: int = 512
    @property
    def _llm_type(self) -> str:
        """Return type of LLM."""
        return "together"
    def _call(self, prompt: str, **kwargs: Any) -> str:
            """Call to Together endpoint."""
            try:
                logging.info("Making API call to Together endpoint.")
                return self._make_api_call(prompt)
            except Exception as e:
                logging.error(f"Error in TogetherLLM _call: {e}", exc_info=True)
                raise
    def _make_api_call(self, prompt: str) -> str:
        """Make the API call to the Together endpoint."""
        together.api_key = self.together_api_key
        output = together.Complete.create(
            prompt,
            model=self.model,
            max_tokens=self.max_tokens,
            temperature=self.temperature,
        )
        logging.info("API call successful.")
        return output['output']['choices'][0]['text']

llm = TogetherLLM(
    model = model,
    max_tokens = 256,
    temperature = 0.8
)

prompt_template = "You are a friendly bot, answer the following question: {question}"
prompt = PromptTemplate(
    input_variables=["question"], template=prompt_template
)

chat = LLMChain(llm=llm, prompt=prompt)

chat("Can AI take over developer jobs?")

INFO:root:Making API call to Together endpoint.
INFO:root:API call successful.

{'question': 'Can AI take over developer jobs?',
 'text': '\n\nNo, AI will not take over developer jobs. AI can assist 
developers in various ways, such as automating repetitive tasks, generating 
code, or analyzing data, but it will not replace human developers. 
Developers are needed to design, build, and maintain complex software systems, 
which require creativity, critical thinking, and problem-solving skills 
that AI systems do not possess. Additionally, the field of software 
development is constantly evolving, and new technologies and techniques 
are constantly being developed, which requires developers to stay 
up-to-date and adapt to new challenges.'}

from typing import List

class LLMChain:
    def __init__(self, llm, prompt):
        self.llm = llm
        self.prompt = prompt
        self.history: List[str] = []  # Initialize an empty list to keep track of the conversation history
    def add_to_history(self, user_input: str, bot_response: str):
        self.history.append(f"<human>: {user_input}")
        self.history.append(f"<bot>: {bot_response}")
    def generate_prompt(self, question: str) -> str:
        history_str = "\n".join(self.history)  # Convert the history list into a single string
        return f"{history_str}\n<human>: {question}\n<bot>:"
    def ask(self, question: str) -> str:
        full_prompt = self.generate_prompt(question)
        response = self.llm._call(full_prompt)  # Assuming _call method handles the actual API call
        self.add_to_history(question, response)
        return response

# Usage
llm = TogetherLLM(
    model = model,
    max_tokens = 256,
    temperature = 0.8
)

prompt_template = "You are a friendly bot, answer the following question: {question}"
prompt = PromptTemplate(
    input_variables=["question"], template=prompt_template
)
chat = LLMChain(llm=llm, prompt=prompt)
# Example interaction
response = chat.ask("What is the weather like today?")
print(response)  # Bot's response
# The next call to chat.ask will include the previous interaction in the prompt
response = chat.ask("How can I enjoy such a weather?")
print(response)