!pip install -q gradio
!pip install openai
!pip install tiktoken

OPEN_API_KEY = your_api_key

# Import the necessary libraries
import openai  # Import OpenAI library for making API requests
import gradio as gr  # Import Gradio for creating a user interface

# Set the OpenAI API key - Replace OPENAI_API_KEY with your actual API key
openai.api_key = OPENAI_API_KEY

# Define a function called get_completion
def get_completion(message, history):
    history_openai_format = []

    # Iterate through the conversation history (a list of tuples with human and assistant messages)
    for human, assistant in history:
        # Add the user's message to the formatted history with the role "user"
        history_openai_format.append({"role": "user", "content": human })

        # Add the assistant's response to the formatted history with the role "assistant"
        history_openai_format.append({"role": "assistant", "content": assistant})

    # Add the current user's message to the formatted history
    history_openai_format.append({"role": "user", "content": message})

    # Make an API request to OpenAI's ChatCompletion model
    response = openai.ChatCompletion.create(
        model='gpt-3.5-turbo',  # Specify the model to use
        messages=history_openai_format,  # Provide the formatted conversation history
        temperature=0,  # Set temperature to 0 for more focused and deterministic responses
    )

    # Extract and return the content of the model's response
    return response.choices[0].message["content"]

gr.ChatInterface(get_completion).queue().launch()

def get_completion_with_streaming(message, history):
    history_openai_format = []

    # Iterate through the conversation history (a list of tuples with human and assistant messages)
    for human, assistant in history:
        # Add the user's message to the formatted history with the role "user"
        history_openai_format.append({"role": "user", "content": human })

        # Add the assistant's response to the formatted history with the role "assistant"
        history_openai_format.append({"role": "assistant", "content": assistant})

    # Add the current user's message to the formatted history
    history_openai_format.append({"role": "user", "content": message})

    # Make an API request to OpenAI's ChatCompletion model with streaming enabled
    response = openai.ChatCompletion.create(
        model='gpt-3.5-turbo',  # Specify the model to use
        messages=history_openai_format,  # Provide the formatted conversation history
        temperature=1.0,  # Set temperature to control the randomness of responses
        stream=True  # Enable streaming mode for partial responses
    )

    # Initialize a variable to hold the partial message
    partial_message = ""

    # Iterate through the response chunks
    for chunk in response:
        # Check if the chunk contains content
        if len(chunk['choices'][0]['delta']) != 0:
            # Append the content of the chunk to the partial message
            partial_message = partial_message + chunk['choices'][0]['delta']['content']

            # Yield the partial message, allowing for streaming responses
            yield partial_message

# Import necessary libraries and modules
import gradio as gr
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, StoppingCriteria, StoppingCriteriaList, TextIteratorStreamer
from threading import Thread

# Move the model to the GPU (cuda:0)
model = model.to('cuda:0')

# Define a custom StoppingCriteria class for text generation
class StopOnTokens(StoppingCriteria):
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
        # Define stop tokens (e.g., [29, 0]) that determine when to stop text generation
        stop_ids = [29, 0]
        for stop_id in stop_ids:
            if input_ids[0][-1] == stop_id:
                return True
        return False

# Define a function called predict for text generation
def predict(message, history):
    # Combine the user's message and conversation history
    history_transformer_format = history + [[message, ""]]

    # Create an instance of the custom StoppingCriteria class
    stop = StopOnTokens()

    # Prepare the conversation history in a specific format
    messages = "".join(["".join(["\n<human>:"+item[0], "\n<bot>:"+item[1]])
                for item in history_transformer_format])

    # Tokenize the formatted conversation history and move it to the GPU
    model_inputs = tokenizer([messages], return_tensors="pt").to("cuda")

    # Create a TextIteratorStreamer to iterate over generated tokens
    streamer = TextIteratorStreamer(tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True)

    # Define text generation parameters
    generate_kwargs = dict(
        model_inputs,
        streamer=streamer,
        max_new_tokens=1024,
        do_sample=True,
        top_p=0.95,
        top_k=1000,
        temperature=1.0,
        num_beams=1,
        stopping_criteria=StoppingCriteriaList([stop])
    )

    # Start text generation in a separate thread
    t = Thread(target=model.generate, kwargs=generate_kwargs)
    t.start()

    partial_message = ""
    
    # Iterate over generated tokens and yield partial messages
    for new_token in streamer:
        if new_token != '<':
            partial_message += new_token
            yield partial_message

gr.ChatInterface(predict).queue().launch()

gr.ChatInterface(
    predict,
    chatbot=gr.Chatbot(height=300),
    textbox=gr.Textbox(placeholder="Send a message", container=False, scale=7),
    title="Chat with Finance Mistral 7B",
    description="Ask me any questions on finance",
    theme="soft",
    examples=["Will capital gains affect my tax bracket?", "What are the common income tax deductions used by rich salaried households?"],
    cache_examples=True,
    retry_btn=None,
    undo_btn="Delete Previous",
    clear_btn="Clear",
).queue().launch()