import requests
import urllib.parse
from tqdm import tqdm

def extract_papers(query: str):
    query = urllib.parse.quote(query)
    url = f"https://paperswithcode.com/api/v1/papers/?q={query}"
    response = requests.get(url)
    response = response.json()
    count = response["count"]
    results = []
    results += response["results"]

    num_pages = count // 50
    for page in tqdm(range(2, num_pages)):
        url = f"https://paperswithcode.com/api/v1/papers/?page={page}&q={query}"
        response = requests.get(url)
        response = response.json()
        results += response["results"]
    return results

query = "Large Language Models"

results = extract_papers(query)

print(len(results))
# 7200

from langchain.docstore.document import Document

documents = [
    Document(
        page_content=result["abstract"],
        metadata={
            "id": result["id"] if result["id"] else "",
            "arxiv_id": result["arxiv_id"] if result["arxiv_id"] else "",
            "url_pdf": result["url_pdf"] if result["url_pdf"] else "",
            "title": result["title"] if result["title"] else "",
            "authors": result["authors"] if result["authors"] else "",
            "published": result["published"] if result["published"] else "",
        },
    )
    for result in results
]

text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=1200,
    chunk_overlap=200,
    separators=["."],
)
splits = text_splitter.split_documents(documents)

len(splits)
# 11308

pip install upstash-vector

from upstash_vector import Index

index = Index(
    url="<UPSTASH_URL>", 
    token="<UPSTASH_TOKEN>"
)

from typing import List, Optional, Tuple, Union
from uuid import uuid4
from langchain.docstore.document import Document
from langchain.embeddings.base import Embeddings
from tqdm import tqdm
from upstash_vector import Index


class UpstashVectorStore:
    def __init__(self, index: Index, embeddings: Embeddings):
        self.index = index
        self.embeddings = embeddings

    def delete_vectors(
        self,
        ids: Union[str, List[str]] = None,
        delete_all: bool = None,
    ):
        if delete_all:
            self.index.reset()
        else:
            self.index.delete(ids)

    def add_documents(
        self,
        documents: List[Document],
        ids: Optional[List[str]] = None,
        batch_size: int = 32,
    ):
        texts = []
        metadatas = []
        all_ids = []

        for document in tqdm(documents):
            text = document.page_content
            metadata = document.metadata
            metadata = {"context": text, **metadata}
            texts.append(text)
            metadatas.append(metadata)

            if len(texts) >= batch_size:
                ids = [str(uuid4()) for _ in range(len(texts))]
                all_ids += ids
                embeddings = self.embeddings.embed_documents(texts, batch_size=250)
                self.index.upsert(
                    vectors=zip(ids, embeddings, metadatas),
                )
                texts = []
                metadatas = []

        if len(texts) > 0:
            ids = [str(uuid4()) for _ in range(len(texts))]
            all_ids += ids
            embeddings = self.embeddings.embed_documents(texts)
            self.index.upsert(
                vectors=zip(ids, embeddings, metadatas),
            )

        n = len(all_ids)
        print(f"Successfully indexed {n} dense vectors to Upstash.")
        print(self.index.stats())
        return all_ids

    def similarity_search_with_score(
        self,
        query: str,
        k: int = 4,
    ) -> List[Tuple[Document, float]]:
        query_embedding = self.embeddings.embed_query(query)
        query_results = self.index.query(
            query_embedding,
            top_k=k,
            include_metadata=True,
        )
        output = []
        for query_result in query_results:
            score = query_result.score
            metadata = query_result.metadata
            context = metadata.pop("context")
            doc = Document(
                page_content=context,
                metadata=metadata,
            )
            output.append((doc, score))
        return output

from langchain.embeddings import VertexAIEmbeddings
from upstash_vector import Index

index = Index(
    url="<UPSTASH_URL>",
    token="<UPSTASH_TOKEN>",
)
embeddings = VertexAIEmbeddings(model_name="textembedding-gecko@003")

upstash_vector_store = UpstashVectorStore(index, embeddings)
ids = upstash_vector_store.add_documents(splits, batch_size=25)

def get_context(query, vector_store):
    results = vector_store.similarity_search_with_score(query)
    context = ""

    for doc, score in results:
        context += doc.page_content + "\n===\n"
    return context

def get_prompt(question, context):
    template = """
    Your task is to answer questions by using a given context.

    Don't invent anything that is outside of the context.
    Answer in at least 350 characters.

    %CONTEXT%
    {context}

    %Question%
    {question}

    Hint: Do not copy the context. Use your own words
    
    Answer:
    """
    prompt = template.format(question=question, context=context)
    return prompt

query = (
    "What are the problems behind the Retrieval Augmented Generation (RAG) framework?"
)

context = get_context(query, upstash_vector_store)
prompt = get_prompt(query, context)

Your task is to answer questions by using a given context.

Don't invent anything that is outside of the context.
Answer in at least 350 characters.

%CONTEXT%

Retrieval-Augmented Generation (RAG) is a promising approach for
mitigating the hallucination of large language models (LLMs). 
However, existing research lacks rigorous evaluation of the impact 
of retrieval-augmented generation on different large language models,
which make it challenging to identify the potential bottlenecks in the 
capabilities of RAG for different LLMs. In this paper, we systematically 
investigate the impact of Retrieval-Augmented Generation on large language 
models. We analyze the performance of different large language models in 
4 fundamental abilities required for RAG, including noise robustness, 
negative rejection, information integration, and counterfactual robustness. 
To this end, we establish Retrieval-Augmented Generation Benchmark (RGB), a 
new corpus for RAG evaluation in both English and Chinese. RGB divides the 
instances within the benchmark into 4 separate testbeds based on the 
aforementioned fundamental abilities required to resolve the case. 
Then we evaluate 6 representative LLMs on RGB to diagnose the challenges 
of current LLMs when applying RAG
===
Despite their remarkable capabilities, large language models (LLMs) often 
produce responses containing factual inaccuracies due to their sole reliance
 on the parametric knowledge they encapsulate. Retrieval-Augmented Generation 
(RAG), an ad hoc approach that augments LMs with retrieval of relevant 
knowledge, decreases such issues. However, indiscriminately retrieving and 
incorporating a fixed number of retrieved passages, regardless of whether 
retrieval is necessary, or passages are relevant, diminishes LM versatility 
or can lead to unhelpful response generation. We introduce a new framework 
called Self-Reflective Retrieval-Augmented Generation (Self-RAG) 
that enhances an LM's quality and factuality through retrieval and 
self-reflection. Our framework trains a single arbitrary LM that adaptively 
retrieves passages on-demand, and generates and reflects on retrieved 
passages and its own generations using special tokens, called reflection 
tokens. Generating reflection tokens makes the LM controllable during the 
inference phase, enabling it to tailor its behavior to diverse task 
requirements
...

from langchain.chat_models import AzureChatOpenAI

llm = AzureChatOpenAI(
    azure_deployment="<AZURE_DEPLOYMENT>",
    model="<MODEL_NAME>",
)

answer = llm.predict(prompt)

The Retrieval Augmented Generation (RAG) framework can suffer from 
problems such as indiscriminate retrieval and incorporation of passages 
that may not be necessary or relevant, leading to unhelpful response 
generation. Additionally, existing research lacks rigorous evaluation 
of the impact of RAG on different Large Language Models (LLMs), 
making it difficult to identify potential bottlenecks in the capabilities 
of RAG for different LLMs. To address these issues, researchers have 
proposed the Self-Reflective Retrieval-Augmented Generation (Self-RAG) 
framework, which enhances an LM's quality and factuality through 
retrieval and self-reflection. Another challenge with LLMs is their 
forgetfulness, as they do not improve over time or acquire new knowledge 
like humans do. To address this, researchers have explored the use of RAG 
to improve problem-solving performance and proposed the ARM-RAG system, 
which learns from its successes without requiring high training costs.