Andrew/langchain llms (#602)

evals/completion_fns/langchain_llm.py

@@ -0,0 +1,33 @@
+import importlib
+from typing import Optional
+
+from langchain.llms import BaseLLM
+
+from evals.prompt.base import CompletionPrompt
+from evals.record import record_sampling
+
+
+class LangChainLLMCompletionResult:
+ def __init__(self, response) -> None:
+ self.response = response
+
+ def get_completions(self) -> list[str]:
+ return [self.response.strip()]
+
+
+class LangChainLLMCompletionFn:
+ def __init__(self, llm: str, llm_kwargs: Optional[dict] = {}) -> None:
+ # Import and resolve self.llm to an instance of llm argument here, assuming it's always a subclass of BaseLLM
+ module = importlib.import_module("langchain.llms")
+ LLMClass = getattr(module, llm)
+
+ if issubclass(LLMClass, BaseLLM):
+ self.llm = LLMClass(**llm_kwargs)
+ else:
+ raise ValueError(f"{llm} is not a subclass of BaseLLM")
+
+ def __call__(self, prompt, **kwargs) -> LangChainLLMCompletionResult:
+ prompt = CompletionPrompt(prompt).to_formatted_prompt()
+ response = self.llm(prompt)
+ record_sampling(prompt=prompt, sampled=response)
+ return LangChainLLMCompletionResult(response)

evals/registry/completion_fns/langchain_llms.yaml

@@ -0,0 +1,20 @@
+langchain/llm/gpt-3.5-turbo:
+ class: evals.completion_fns.langchain_llm:LangChainLLMCompletionFn
+ args:
+ llm: OpenAI
+ llm_kwargs:
+ model_name: gpt-3.5-turbo
+
+langchain/llm/text-davinci-003:
+ class: evals.completion_fns.langchain_llm:LangChainLLMCompletionFn
+ args:
+ llm: OpenAI
+ llm_kwargs:
+ model_name: text-davinci-003
+
+langchain/llm/flan-t5-xl:
+ class: evals.completion_fns.langchain_llm:LangChainLLMCompletionFn
+ args:
+ llm: HuggingFaceHub
+ llm_kwargs:
+ repo_id: google/flan-t5-xl