From d1975d37ac4fb204c3e7873a98dd42951f670f82 Mon Sep 17 00:00:00 2001 From: Jerry Liu Date: Tue, 18 Jun 2024 09:01:24 -0700 Subject: [PATCH 1/5] cr --- dspy/predict/llamaindex.py | 304 ++++++ examples/llamaindex/dspy_llamaindex_rag.ipynb | 985 ++++++++++++++++++ 2 files changed, 1289 insertions(+) create mode 100644 dspy/predict/llamaindex.py create mode 100644 examples/llamaindex/dspy_llamaindex_rag.ipynb diff --git a/dspy/predict/llamaindex.py b/dspy/predict/llamaindex.py new file mode 100644 index 000000000..407124248 --- /dev/null +++ b/dspy/predict/llamaindex.py @@ -0,0 +1,304 @@ +from llama_index.core.prompts import BasePromptTemplate +from dspy import Predict +import dspy +from abc import abstractmethod +from typing import Any, Optional, List, Dict, Callable +from llama_index.core.base.llms.base import BaseLLM +from llama_index.core.base.llms.types import ChatMessage +from llama_index.core.base.llms.generic_utils import ( + prompt_to_messages, +) +from llama_index.core.base.query_pipeline.query import QueryComponent, InputKeys, OutputKeys +from llama_index.core.query_pipeline import QueryPipeline +from dspy.signatures.signature import ensure_signature, signature_to_template, infer_prefix, make_signature +from dspy.signatures.field import InputField, OutputField +from dspy.primitives import ProgramMeta +import dsp +from copy import deepcopy +import re +from llama_index.core.callbacks.base import CallbackManager +from llama_index.core.bridge.pydantic import BaseModel, create_model +from llama_index.core.prompts import PromptTemplate + + + + +def get_formatted_template(predict_module: Predict, kwargs: Dict[str, Any]) -> str: + """Get formatted template from predict module.""" + # Extract the three privileged keyword arguments. + signature = ensure_signature(predict_module.signature) + demos = predict_module.demos + + # All of the other kwargs are presumed to fit a prefix of the signature. + # That is, they are input variables for the bottom most generation, so + # we place them inside the input - x - together with the demos. + x = dsp.Example(demos=demos, **kwargs) + + # Switch to legacy format for dsp.generate + template = signature_to_template(signature) + + return template(x) + + +def replace_placeholder(text: str) -> str: + # Use a regular expression to find and replace ${...} with ${{...}} + return re.sub(r'\$\{([^\{\}]*)\}', r'${{\1}}', text) + + +def _input_keys_from_template(template: dsp.Template) -> InputKeys: + """Get input keys from template.""" + # get only fields that are marked OldInputField and NOT OldOutputField + # template_vars = list(template.kwargs.keys()) + return [ + k for k, v in template.kwargs.items() if isinstance(v, dspy.signatures.OldInputField) + ] + +def _output_keys_from_template(template: dsp.Template) -> InputKeys: + """Get input keys from template.""" + # get only fields that are marked OldInputField and NOT OldOutputField + # template_vars = list(template.kwargs.keys()) + return [ + k for k, v in template.kwargs.items() if isinstance(v, dspy.signatures.OldOutputField) + ] + + +class DSPyPromptTemplate(BasePromptTemplate): + """A prompt template for DSPy. + + Takes in a predict module from DSPy (whether unoptimized or optimized), + and extracts the relevant prompt template from it given the input. + + """ + + predict_module: Predict + + def __init__( + self, + predict_module: Predict, + metadata: Optional[Dict[str, Any]] = None, + template_var_mappings: Optional[Dict[str, Any]] = None, + function_mappings: Optional[Dict[str, Callable]] = None, + **kwargs: Any + ) -> None: + template = signature_to_template(predict_module.signature) + template_vars = _input_keys_from_template(template) + # print(f"TEMPLATE VARS: {template_vars}") + # raise Exception + + super().__init__( + predict_module=predict_module, + metadata=metadata or {}, + template_vars=template_vars, + kwargs=kwargs, + template_var_mappings=template_var_mappings, + function_mappings=function_mappings, + ) + + def partial_format(self, **kwargs: Any) -> "BasePromptTemplate": + """Returns a new prompt template with the provided kwargs.""" + # NOTE: this is a copy of the implementation in `PromptTemplate` + output_parser = self.output_parser + self.output_parser = None + + # get function and fixed kwargs, and add that to a copy + # of the current prompt object + prompt = deepcopy(self) + prompt.kwargs.update(kwargs) + + # NOTE: put the output parser back + prompt.output_parser = output_parser + self.output_parser = output_parser + return prompt + + def format(self, llm: Optional[BaseLLM] = None, **kwargs: Any) -> str: + """Formats the prompt template.""" + mapped_kwargs = self._map_all_vars(kwargs) + return get_formatted_template(self.predict_module, mapped_kwargs) + + def format_messages( + self, llm: Optional[BaseLLM] = None, **kwargs: Any + ) -> List[ChatMessage]: + """Formats the prompt template into chat messages.""" + del llm # unused + prompt = self.format(**kwargs) + return prompt_to_messages(prompt) + + def get_template(self, llm: Optional[BaseLLM] = None) -> str: + """Get template.""" + # get kwarg templates + kwarg_tmpl_map = {k: f"{{k}}" for k in self.template_vars} + + # get "raw" template with all the values filled in with {var_name} + template0 = get_formatted_template(self.predict_module, kwarg_tmpl_map) + # HACK: there are special 'format' variables of the form ${var_name} that are meant to + # prompt the LLM, but we do NOT want to replace with actual prompt variable values. + # Replace those with double brackets + template1 = replace_placeholder(template0) + + return template1 + + +# copied from langchain.py +class Template2Signature(dspy.Signature): + """You are a processor for prompts. I will give you a prompt template (Python f-string) for an arbitrary task for other LMs. +Your job is to prepare three modular pieces: (i) any essential task instructions or guidelines, (ii) a list of variable names for inputs, (iv) the variable name for output.""" + + template = dspy.InputField(format=lambda x: f"```\n\n{x.strip()}\n\n```\n\nLet's now prepare three modular pieces.") + essential_instructions = dspy.OutputField() + input_keys = dspy.OutputField(desc='comma-separated list of valid variable names') + output_key = dspy.OutputField(desc='a valid variable name') + + +def build_signature(prompt: PromptTemplate) -> dspy.Signature: + """Attempt to build signature from prompt.""" + # TODO: allow plugging in any llamaindex LLM + gpt4T = dspy.OpenAI(model='gpt-4-1106-preview', max_tokens=4000, model_type='chat') + + with dspy.context(lm=gpt4T): + parts = dspy.Predict(Template2Signature)(template=prompt.template) + + inputs = {k.strip(): InputField() for k in parts.input_keys.split(',')} + outputs = {k.strip(): OutputField() for k in parts.output_key.split(',')} + + # dynamically create a pydantic model that subclasses dspy.Signature + fields = { + k: (str, v) for k, v in {**inputs, **outputs}.items() + } + signature = make_signature(fields, parts.essential_instructions) + return signature + + +class DSPyComponent(QueryComponent): + """DSPy Query Component. + + Can take in either a predict module directly. + TODO: add ability to translate from an existing prompt template / LLM. + + """ + predict_module: dspy.Predict + predict_template: dsp.Template + + class Config: + arbitrary_types_allowed = True + + def __init__( + self, + predict_module: dspy.Predict, + ) -> None: + """Initialize.""" + return super().__init__( + predict_module=predict_module, + predict_template=signature_to_template(predict_module.signature), + ) + # self.predict_module = predict_module + # self.predict_template = signature_to_template(predict_module.signature) + # super().__init__() + + @classmethod + def from_prompt( + cls, + prompt_template: BasePromptTemplate, + # llm: BaseLLM, + ) -> "DSPyComponent": + """Initialize from prompt template. + + LLM is a TODO - currently use DSPy LLM classes. + + """ + signature = build_signature(prompt_template) + predict_module = Predict(signature) + return cls(predict_module=predict_module) + + def set_callback_manager(self, callback_manager: CallbackManager) -> None: + """Set callback manager.""" + # TODO: implement + pass + + def _validate_component_inputs(self, input: Dict[str, Any]) -> Dict[str, Any]: + """Validate component inputs during run_component.""" + return input + + def _run_component(self, **kwargs: Any) -> Dict: + """Run component.""" + prediction = self.predict_module(**kwargs) + return { + k: getattr(prediction, k) for k in self.output_keys.required_keys + } + + async def _arun_component(self, **kwargs: Any) -> Any: + """Run component (async).""" + # TODO: no async predict module yet + return self._run_component(**kwargs) + + # def forward(self, *args: Any, **kwargs: Any) -> Any: + # """Forward.""" + # return self._run_component(*args, **kwargs)[self.output_key] + + @property + def input_keys(self) -> InputKeys: + """Input keys.""" + input_keys = _input_keys_from_template(self.predict_template) + return InputKeys.from_keys(input_keys) + + @property + def output_keys(self) -> OutputKeys: + """Output keys.""" + output_keys = _output_keys_from_template(self.predict_template) + return OutputKeys.from_keys(output_keys) + + +class LlamaIndexModuleMeta(ProgramMeta, type(BaseModel)): + pass + + +# class LlamaIndexModule(dspy.Module, QueryComponent, metaclass=LlamaIndexModuleMeta): +class LlamaIndexModule(dspy.Module): + """A module for LlamaIndex. + + Wraps a QueryPipeline and exposes it as a dspy module for optimization. + + """ + + class Config: + arbitrary_types_allowed = True + + def __init__(self, query_pipeline: QueryPipeline) -> None: + """Initialize.""" + super().__init__() + self.query_pipeline = query_pipeline + self.predict_modules = [] + for module in query_pipeline.module_dict.values(): + if isinstance(module, DSPyComponent): + self.predict_modules.append(module.predict_module) + + + def forward(self, **kwargs: Any) -> Dict[str, Any]: + """Forward.""" + output_dict = self.query_pipeline.run(**kwargs, return_values_direct=False) + return dspy.Prediction(**output_dict) + + # def set_callback_manager(self, callback_manager: CallbackManager) -> None: + # """Set callback manager.""" + # self.query_pipeline.set_callback_manager(callback_manager) + + # def _validate_component_inputs(self, input: Dict[str, Any]) -> Dict[str, Any]: + # """Validate component inputs during run_component.""" + # return self.query_pipeline.validate_component_inputs(input) + + # def _run_component(self, **kwargs: Any) -> Dict: + # """Run component.""" + # return self.query_pipeline.run_component(**kwargs) + + # async def _arun_component(self, **kwargs: Any) -> Any: + # """Run component (async).""" + # return await self.query_pipeline.arun_component(**kwargs) + + # @property + # def input_keys(self) -> InputKeys: + # """Input keys.""" + # return self.query_pipeline.input_keys + + # @property + # def output_keys(self) -> OutputKeys: + # """Output keys.""" + # return self.query_pipeline.output_keys \ No newline at end of file diff --git a/examples/llamaindex/dspy_llamaindex_rag.ipynb b/examples/llamaindex/dspy_llamaindex_rag.ipynb new file mode 100644 index 000000000..630efaa41 --- /dev/null +++ b/examples/llamaindex/dspy_llamaindex_rag.ipynb @@ -0,0 +1,985 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "849dbd89-ce04-4a18-84fb-c19f3db5504a", + "metadata": {}, + "source": [ + "# Building optimized RAG with LlamaIndex + DSPy\n", + "\n", + "This notebook provides a comprehensive overview of LlamaIndex + DSPy integrations.\n", + "\n", + "We show **three** core integrations:\n", + "1. **Build and optimize Query Pipelines with DSPy predictors**: The first section shows you how to write DSPy code to define signatures for LLM inputs/outputs. Then port over these components to overall workflows within LlamaIndex Query pipelines, and then end-to-end optimize the entire system.\n", + "\n", + "2. **Build and optimize Query Pipelines with Existing Prompts**: Instead of writing DSPy signatures, you can just define a LlamaIndex prompt template, and our converter will auto-optimize it for you.\n", + "\n", + "3. **Port over DSPy-Optimized Prompts to any LlamaIndex Module**: Possible through our `DSPyPromptTemplate` - translate an optimized prompt through DSPy into any module that requires prompts in LlamaIndex." + ] + }, + { + "cell_type": "markdown", + "id": "3d223313-af5b-4155-8896-c24aa4cb6925", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "Define the LLM setting for DSPy (note: this is separate from using the LlamaIndex LLMs), and also the answer signature." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "c5c50d5e-7046-40c5-bf3e-a8d2a2a2c6f8", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import dspy\n", + "\n", + "turbo = dspy.OpenAI(model='gpt-3.5-turbo')\n", + "dspy.settings.configure(lm=turbo)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "ac68be4f-36b1-4054-99dd-707ce42f61a4", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import dspy\n", + "\n", + "class GenerateAnswer(dspy.Signature):\n", + " \"\"\"Answer questions with short factoid answers.\"\"\"\n", + "\n", + " context_str = dspy.InputField(desc=\"contains relevant facts\")\n", + " query_str = dspy.InputField()\n", + " answer = dspy.OutputField(desc=\"often between 1 and 5 words\")" + ] + }, + { + "cell_type": "markdown", + "id": "eec2a981-2ee1-497a-b675-15e1029183f6", + "metadata": {}, + "source": [ + "## [Part 1] Build and Optimize a Query Pipeline with DSPy Modules\n", + "\n", + "Use our DSPy query components to plugin DSPy prompts/LLMs, stitch together with our query pipeline abstraction.\n", + "\n", + "Any query pipeline can be plugged into our `LlamaIndexModule`. We can then let DSPy optimize the entire thing e2e." + ] + }, + { + "cell_type": "markdown", + "id": "113f0637-993f-4bef-bd8a-1122951cf7f8", + "metadata": {}, + "source": [ + "#### Load Data, Build Index" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "55c9d434-c8f6-403e-9499-36059eb09907", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--2024-06-17 23:54:09-- https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt\n", + "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 2606:50c0:8001::154, 2606:50c0:8002::154, 2606:50c0:8000::154, ...\n", + "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|2606:50c0:8001::154|:443... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 75042 (73K) [text/plain]\n", + "Saving to: ‘paul_graham_essay.txt’\n", + "\n", + "paul_graham_essay.t 100%[===================>] 73.28K --.-KB/s in 0.01s \n", + "\n", + "2024-06-17 23:54:10 (7.48 MB/s) - ‘paul_graham_essay.txt’ saved [75042/75042]\n", + "\n" + ] + } + ], + "source": [ + "# port it over to another index (paul graham example) \n", + "\n", + "!wget https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt -O paul_graham_essay.txt" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "367a7985-2f89-47be-b2af-14fef2e845c9", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from llama_index.core import SimpleDirectoryReader, VectorStoreIndex\n", + "\n", + "reader = SimpleDirectoryReader(input_files=[\"paul_graham_essay.txt\"])\n", + "docs = reader.load_data()\n", + "\n", + "index = VectorStoreIndex.from_documents(docs)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "c9a68920-6517-465b-8c0c-4c6e0b8390a3", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "retriever = index.as_retriever(similarity_top_k=2)" + ] + }, + { + "cell_type": "markdown", + "id": "a8d43c6c-9dee-4cbc-b39d-5d8ad617b6ea", + "metadata": {}, + "source": [ + "#### Build Query Pipeline\n", + "\n", + "Replace the synthesis piece with the DSPy component (make sure GenerateAnswer matches signature of inputs/outputs)." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ac7df48b-9369-4f17-8542-0f8afd9e621e", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from llama_index.core.query_pipeline import QueryPipeline as QP, InputComponent, FnComponent\n", + "from llamaindex import DSPyComponent, LlamaIndexModule\n", + "\n", + "dspy_component = DSPyComponent(\n", + " dspy.ChainOfThought(GenerateAnswer)\n", + ")\n", + "\n", + "retriever_post = FnComponent(\n", + " lambda contexts: \"\\n\\n\".join([n.get_content() for n in contexts])\n", + ")\n", + "\n", + "\n", + "p = QP(verbose=True)\n", + "p.add_modules(\n", + " {\n", + " \"input\": InputComponent(),\n", + " \"retriever\": retriever,\n", + " \"retriever_post\": retriever_post,\n", + " \"synthesizer\": dspy_component,\n", + " }\n", + ")\n", + "p.add_link(\"input\", \"retriever\")\n", + "p.add_link(\"retriever\", \"retriever_post\")\n", + "p.add_link(\"input\", \"synthesizer\", dest_key=\"query_str\")\n", + "p.add_link(\"retriever_post\", \"synthesizer\", dest_key=\"context_str\")\n", + "\n", + "\n", + "dspy_qp = LlamaIndexModule(p)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "b457f1db-3315-476c-8673-bf19ad6e1657", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;3;38;2;155;135;227m> Running module input with input: \n", + "query_str: what did the author do in YC\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module retriever with input: \n", + "input: what did the author do in YC\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module retriever_post with input: \n", + "contexts: [NodeWithScore(node=TextNode(id_='49a290f5-5f29-413c-97e7-9fdf15169cf4', embedding=None, metadata={'file_path': 'paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain'...\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module synthesizer with input: \n", + "query_str: what did the author do in YC\n", + "context_str: YC was different from other kinds of work I've done. Instead of deciding for myself what to work on, the problems came to me. Every 6 months there was a new batch of startups, and their problems, what...\n", + "\n", + "\u001b[0m" + ] + } + ], + "source": [ + "output = dspy_qp(query_str=\"what did the author do in YC\")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "7b78501f-e281-4e68-bdb6-1eeca6bf2464", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Prediction(\n", + " answer='Worked with startups, funded them.'\n", + ")" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output" + ] + }, + { + "cell_type": "markdown", + "id": "691d5d86-1ce0-46fd-9fbc-eca68043c935", + "metadata": {}, + "source": [ + "#### Optimize Query Pipeline\n", + "\n", + "Let's try optimizing the query pipeline with few-shot examples.\n", + "\n", + "We define a toy dataset with two examples. We then use our `SemanticSimilarityEvaluator` to define a custom eval function to pass to the DSPy teleprompter.\n", + "- Because our passing threshold is set to very low, every example should pass with a reasonable LLM. \n", + "- What this practically means is that all training examples will be added as few-shot examples to the prompt." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "ec3a783c-2025-4b1c-9f5b-cdcde7211ace", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from dspy import Example\n", + "\n", + "train_examples = [\n", + " Example(query_str=\"What did the author do growing up?\", answer=\"The author wrote short stories and also worked on programming.\"),\n", + " Example(query_str=\"What did the author do during his time at YC?\", answer=\"organizing a Summer Founders Program, funding startups, writing essays, working on a new version of Arc, creating Hacker News, and developing internal software for YC\")\n", + "]\n", + "\n", + "train_examples = [t.with_inputs(\"query_str\") for t in train_examples]" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "2e43a18c-5569-4947-a02d-08f52331b134", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import nest_asyncio\n", + "nest_asyncio.apply()" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "502cc403-8ebc-4bbf-be7c-c15385909b7a", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/2 [00:00 Running module input with input: \n", + "query_str: What did the author do growing up?\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module retriever with input: \n", + "input: What did the author do growing up?\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module retriever_post with input: \n", + "contexts: [NodeWithScore(node=TextNode(id_='20a73a69-f604-450e-b07d-cace28b471a5', embedding=None, metadata={'file_path': 'paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain'...\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module synthesizer with input: \n", + "query_str: What did the author do growing up?\n", + "context_str: What I Worked On\n", + "\n", + "February 2021\n", + "\n", + "Before college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to...\n", + "\n", + "\u001b[0m" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 50%|████████████████████████████████ | 1/2 [00:00<00:00, 1.85it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;3;38;2;155;135;227m> Running module input with input: \n", + "query_str: What did the author do during his time at YC?\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module retriever with input: \n", + "input: What did the author do during his time at YC?\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module retriever_post with input: \n", + "contexts: [NodeWithScore(node=TextNode(id_='49a290f5-5f29-413c-97e7-9fdf15169cf4', embedding=None, metadata={'file_path': 'paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain'...\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module synthesizer with input: \n", + "query_str: What did the author do during his time at YC?\n", + "context_str: YC was different from other kinds of work I've done. Instead of deciding for myself what to work on, the problems came to me. Every 6 months there was a new batch of startups, and their problems, what...\n", + "\n", + "\u001b[0m" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 2.07it/s]\n" + ] + } + ], + "source": [ + "from dspy.teleprompt import BootstrapFewShot\n", + "from llama_index.core.evaluation import SemanticSimilarityEvaluator\n", + "\n", + "evaluator = SemanticSimilarityEvaluator(similarity_threshold=0.5)\n", + "\n", + "# Validation logic: check that the predicted answer is correct.\n", + "# Also check that the retrieved context does actually contain that answer.\n", + "def validate_context_and_answer(example, pred, trace=None):\n", + " result = evaluator.evaluate(response=pred.answer, reference=example.answer)\n", + " return result.passing\n", + "\n", + "# Set up a basic teleprompter, which will compile our RAG program.\n", + "teleprompter = BootstrapFewShot(max_labeled_demos=0, metric=validate_context_and_answer)\n", + "\n", + "# Compile!\n", + "compiled_dspy_qp = teleprompter.compile(dspy_qp, trainset=train_examples)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "3a147c3e-febe-43fa-bca5-152432d3662b", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;3;38;2;155;135;227m> Running module input with input: \n", + "query_str: How did PG meet Jessica Livingston?\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module retriever with input: \n", + "input: How did PG meet Jessica Livingston?\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module retriever_post with input: \n", + "contexts: [NodeWithScore(node=TextNode(id_='465a8143-6740-4bbd-8e6c-bb5eba4bb5a3', embedding=None, metadata={'file_path': 'paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain'...\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module synthesizer with input: \n", + "query_str: How did PG meet Jessica Livingston?\n", + "context_str: Over the next several years I wrote lots of essays about all kinds of different topics. O'Reilly reprinted a collection of them as a book, called Hackers & Painters after one of the essays in it. I al...\n", + "\n", + "\u001b[0m" + ] + }, + { + "data": { + "text/plain": [ + "Prediction(\n", + " answer='Met at a party in 2003.'\n", + ")" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# test this out \n", + "compiled_dspy_qp(query_str=\"How did PG meet Jessica Livingston?\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "96a2ba4e-6677-44e7-a474-db382ad1a56a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# [optional]: inspect history\n", + "turbo.inspect_history(n=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "ab29598c-2bf5-4472-ab13-43772e678a9f", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# from dspy.signatures.signature import signature_to_template\n", + "# from llamaindex import DSPyPromptTemplate\n", + "\n", + "# # tmpl = signature_to_template(compiled_dspy_qp.query_pipeline.module_dict[\"synthesizer\"].predict_module.signature)\n", + "\n", + "# # qa_prompt_tmpl = DSPyPromptTemplate(RAG().generate_answer)\n", + "# qa_prompt_tmpl = DSPyPromptTemplate(compiled_dspy_qp.query_pipeline.module_dict[\"synthesizer\"].predict_module)\n", + "# print(qa_prompt_tmpl.format(query_str=\"hello?\", context_str=\"this is my context\"))" + ] + }, + { + "cell_type": "markdown", + "id": "9895ceff-5aeb-4285-8ede-99bb35a50a45", + "metadata": {}, + "source": [ + "## [Part 2] Build and Optimize Query Pipelines with Existing Prompts\n", + "\n", + "Build a query pipeline similar to the previous section. But instead of directly using DSPy signatures/predictors, we can build DSPyComponent modules from LlamaIndex prompts directly. \n", + "\n", + "This allows you to write any LlamaIndex prompt and trust that it'll be optimized in DSPy." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "8efff3a9-77d9-4a89-a0f7-207d63e73ab0", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from llama_index.core.prompts import PromptTemplate\n", + "\n", + "# let's try a fun prompt that writes in Shakespeare! \n", + "qa_prompt_template = PromptTemplate(\"\"\"\\\n", + "Context information is below.\n", + "---------------------\n", + "{context_str}\n", + "---------------------\n", + "Given the context information and not prior knowledge, \\\n", + "answer the query.\n", + "\n", + "Write in the style of a Shakespearean sonnet.\n", + "\n", + "Query: {query_str}\n", + "Answer: \n", + "\"\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "a84aa7b2-b9b8-4740-b862-9073470c31c2", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from llama_index.core.query_pipeline import QueryPipeline as QP, InputComponent, FnComponent\n", + "from llamaindex import DSPyComponent, LlamaIndexModule\n", + "\n", + "dspy_component = DSPyComponent.from_prompt(qa_prompt_template)\n", + "\n", + "retriever_post = FnComponent(\n", + " lambda contexts: \"\\n\\n\".join([n.get_content() for n in contexts])\n", + ")\n", + "\n", + "\n", + "p = QP(verbose=True)\n", + "p.add_modules(\n", + " {\n", + " \"input\": InputComponent(),\n", + " \"retriever\": retriever,\n", + " \"retriever_post\": retriever_post,\n", + " \"synthesizer\": dspy_component,\n", + " }\n", + ")\n", + "p.add_link(\"input\", \"retriever\")\n", + "p.add_link(\"retriever\", \"retriever_post\")\n", + "p.add_link(\"input\", \"synthesizer\", dest_key=\"query_str\")\n", + "p.add_link(\"retriever_post\", \"synthesizer\", dest_key=\"context_str\")\n", + "\n", + "\n", + "dspy_qp = LlamaIndexModule(p)" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "e23b7662-cb87-4906-bbaf-e2ffd74e20d8", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "StringSignature(context_str, query_str -> sonnet_answer\n", + " instructions='Essential Instructions: Provide an answer to the query based solely on the context information provided. The response should be written in the style of a Shakespearean sonnet, which typically consists of 14 lines written in iambic pentameter, with a rhyme scheme of ABABCDCDEFEFGG.'\n", + " context_str = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context Str:', 'desc': '${context_str}'})\n", + " query_str = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Query Str:', 'desc': '${query_str}'})\n", + " sonnet_answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Sonnet Answer:', 'desc': '${sonnet_answer}'})\n", + ")" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# check the inferred signature\n", + "dspy_component.predict_module.signature" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "6044855f-cdd2-43f0-9d2e-e7ba64a28e27", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/2 [00:00 Running module input with input: \n", + "query_str: What did the author do growing up?\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module retriever with input: \n", + "input: What did the author do growing up?\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module retriever_post with input: \n", + "contexts: [NodeWithScore(node=TextNode(id_='20a73a69-f604-450e-b07d-cace28b471a5', embedding=None, metadata={'file_path': 'paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain'...\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module synthesizer with input: \n", + "query_str: What did the author do growing up?\n", + "context_str: What I Worked On\n", + "\n", + "February 2021\n", + "\n", + "Before college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to...\n", + "\n", + "\u001b[0m" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 50%|████████████████████████████████ | 1/2 [00:00<00:00, 1.94it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;3;38;2;155;135;227m> Running module input with input: \n", + "query_str: What did the author do during his time at YC?\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module retriever with input: \n", + "input: What did the author do during his time at YC?\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module retriever_post with input: \n", + "contexts: [NodeWithScore(node=TextNode(id_='49a290f5-5f29-413c-97e7-9fdf15169cf4', embedding=None, metadata={'file_path': 'paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain'...\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module synthesizer with input: \n", + "query_str: What did the author do during his time at YC?\n", + "context_str: YC was different from other kinds of work I've done. Instead of deciding for myself what to work on, the problems came to me. Every 6 months there was a new batch of startups, and their problems, what...\n", + "\n", + "\u001b[0m" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████| 2/2 [00:01<00:00, 1.95it/s]\n" + ] + } + ], + "source": [ + "from dspy.teleprompt import BootstrapFewShot\n", + "from llama_index.core.evaluation import SemanticSimilarityEvaluator\n", + "from dspy import Example\n", + "\n", + "output_key = \"sonnet_answer\"\n", + "train_example_dicts = [\n", + " {\"query_str\": \"What did the author do growing up?\", output_key: \"The author wrote short stories and also worked on programming.\"},\n", + " {\"query_str\": \"What did the author do during his time at YC?\", output_key: \"organizing a Summer Founders Program, funding startups, writing essays, working on a new version of Arc, creating Hacker News, and developing internal software for YC\"}\n", + "]\n", + "train_examples = [Example(**t).with_inputs(\"query_str\") for t in train_example_dicts]\n", + "\n", + "evaluator = SemanticSimilarityEvaluator(similarity_threshold=0.5)\n", + "# Validation logic: check that the predicted answer is correct.\n", + "# Also check that the retrieved context does actually contain that answer.\n", + "def validate_context_and_answer(example, pred, trace=None):\n", + " result = evaluator.evaluate(response=getattr(pred, output_key), reference=getattr(example, output_key))\n", + " return result.passing\n", + "\n", + "# Set up a basic teleprompter, which will compile our RAG program.\n", + "teleprompter = BootstrapFewShot(max_labeled_demos=0, metric=validate_context_and_answer)\n", + "\n", + "# Compile!\n", + "compiled_dspy_qp = teleprompter.compile(dspy_qp, trainset=train_examples)" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "16fcb28c-c442-4f1d-8876-9c6c6c47eda0", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;3;38;2;155;135;227m> Running module input with input: \n", + "query_str: How did PG meet Jessica Livingston?\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module retriever with input: \n", + "input: How did PG meet Jessica Livingston?\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module retriever_post with input: \n", + "contexts: [NodeWithScore(node=TextNode(id_='465a8143-6740-4bbd-8e6c-bb5eba4bb5a3', embedding=None, metadata={'file_path': 'paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain'...\n", + "\n", + "\u001b[0m\u001b[1;3;38;2;155;135;227m> Running module synthesizer with input: \n", + "query_str: How did PG meet Jessica Livingston?\n", + "context_str: Over the next several years I wrote lots of essays about all kinds of different topics. O'Reilly reprinted a collection of them as a book, called Hackers & Painters after one of the essays in it. I al...\n", + "\n", + "\u001b[0m" + ] + }, + { + "data": { + "text/plain": [ + "Prediction(\n", + " sonnet_answer=\"In the midst of a party, bright and gay,\\nA clever scheme brought guests together, true.\\nAmong them, Jessica, in a charming way,\\nCaught the author's eye, a friendship grew.\\n\\nShe, a marketer in a bank of old,\\nDiscovered startup tales, colorful and bold.\\nAs the bank faced troubles, she sought anew,\\nVenture capital's flaws came into view.\\n\\nTheir paths converged on a fateful night,\\nAt the corner of Garden and Walker streets.\\nA decision made, a future bright,\\nTo start an investment firm, their feats.\\n\\nThus, through ignorance and boldness, they began,\\nA journey in angel investing, a novel plan.\"\n", + ")" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# test this out \n", + "compiled_dspy_qp(query_str=\"How did PG meet Jessica Livingston?\")" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "e2b1b5f7-66d0-49d2-9bae-ca94f12a124d", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "Essential Instructions: Provide an answer to the query based solely on the context information provided. The response should be written in the style of a Shakespearean sonnet, which typically consists of 14 lines written in iambic pentameter, with a rhyme scheme of ABABCDCDEFEFGG.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "Context Str: ${context_str}\n", + "Query Str: ${query_str}\n", + "Sonnet Answer: ${sonnet_answer}\n", + "\n", + "---\n", + "\n", + "Context Str: What I Worked On February 2021 Before college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep. The first programs I tried writing were on the IBM 1401 that our school district used for what was then called \"data processing.\" This was in 9th grade, so I was 13 or 14. The school district's 1401 happened to be in the basement of our junior high school, and my friend Rich Draves and I got permission to use it. It was like a mini Bond villain's lair down there, with all these alien-looking machines — CPU, disk drives, printer, card reader — sitting up on a raised floor under bright fluorescent lights. The language we used was an early version of Fortran. You had to type programs on punch cards, then stack them in the card reader and press a button to load the program into memory and run it. The result would ordinarily be to print something on the spectacularly loud printer. I was puzzled by the 1401. I couldn't figure out what to do with it. And in retrospect there's not much I could have done with it. The only form of input to programs was data stored on punched cards, and I didn't have any data stored on punched cards. The only other option was to do things that didn't rely on any input, like calculate approximations of pi, but I didn't know enough math to do anything interesting of that type. So I'm not surprised I can't remember any programs I wrote, because they can't have done much. My clearest memory is of the moment I learned it was possible for programs not to terminate, when one of mine didn't. On a machine without time-sharing, this was a social as well as a technical error, as the data center manager's expression made clear. With microcomputers, everything changed. Now you could have a computer sitting right in front of you, on a desk, that could respond to your keystrokes as it was running instead of just churning through a stack of punch cards and then stopping. [1] The first of my friends to get a microcomputer built it himself. It was sold as a kit by Heathkit. I remember vividly how impressed and envious I felt watching him sitting in front of it, typing programs right into the computer. Computers were expensive in those days and it took me years of nagging before I convinced my father to buy one, a TRS-80, in about 1980. The gold standard then was the Apple II, but a TRS-80 was good enough. This was when I really started programming. I wrote simple games, a program to predict how high my model rockets would fly, and a word processor that my father used to write at least one book. There was only room in memory for about 2 pages of text, so he'd write 2 pages at a time and then print them out, but it was a lot better than a typewriter. Though I liked programming, I didn't plan to study it in college. In college I was going to study philosophy, which sounded much more powerful. It seemed, to my naive high school self, to be the study of the ultimate truths, compared to which the things studied in other fields would be mere domain knowledge. What I discovered when I got to college was that the other fields took up so much of the space of ideas that there wasn't much left for these supposed ultimate truths. All that seemed left for philosophy were edge cases that people in other fields felt could safely be ignored. I couldn't have put this into words when I was 18. All I knew at the time was that I kept taking philosophy courses and they kept being boring. So I decided to switch to AI. AI was in the air in the mid 1980s, but there were two things especially that made me want to work on it: a novel by Heinlein called The Moon is a Harsh Mistress, which featured an intelligent computer called Mike, and a PBS documentary that showed Terry Winograd using SHRDLU. I haven't tried rereading The Moon is a Harsh Mistress, so I don't know how well it has aged, but when I read it I was drawn entirely into its world. It seemed only a matter of time before we'd have Mike, and when I saw Winograd using SHRDLU, it seemed like that time would be a few years at most. All you had to do was teach SHRDLU more words. I didn't want to drop out of grad school, but how else was I going to get out? I remember when my friend Robert Morris got kicked out of Cornell for writing the internet worm of 1988, I was envious that he'd found such a spectacular way to get out of grad school. Then one day in April 1990 a crack appeared in the wall. I ran into professor Cheatham and he asked if I was far enough along to graduate that June. I didn't have a word of my dissertation written, but in what must have been the quickest bit of thinking in my life, I decided to take a shot at writing one in the 5 weeks or so that remained before the deadline, reusing parts of On Lisp where I could, and I was able to respond, with no perceptible delay \"Yes, I think so. I'll give you something to read in a few days.\" I picked applications of continuations as the topic. In retrospect I should have written about macros and embedded languages. There's a whole world there that's barely been explored. But all I wanted was to get out of grad school, and my rapidly written dissertation sufficed, just barely. Meanwhile I was applying to art schools. I applied to two: RISD in the US, and the Accademia di Belli Arti in Florence, which, because it was the oldest art school, I imagined would be good. RISD accepted me, and I never heard back from the Accademia, so off to Providence I went. I'd applied for the BFA program at RISD, which meant in effect that I had to go to college again. This was not as strange as it sounds, because I was only 25, and art schools are full of people of different ages. RISD counted me as a transfer sophomore and said I had to do the foundation that summer. The foundation means the classes that everyone has to take in fundamental subjects like drawing, color, and design. Toward the end of the summer I got a big surprise: a letter from the Accademia, which had been delayed because they'd sent it to Cambridge England instead of Cambridge Massachusetts, inviting me to take the entrance exam in Florence that fall. This was now only weeks away. My nice landlady let me leave my stuff in her attic. I had some money saved from consulting work I'd done in grad school; there was probably enough to last a year if I lived cheaply. Now all I had to do was learn Italian. Only stranieri (foreigners) had to take this entrance exam. In retrospect it may well have been a way of excluding them, because there were so many stranieri attracted by the idea of studying art in Florence that the Italian students would otherwise have been outnumbered. I was in decent shape at painting and drawing from the RISD foundation that summer, but I still don't know how I managed to pass the written exam. I remember that I answered the essay question by writing about Cezanne, and that I cranked up the intellectual level as high as I could to make the most of my limited vocabulary. [2] I'm only up to age 25 and already there are such conspicuous patterns. Here I was, yet again about to attend some august institution in the hopes of learning about some prestigious subject, and yet again about to be disappointed. The students and faculty in the painting department at the Accademia were the nicest people you could imagine, but they had long since arrived at an arrangement whereby the students wouldn't require the faculty to teach anything, and in return the faculty wouldn't require the students to learn anything. And at the same time all involved would adhere outwardly to the conventions of a 19th century atelier. We actually had one of those little stoves, fed with kindling, that you see in 19th century studio paintings, and a nude model sitting as close to it as possible without getting burned. Except hardly anyone else painted her besides me. The rest of the students spent their time chatting or occasionally trying to imitate things they'd seen in American art magazines. Our model turned out to live just down the street from me. She made a living from a combination of modelling and making fakes for a local antique dealer. She'd copy an obscure old painting out of a book, and then he'd take the copy and maltreat it to make it look old. [3] While I was a student at the Accademia I started painting still lives in my bedroom at night. These paintings were tiny, because the room was, and because I painted them on leftover scraps of canvas, which was all I could afford at the time.\n", + "Query Str: What did the author do growing up?\n", + "Sonnet Answer: In youth, the author's path was set to write, With tales of characters and feelings deep, Yet lacking plots, his stories took to flight, In IBM's lair, programming did creep. With punch cards stacked, the Fortran language spoke, A world of data processing unfurled, But lacking input, programs barely woke, 'Til microcomputers changed the world. A TRS-80 sparked the flame of code, Simple games and rockets soared on the screen, Philosophy's allure began to erode, AI's promise in the air was keen. From grad school's grasp, a dissertation penned, Art schools and painting, a new chapter penned.\n", + "\n", + "---\n", + "\n", + "Context Str: YC was different from other kinds of work I've done. Instead of deciding for myself what to work on, the problems came to me. Every 6 months there was a new batch of startups, and their problems, whatever they were, became our problems. It was very engaging work, because their problems were quite varied, and the good founders were very effective. If you were trying to learn the most you could about startups in the shortest possible time, you couldn't have picked a better way to do it. There were parts of the job I didn't like. Disputes between cofounders, figuring out when people were lying to us, fighting with people who maltreated the startups, and so on. But I worked hard even at the parts I didn't like. I was haunted by something Kevin Hale once said about companies: \"No one works harder than the boss.\" He meant it both descriptively and prescriptively, and it was the second part that scared me. I wanted YC to be good, so if how hard I worked set the upper bound on how hard everyone else worked, I'd better work very hard. One day in 2010, when he was visiting California for interviews, Robert Morris did something astonishing: he offered me unsolicited advice. I can only remember him doing that once before. One day at Viaweb, when I was bent over double from a kidney stone, he suggested that it would be a good idea for him to take me to the hospital. That was what it took for Rtm to offer unsolicited advice. So I remember his exact words very clearly. \"You know,\" he said, \"you should make sure Y Combinator isn't the last cool thing you do.\" At the time I didn't understand what he meant, but gradually it dawned on me that he was saying I should quit. This seemed strange advice, because YC was doing great. But if there was one thing rarer than Rtm offering advice, it was Rtm being wrong. So this set me thinking. It was true that on my current trajectory, YC would be the last thing I did, because it was only taking up more of my attention. It had already eaten Arc, and was in the process of eating essays too. Either YC was my life's work or I'd have to leave eventually. And it wasn't, so I would. In the summer of 2012 my mother had a stroke, and the cause turned out to be a blood clot caused by colon cancer. The stroke destroyed her balance, and she was put in a nursing home, but she really wanted to get out of it and back to her house, and my sister and I were determined to help her do it. I used to fly up to Oregon to visit her regularly, and I had a lot of time to think on those flights. On one of them I realized I was ready to hand YC over to someone else. I asked Jessica if she wanted to be president, but she didn't, so we decided we'd try to recruit Sam Altman. We talked to Robert and Trevor and we agreed to make it a complete changing of the guard. Up till that point YC had been controlled by the original LLC we four had started. But we wanted YC to last for a long time, and to do that it couldn't be controlled by the founders. So if Sam said yes, we'd let him reorganize YC. Robert and I would retire, and Jessica and Trevor would become ordinary partners. When we asked Sam if he wanted to be president of YC, initially he said no. He wanted to start a startup to make nuclear reactors. But I kept at it, and in October 2013 he finally agreed. We decided he'd take over starting with the winter 2014 batch. For the rest of 2013 I left running YC more and more to Sam, partly so he could learn the job, and partly because I was focused on my mother, whose cancer had returned. She died on January 15, 2014. We knew this was coming, but it was still hard when it did. I kept working on YC till March, to help get that batch of startups through Demo Day, then I checked out pretty completely. (I still talk to alumni and to new startups working on things I'm interested in, but that only takes a few hours a week.) What should I do next? Rtm's advice hadn't included anything about that. I wanted to do something completely different, so I decided I'd paint. I wanted to see how good I could get if I really focused on it. So the day after I stopped working on YC, I started painting. I don't think it was entirely luck that the first batch was so good. You had to be pretty bold to sign up for a weird thing like the Summer Founders Program instead of a summer job at a legit place like Microsoft or Goldman Sachs. The deal for startups was based on a combination of the deal we did with Julian ($10k for 10%) and what Robert said MIT grad students got for the summer ($6k). We invested $6k per founder, which in the typical two-founder case was $12k, in return for 6%. That had to be fair, because it was twice as good as the deal we ourselves had taken. Plus that first summer, which was really hot, Jessica brought the founders free air conditioners. [16] Fairly quickly I realized that we had stumbled upon the way to scale startup funding. Funding startups in batches was more convenient for us, because it meant we could do things for a lot of startups at once, but being part of a batch was better for the startups too. It solved one of the biggest problems faced by founders: the isolation. Now you not only had colleagues, but colleagues who understood the problems you were facing and could tell you how they were solving them. As YC grew, we started to notice other advantages of scale. The alumni became a tight community, dedicated to helping one another, and especially the current batch, whose shoes they remembered being in. We also noticed that the startups were becoming one another's customers. We used to refer jokingly to the \"YC GDP,\" but as YC grows this becomes less and less of a joke. Now lots of startups get their initial set of customers almost entirely from among their batchmates. I had not originally intended YC to be a full-time job. I was going to do three things: hack, write essays, and work on YC. As YC grew, and I grew more excited about it, it started to take up a lot more than a third of my attention. But for the first few years I was still able to work on other things. In the summer of 2006, Robert and I started working on a new version of Arc. This one was reasonably fast, because it was compiled into Scheme. To test this new Arc, I wrote Hacker News in it. It was originally meant to be a news aggregator for startup founders and was called Startup News, but after a few months I got tired of reading about nothing but startups. Plus it wasn't startup founders we wanted to reach. It was future startup founders. So I changed the name to Hacker News and the topic to whatever engaged one's intellectual curiosity. HN was no doubt good for YC, but it was also by far the biggest source of stress for me. If all I'd had to do was select and help founders, life would have been so easy. And that implies that HN was a mistake. Surely the biggest source of stress in one's work should at least be something close to the core of the work. Whereas I was like someone who was in pain while running a marathon not from the exertion of running, but because I had a blister from an ill-fitting shoe. When I was dealing with some urgent problem during YC, there was about a 60% chance it had to do with HN, and a 40% chance it had do with everything else combined. [17] As well as HN, I wrote all of YC's internal software in Arc. But while I continued to work a good deal in Arc, I gradually stopped working on Arc, partly because I didn't have time to, and partly because it was a lot less attractive to mess around with the language now that we had all this infrastructure depending on it. So now my three projects were reduced to two: writing essays and working on YC. YC was different from other kinds of work I've done. Instead of deciding for myself what to work on, the problems came to me. Every 6 months there was a new batch of startups, and their problems, whatever they were, became our problems. It was very engaging work, because their problems were quite varied, and the good founders were very effective. If you were trying to learn the most you could about startups in the shortest possible time, you couldn't have picked a better way to do it. There were parts of the job I didn't like. Disputes between cofounders, figuring out when people were lying to us, fighting with people who maltreated the startups, and so on. But I worked hard even at the parts I didn't like. I was haunted by something Kevin Hale once said about companies: \"No one works harder than the boss.\"\n", + "Query Str: What did the author do during his time at YC?\n", + "Sonnet Answer: In YC's realm, the author did reside, Where problems of startups did swiftly glide. Each batch brought forth a new set of tasks, Engaging work, where founders wore their masks. Disputes and lies, a part of the game, Yet the author worked hard, not seeking fame. Haunted by words of Kevin Hale's decree, \"No one works harder than the boss,\" said he. In 2012, a shift was in the air, The author decided to leave with care. Handing YC to another's command, To focus on painting, a new strand. Through YC's growth, the author did strive, To learn the most about startups alive.\n", + "\n", + "---\n", + "\n", + "Context Str: Over the next several years I wrote lots of essays about all kinds of different topics. O'Reilly reprinted a collection of them as a book, called Hackers & Painters after one of the essays in it. I also worked on spam filters, and did some more painting. I used to have dinners for a group of friends every thursday night, which taught me how to cook for groups. And I bought another building in Cambridge, a former candy factory (and later, twas said, porn studio), to use as an office. One night in October 2003 there was a big party at my house. It was a clever idea of my friend Maria Daniels, who was one of the thursday diners. Three separate hosts would all invite their friends to one party. So for every guest, two thirds of the other guests would be people they didn't know but would probably like. One of the guests was someone I didn't know but would turn out to like a lot: a woman called Jessica Livingston. A couple days later I asked her out. Jessica was in charge of marketing at a Boston investment bank. This bank thought it understood startups, but over the next year, as she met friends of mine from the startup world, she was surprised how different reality was. And how colorful their stories were. So she decided to compile a book of interviews with startup founders. When the bank had financial problems and she had to fire half her staff, she started looking for a new job. In early 2005 she interviewed for a marketing job at a Boston VC firm. It took them weeks to make up their minds, and during this time I started telling her about all the things that needed to be fixed about venture capital. They should make a larger number of smaller investments instead of a handful of giant ones, they should be funding younger, more technical founders instead of MBAs, they should let the founders remain as CEO, and so on. One of my tricks for writing essays had always been to give talks. The prospect of having to stand up in front of a group of people and tell them something that won't waste their time is a great spur to the imagination. When the Harvard Computer Society, the undergrad computer club, asked me to give a talk, I decided I would tell them how to start a startup. Maybe they'd be able to avoid the worst of the mistakes we'd made. So I gave this talk, in the course of which I told them that the best sources of seed funding were successful startup founders, because then they'd be sources of advice too. Whereupon it seemed they were all looking expectantly at me. Horrified at the prospect of having my inbox flooded by business plans (if I'd only known), I blurted out \"But not me!\" and went on with the talk. But afterward it occurred to me that I should really stop procrastinating about angel investing. I'd been meaning to since Yahoo bought us, and now it was 7 years later and I still hadn't done one angel investment. Meanwhile I had been scheming with Robert and Trevor about projects we could work on together. I missed working with them, and it seemed like there had to be something we could collaborate on. As Jessica and I were walking home from dinner on March 11, at the corner of Garden and Walker streets, these three threads converged. Screw the VCs who were taking so long to make up their minds. We'd start our own investment firm and actually implement the ideas we'd been talking about. I'd fund it, and Jessica could quit her job and work for it, and we'd get Robert and Trevor as partners too. [13] Once again, ignorance worked in our favor. We had no idea how to be angel investors, and in Boston in 2005 there were no Ron Conways to learn from. So we just made what seemed like the obvious choices, and some of the things we did turned out to be novel. There are multiple components to Y Combinator, and we didn't figure them all out at once. The part we got first was to be an angel firm. In those days, those two words didn't go together. There were VC firms, which were organized companies with people whose job it was to make investments, but they only did big, million dollar investments. And there were angels, who did smaller investments, but these were individuals who were usually focused on other things and made investments on the side. And neither of them helped founders enough in the beginning. We knew how helpless founders were in some respects, because we remembered how helpless we'd been. For example, one thing Julian had done for us that seemed to us like magic was to get us set up as a company. Nor had I changed my grad student lifestyle significantly since we started. So when Yahoo bought us it felt like going from rags to riches. Since we were going to California, I bought a car, a yellow 1998 VW GTI. I remember thinking that its leather seats alone were by far the most luxurious thing I owned. The next year, from the summer of 1998 to the summer of 1999, must have been the least productive of my life. I didn't realize it at the time, but I was worn out from the effort and stress of running Viaweb. For a while after I got to California I tried to continue my usual m.o. of programming till 3 in the morning, but fatigue combined with Yahoo's prematurely aged culture and grim cube farm in Santa Clara gradually dragged me down. After a few months it felt disconcertingly like working at Interleaf. Yahoo had given us a lot of options when they bought us. At the time I thought Yahoo was so overvalued that they'd never be worth anything, but to my astonishment the stock went up 5x in the next year. I hung on till the first chunk of options vested, then in the summer of 1999 I left. It had been so long since I'd painted anything that I'd half forgotten why I was doing this. My brain had been entirely full of software and men's shirts for 4 years. But I had done this to get rich so I could paint, I reminded myself, and now I was rich, so I should go paint. When I said I was leaving, my boss at Yahoo had a long conversation with me about my plans. I told him all about the kinds of pictures I wanted to paint. At the time I was touched that he took such an interest in me. Now I realize it was because he thought I was lying. My options at that point were worth about $2 million a month. If I was leaving that kind of money on the table, it could only be to go and start some new startup, and if I did, I might take people with me. This was the height of the Internet Bubble, and Yahoo was ground zero of it. My boss was at that moment a billionaire. Leaving then to start a new startup must have seemed to him an insanely, and yet also plausibly, ambitious plan. But I really was quitting to paint, and I started immediately. There was no time to lose. I'd already burned 4 years getting rich. Now when I talk to founders who are leaving after selling their companies, my advice is always the same: take a vacation. That's what I should have done, just gone off somewhere and done nothing for a month or two, but the idea never occurred to me. So I tried to paint, but I just didn't seem to have any energy or ambition. Part of the problem was that I didn't know many people in California. I'd compounded this problem by buying a house up in the Santa Cruz Mountains, with a beautiful view but miles from anywhere. I stuck it out for a few more months, then in desperation I went back to New York, where unless you understand about rent control you'll be surprised to hear I still had my apartment, sealed up like a tomb of my old life. Idelle was in New York at least, and there were other people trying to paint there, even though I didn't know any of them. When I got back to New York I resumed my old life, except now I was rich. It was as weird as it sounds. I resumed all my old patterns, except now there were doors where there hadn't been. Now when I was tired of walking, all I had to do was raise my hand, and (unless it was raining) a taxi would stop to pick me up. Now when I walked past charming little restaurants I could go in and order lunch. It was exciting for a while. Painting started to go better. I experimented with a new kind of still life where I'd paint one painting in the old way, then photograph it and print it, blown up, on canvas, and then use that as the underpainting for a second still life, painted from the same objects (which hopefully hadn't rotted yet). Meanwhile I looked for an apartment to buy. Now I could actually choose what neighborhood to live in. Where, I asked myself and various real estate agents, is the Cambridge of New York? Aided by occasional visits to actual Cambridge, I gradually realized there wasn't one. Huh. Around this time, in the spring of 2000, I had an idea. It was clear from our experience with Viaweb that web apps were the future.\n", + "Query Str: How did PG meet Jessica Livingston?\n", + "Sonnet Answer:\u001b[32m In the midst of a party, bright and gay,\n", + "A clever scheme brought guests together, true.\n", + "Among them, Jessica, in a charming way,\n", + "Caught the author's eye, a friendship grew.\n", + "\n", + "She, a marketer in a bank of old,\n", + "Discovered startup tales, colorful and bold.\n", + "As the bank faced troubles, she sought anew,\n", + "Venture capital's flaws came into view.\n", + "\n", + "Their paths converged on a fateful night,\n", + "At the corner of Garden and Walker streets.\n", + "A decision made, a future bright,\n", + "To start an investment firm, their feats.\n", + "\n", + "Thus, through ignorance and boldness, they began,\n", + "A journey in angel investing, a novel plan.\u001b[0m\n", + "\n", + "\n", + "\n" + ] + }, + { + "data": { + "text/plain": [ + "'\\n\\n\\nEssential Instructions: Provide an answer to the query based solely on the context information provided. The response should be written in the style of a Shakespearean sonnet, which typically consists of 14 lines written in iambic pentameter, with a rhyme scheme of ABABCDCDEFEFGG.\\n\\n---\\n\\nFollow the following format.\\n\\nContext Str: ${context_str}\\nQuery Str: ${query_str}\\nSonnet Answer: ${sonnet_answer}\\n\\n---\\n\\nContext Str: What I Worked On February 2021 Before college the two main things I worked on, outside of school, were writing and programming. I didn\\'t write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep. The first programs I tried writing were on the IBM 1401 that our school district used for what was then called \"data processing.\" This was in 9th grade, so I was 13 or 14. The school district\\'s 1401 happened to be in the basement of our junior high school, and my friend Rich Draves and I got permission to use it. It was like a mini Bond villain\\'s lair down there, with all these alien-looking machines — CPU, disk drives, printer, card reader — sitting up on a raised floor under bright fluorescent lights. The language we used was an early version of Fortran. You had to type programs on punch cards, then stack them in the card reader and press a button to load the program into memory and run it. The result would ordinarily be to print something on the spectacularly loud printer. I was puzzled by the 1401. I couldn\\'t figure out what to do with it. And in retrospect there\\'s not much I could have done with it. The only form of input to programs was data stored on punched cards, and I didn\\'t have any data stored on punched cards. The only other option was to do things that didn\\'t rely on any input, like calculate approximations of pi, but I didn\\'t know enough math to do anything interesting of that type. So I\\'m not surprised I can\\'t remember any programs I wrote, because they can\\'t have done much. My clearest memory is of the moment I learned it was possible for programs not to terminate, when one of mine didn\\'t. On a machine without time-sharing, this was a social as well as a technical error, as the data center manager\\'s expression made clear. With microcomputers, everything changed. Now you could have a computer sitting right in front of you, on a desk, that could respond to your keystrokes as it was running instead of just churning through a stack of punch cards and then stopping. [1] The first of my friends to get a microcomputer built it himself. It was sold as a kit by Heathkit. I remember vividly how impressed and envious I felt watching him sitting in front of it, typing programs right into the computer. Computers were expensive in those days and it took me years of nagging before I convinced my father to buy one, a TRS-80, in about 1980. The gold standard then was the Apple II, but a TRS-80 was good enough. This was when I really started programming. I wrote simple games, a program to predict how high my model rockets would fly, and a word processor that my father used to write at least one book. There was only room in memory for about 2 pages of text, so he\\'d write 2 pages at a time and then print them out, but it was a lot better than a typewriter. Though I liked programming, I didn\\'t plan to study it in college. In college I was going to study philosophy, which sounded much more powerful. It seemed, to my naive high school self, to be the study of the ultimate truths, compared to which the things studied in other fields would be mere domain knowledge. What I discovered when I got to college was that the other fields took up so much of the space of ideas that there wasn\\'t much left for these supposed ultimate truths. All that seemed left for philosophy were edge cases that people in other fields felt could safely be ignored. I couldn\\'t have put this into words when I was 18. All I knew at the time was that I kept taking philosophy courses and they kept being boring. So I decided to switch to AI. AI was in the air in the mid 1980s, but there were two things especially that made me want to work on it: a novel by Heinlein called The Moon is a Harsh Mistress, which featured an intelligent computer called Mike, and a PBS documentary that showed Terry Winograd using SHRDLU. I haven\\'t tried rereading The Moon is a Harsh Mistress, so I don\\'t know how well it has aged, but when I read it I was drawn entirely into its world. It seemed only a matter of time before we\\'d have Mike, and when I saw Winograd using SHRDLU, it seemed like that time would be a few years at most. All you had to do was teach SHRDLU more words. I didn\\'t want to drop out of grad school, but how else was I going to get out? I remember when my friend Robert Morris got kicked out of Cornell for writing the internet worm of 1988, I was envious that he\\'d found such a spectacular way to get out of grad school. Then one day in April 1990 a crack appeared in the wall. I ran into professor Cheatham and he asked if I was far enough along to graduate that June. I didn\\'t have a word of my dissertation written, but in what must have been the quickest bit of thinking in my life, I decided to take a shot at writing one in the 5 weeks or so that remained before the deadline, reusing parts of On Lisp where I could, and I was able to respond, with no perceptible delay \"Yes, I think so. I\\'ll give you something to read in a few days.\" I picked applications of continuations as the topic. In retrospect I should have written about macros and embedded languages. There\\'s a whole world there that\\'s barely been explored. But all I wanted was to get out of grad school, and my rapidly written dissertation sufficed, just barely. Meanwhile I was applying to art schools. I applied to two: RISD in the US, and the Accademia di Belli Arti in Florence, which, because it was the oldest art school, I imagined would be good. RISD accepted me, and I never heard back from the Accademia, so off to Providence I went. I\\'d applied for the BFA program at RISD, which meant in effect that I had to go to college again. This was not as strange as it sounds, because I was only 25, and art schools are full of people of different ages. RISD counted me as a transfer sophomore and said I had to do the foundation that summer. The foundation means the classes that everyone has to take in fundamental subjects like drawing, color, and design. Toward the end of the summer I got a big surprise: a letter from the Accademia, which had been delayed because they\\'d sent it to Cambridge England instead of Cambridge Massachusetts, inviting me to take the entrance exam in Florence that fall. This was now only weeks away. My nice landlady let me leave my stuff in her attic. I had some money saved from consulting work I\\'d done in grad school; there was probably enough to last a year if I lived cheaply. Now all I had to do was learn Italian. Only stranieri (foreigners) had to take this entrance exam. In retrospect it may well have been a way of excluding them, because there were so many stranieri attracted by the idea of studying art in Florence that the Italian students would otherwise have been outnumbered. I was in decent shape at painting and drawing from the RISD foundation that summer, but I still don\\'t know how I managed to pass the written exam. I remember that I answered the essay question by writing about Cezanne, and that I cranked up the intellectual level as high as I could to make the most of my limited vocabulary. [2] I\\'m only up to age 25 and already there are such conspicuous patterns. Here I was, yet again about to attend some august institution in the hopes of learning about some prestigious subject, and yet again about to be disappointed. The students and faculty in the painting department at the Accademia were the nicest people you could imagine, but they had long since arrived at an arrangement whereby the students wouldn\\'t require the faculty to teach anything, and in return the faculty wouldn\\'t require the students to learn anything. And at the same time all involved would adhere outwardly to the conventions of a 19th century atelier. We actually had one of those little stoves, fed with kindling, that you see in 19th century studio paintings, and a nude model sitting as close to it as possible without getting burned. Except hardly anyone else painted her besides me. The rest of the students spent their time chatting or occasionally trying to imitate things they\\'d seen in American art magazines. Our model turned out to live just down the street from me. She made a living from a combination of modelling and making fakes for a local antique dealer. She\\'d copy an obscure old painting out of a book, and then he\\'d take the copy and maltreat it to make it look old. [3] While I was a student at the Accademia I started painting still lives in my bedroom at night. These paintings were tiny, because the room was, and because I painted them on leftover scraps of canvas, which was all I could afford at the time.\\nQuery Str: What did the author do growing up?\\nSonnet Answer: In youth, the author\\'s path was set to write, With tales of characters and feelings deep, Yet lacking plots, his stories took to flight, In IBM\\'s lair, programming did creep. With punch cards stacked, the Fortran language spoke, A world of data processing unfurled, But lacking input, programs barely woke, \\'Til microcomputers changed the world. A TRS-80 sparked the flame of code, Simple games and rockets soared on the screen, Philosophy\\'s allure began to erode, AI\\'s promise in the air was keen. From grad school\\'s grasp, a dissertation penned, Art schools and painting, a new chapter penned.\\n\\n---\\n\\nContext Str: YC was different from other kinds of work I\\'ve done. Instead of deciding for myself what to work on, the problems came to me. Every 6 months there was a new batch of startups, and their problems, whatever they were, became our problems. It was very engaging work, because their problems were quite varied, and the good founders were very effective. If you were trying to learn the most you could about startups in the shortest possible time, you couldn\\'t have picked a better way to do it. There were parts of the job I didn\\'t like. Disputes between cofounders, figuring out when people were lying to us, fighting with people who maltreated the startups, and so on. But I worked hard even at the parts I didn\\'t like. I was haunted by something Kevin Hale once said about companies: \"No one works harder than the boss.\" He meant it both descriptively and prescriptively, and it was the second part that scared me. I wanted YC to be good, so if how hard I worked set the upper bound on how hard everyone else worked, I\\'d better work very hard. One day in 2010, when he was visiting California for interviews, Robert Morris did something astonishing: he offered me unsolicited advice. I can only remember him doing that once before. One day at Viaweb, when I was bent over double from a kidney stone, he suggested that it would be a good idea for him to take me to the hospital. That was what it took for Rtm to offer unsolicited advice. So I remember his exact words very clearly. \"You know,\" he said, \"you should make sure Y Combinator isn\\'t the last cool thing you do.\" At the time I didn\\'t understand what he meant, but gradually it dawned on me that he was saying I should quit. This seemed strange advice, because YC was doing great. But if there was one thing rarer than Rtm offering advice, it was Rtm being wrong. So this set me thinking. It was true that on my current trajectory, YC would be the last thing I did, because it was only taking up more of my attention. It had already eaten Arc, and was in the process of eating essays too. Either YC was my life\\'s work or I\\'d have to leave eventually. And it wasn\\'t, so I would. In the summer of 2012 my mother had a stroke, and the cause turned out to be a blood clot caused by colon cancer. The stroke destroyed her balance, and she was put in a nursing home, but she really wanted to get out of it and back to her house, and my sister and I were determined to help her do it. I used to fly up to Oregon to visit her regularly, and I had a lot of time to think on those flights. On one of them I realized I was ready to hand YC over to someone else. I asked Jessica if she wanted to be president, but she didn\\'t, so we decided we\\'d try to recruit Sam Altman. We talked to Robert and Trevor and we agreed to make it a complete changing of the guard. Up till that point YC had been controlled by the original LLC we four had started. But we wanted YC to last for a long time, and to do that it couldn\\'t be controlled by the founders. So if Sam said yes, we\\'d let him reorganize YC. Robert and I would retire, and Jessica and Trevor would become ordinary partners. When we asked Sam if he wanted to be president of YC, initially he said no. He wanted to start a startup to make nuclear reactors. But I kept at it, and in October 2013 he finally agreed. We decided he\\'d take over starting with the winter 2014 batch. For the rest of 2013 I left running YC more and more to Sam, partly so he could learn the job, and partly because I was focused on my mother, whose cancer had returned. She died on January 15, 2014. We knew this was coming, but it was still hard when it did. I kept working on YC till March, to help get that batch of startups through Demo Day, then I checked out pretty completely. (I still talk to alumni and to new startups working on things I\\'m interested in, but that only takes a few hours a week.) What should I do next? Rtm\\'s advice hadn\\'t included anything about that. I wanted to do something completely different, so I decided I\\'d paint. I wanted to see how good I could get if I really focused on it. So the day after I stopped working on YC, I started painting. I don\\'t think it was entirely luck that the first batch was so good. You had to be pretty bold to sign up for a weird thing like the Summer Founders Program instead of a summer job at a legit place like Microsoft or Goldman Sachs. The deal for startups was based on a combination of the deal we did with Julian ($10k for 10%) and what Robert said MIT grad students got for the summer ($6k). We invested $6k per founder, which in the typical two-founder case was $12k, in return for 6%. That had to be fair, because it was twice as good as the deal we ourselves had taken. Plus that first summer, which was really hot, Jessica brought the founders free air conditioners. [16] Fairly quickly I realized that we had stumbled upon the way to scale startup funding. Funding startups in batches was more convenient for us, because it meant we could do things for a lot of startups at once, but being part of a batch was better for the startups too. It solved one of the biggest problems faced by founders: the isolation. Now you not only had colleagues, but colleagues who understood the problems you were facing and could tell you how they were solving them. As YC grew, we started to notice other advantages of scale. The alumni became a tight community, dedicated to helping one another, and especially the current batch, whose shoes they remembered being in. We also noticed that the startups were becoming one another\\'s customers. We used to refer jokingly to the \"YC GDP,\" but as YC grows this becomes less and less of a joke. Now lots of startups get their initial set of customers almost entirely from among their batchmates. I had not originally intended YC to be a full-time job. I was going to do three things: hack, write essays, and work on YC. As YC grew, and I grew more excited about it, it started to take up a lot more than a third of my attention. But for the first few years I was still able to work on other things. In the summer of 2006, Robert and I started working on a new version of Arc. This one was reasonably fast, because it was compiled into Scheme. To test this new Arc, I wrote Hacker News in it. It was originally meant to be a news aggregator for startup founders and was called Startup News, but after a few months I got tired of reading about nothing but startups. Plus it wasn\\'t startup founders we wanted to reach. It was future startup founders. So I changed the name to Hacker News and the topic to whatever engaged one\\'s intellectual curiosity. HN was no doubt good for YC, but it was also by far the biggest source of stress for me. If all I\\'d had to do was select and help founders, life would have been so easy. And that implies that HN was a mistake. Surely the biggest source of stress in one\\'s work should at least be something close to the core of the work. Whereas I was like someone who was in pain while running a marathon not from the exertion of running, but because I had a blister from an ill-fitting shoe. When I was dealing with some urgent problem during YC, there was about a 60% chance it had to do with HN, and a 40% chance it had do with everything else combined. [17] As well as HN, I wrote all of YC\\'s internal software in Arc. But while I continued to work a good deal in Arc, I gradually stopped working on Arc, partly because I didn\\'t have time to, and partly because it was a lot less attractive to mess around with the language now that we had all this infrastructure depending on it. So now my three projects were reduced to two: writing essays and working on YC. YC was different from other kinds of work I\\'ve done. Instead of deciding for myself what to work on, the problems came to me. Every 6 months there was a new batch of startups, and their problems, whatever they were, became our problems. It was very engaging work, because their problems were quite varied, and the good founders were very effective. If you were trying to learn the most you could about startups in the shortest possible time, you couldn\\'t have picked a better way to do it. There were parts of the job I didn\\'t like. Disputes between cofounders, figuring out when people were lying to us, fighting with people who maltreated the startups, and so on. But I worked hard even at the parts I didn\\'t like. I was haunted by something Kevin Hale once said about companies: \"No one works harder than the boss.\"\\nQuery Str: What did the author do during his time at YC?\\nSonnet Answer: In YC\\'s realm, the author did reside, Where problems of startups did swiftly glide. Each batch brought forth a new set of tasks, Engaging work, where founders wore their masks. Disputes and lies, a part of the game, Yet the author worked hard, not seeking fame. Haunted by words of Kevin Hale\\'s decree, \"No one works harder than the boss,\" said he. In 2012, a shift was in the air, The author decided to leave with care. Handing YC to another\\'s command, To focus on painting, a new strand. Through YC\\'s growth, the author did strive, To learn the most about startups alive.\\n\\n---\\n\\nContext Str: Over the next several years I wrote lots of essays about all kinds of different topics. O\\'Reilly reprinted a collection of them as a book, called Hackers & Painters after one of the essays in it. I also worked on spam filters, and did some more painting. I used to have dinners for a group of friends every thursday night, which taught me how to cook for groups. And I bought another building in Cambridge, a former candy factory (and later, twas said, porn studio), to use as an office. One night in October 2003 there was a big party at my house. It was a clever idea of my friend Maria Daniels, who was one of the thursday diners. Three separate hosts would all invite their friends to one party. So for every guest, two thirds of the other guests would be people they didn\\'t know but would probably like. One of the guests was someone I didn\\'t know but would turn out to like a lot: a woman called Jessica Livingston. A couple days later I asked her out. Jessica was in charge of marketing at a Boston investment bank. This bank thought it understood startups, but over the next year, as she met friends of mine from the startup world, she was surprised how different reality was. And how colorful their stories were. So she decided to compile a book of interviews with startup founders. When the bank had financial problems and she had to fire half her staff, she started looking for a new job. In early 2005 she interviewed for a marketing job at a Boston VC firm. It took them weeks to make up their minds, and during this time I started telling her about all the things that needed to be fixed about venture capital. They should make a larger number of smaller investments instead of a handful of giant ones, they should be funding younger, more technical founders instead of MBAs, they should let the founders remain as CEO, and so on. One of my tricks for writing essays had always been to give talks. The prospect of having to stand up in front of a group of people and tell them something that won\\'t waste their time is a great spur to the imagination. When the Harvard Computer Society, the undergrad computer club, asked me to give a talk, I decided I would tell them how to start a startup. Maybe they\\'d be able to avoid the worst of the mistakes we\\'d made. So I gave this talk, in the course of which I told them that the best sources of seed funding were successful startup founders, because then they\\'d be sources of advice too. Whereupon it seemed they were all looking expectantly at me. Horrified at the prospect of having my inbox flooded by business plans (if I\\'d only known), I blurted out \"But not me!\" and went on with the talk. But afterward it occurred to me that I should really stop procrastinating about angel investing. I\\'d been meaning to since Yahoo bought us, and now it was 7 years later and I still hadn\\'t done one angel investment. Meanwhile I had been scheming with Robert and Trevor about projects we could work on together. I missed working with them, and it seemed like there had to be something we could collaborate on. As Jessica and I were walking home from dinner on March 11, at the corner of Garden and Walker streets, these three threads converged. Screw the VCs who were taking so long to make up their minds. We\\'d start our own investment firm and actually implement the ideas we\\'d been talking about. I\\'d fund it, and Jessica could quit her job and work for it, and we\\'d get Robert and Trevor as partners too. [13] Once again, ignorance worked in our favor. We had no idea how to be angel investors, and in Boston in 2005 there were no Ron Conways to learn from. So we just made what seemed like the obvious choices, and some of the things we did turned out to be novel. There are multiple components to Y Combinator, and we didn\\'t figure them all out at once. The part we got first was to be an angel firm. In those days, those two words didn\\'t go together. There were VC firms, which were organized companies with people whose job it was to make investments, but they only did big, million dollar investments. And there were angels, who did smaller investments, but these were individuals who were usually focused on other things and made investments on the side. And neither of them helped founders enough in the beginning. We knew how helpless founders were in some respects, because we remembered how helpless we\\'d been. For example, one thing Julian had done for us that seemed to us like magic was to get us set up as a company. Nor had I changed my grad student lifestyle significantly since we started. So when Yahoo bought us it felt like going from rags to riches. Since we were going to California, I bought a car, a yellow 1998 VW GTI. I remember thinking that its leather seats alone were by far the most luxurious thing I owned. The next year, from the summer of 1998 to the summer of 1999, must have been the least productive of my life. I didn\\'t realize it at the time, but I was worn out from the effort and stress of running Viaweb. For a while after I got to California I tried to continue my usual m.o. of programming till 3 in the morning, but fatigue combined with Yahoo\\'s prematurely aged culture and grim cube farm in Santa Clara gradually dragged me down. After a few months it felt disconcertingly like working at Interleaf. Yahoo had given us a lot of options when they bought us. At the time I thought Yahoo was so overvalued that they\\'d never be worth anything, but to my astonishment the stock went up 5x in the next year. I hung on till the first chunk of options vested, then in the summer of 1999 I left. It had been so long since I\\'d painted anything that I\\'d half forgotten why I was doing this. My brain had been entirely full of software and men\\'s shirts for 4 years. But I had done this to get rich so I could paint, I reminded myself, and now I was rich, so I should go paint. When I said I was leaving, my boss at Yahoo had a long conversation with me about my plans. I told him all about the kinds of pictures I wanted to paint. At the time I was touched that he took such an interest in me. Now I realize it was because he thought I was lying. My options at that point were worth about $2 million a month. If I was leaving that kind of money on the table, it could only be to go and start some new startup, and if I did, I might take people with me. This was the height of the Internet Bubble, and Yahoo was ground zero of it. My boss was at that moment a billionaire. Leaving then to start a new startup must have seemed to him an insanely, and yet also plausibly, ambitious plan. But I really was quitting to paint, and I started immediately. There was no time to lose. I\\'d already burned 4 years getting rich. Now when I talk to founders who are leaving after selling their companies, my advice is always the same: take a vacation. That\\'s what I should have done, just gone off somewhere and done nothing for a month or two, but the idea never occurred to me. So I tried to paint, but I just didn\\'t seem to have any energy or ambition. Part of the problem was that I didn\\'t know many people in California. I\\'d compounded this problem by buying a house up in the Santa Cruz Mountains, with a beautiful view but miles from anywhere. I stuck it out for a few more months, then in desperation I went back to New York, where unless you understand about rent control you\\'ll be surprised to hear I still had my apartment, sealed up like a tomb of my old life. Idelle was in New York at least, and there were other people trying to paint there, even though I didn\\'t know any of them. When I got back to New York I resumed my old life, except now I was rich. It was as weird as it sounds. I resumed all my old patterns, except now there were doors where there hadn\\'t been. Now when I was tired of walking, all I had to do was raise my hand, and (unless it was raining) a taxi would stop to pick me up. Now when I walked past charming little restaurants I could go in and order lunch. It was exciting for a while. Painting started to go better. I experimented with a new kind of still life where I\\'d paint one painting in the old way, then photograph it and print it, blown up, on canvas, and then use that as the underpainting for a second still life, painted from the same objects (which hopefully hadn\\'t rotted yet). Meanwhile I looked for an apartment to buy. Now I could actually choose what neighborhood to live in. Where, I asked myself and various real estate agents, is the Cambridge of New York? Aided by occasional visits to actual Cambridge, I gradually realized there wasn\\'t one. Huh. Around this time, in the spring of 2000, I had an idea. It was clear from our experience with Viaweb that web apps were the future.\\nQuery Str: How did PG meet Jessica Livingston?\\nSonnet Answer:\\x1b[32m In the midst of a party, bright and gay,\\nA clever scheme brought guests together, true.\\nAmong them, Jessica, in a charming way,\\nCaught the author\\'s eye, a friendship grew.\\n\\nShe, a marketer in a bank of old,\\nDiscovered startup tales, colorful and bold.\\nAs the bank faced troubles, she sought anew,\\nVenture capital\\'s flaws came into view.\\n\\nTheir paths converged on a fateful night,\\nAt the corner of Garden and Walker streets.\\nA decision made, a future bright,\\nTo start an investment firm, their feats.\\n\\nThus, through ignorance and boldness, they began,\\nA journey in angel investing, a novel plan.\\x1b[0m\\n\\n\\n'" + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# [optional]: inspect the optimized prompt \n", + "turbo.inspect_history(n=1)" + ] + }, + { + "cell_type": "markdown", + "id": "d4daae77-bf36-418f-9a80-c9c0fb6f20ec", + "metadata": {}, + "source": [ + "## [Part 3] Port over Optimized Prompts to LlamaIndex using the DSPy Prompt Template\n", + "\n", + "Extract out a prompt from an existing compiled DSPy module, and then port it over to any LlamaIndex pipeline! \n", + "\n", + "In the example below we use our `DSPyPromptTemplate` to extract out the compiled few-shot prompt from the optimized query pipeline. \n", + "\n", + "We then plug it into a separate query engine over the PG essay." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "8180ea2b-412a-41a6-a707-f2945d9dcda0", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from llamaindex import DSPyPromptTemplate\n", + "\n", + "# NOTE: you cannot do DSPyPromptTemplate(dspy_component.predict_module) - the predict_module is replaced.\n", + "qa_prompt_tmpl = DSPyPromptTemplate(compiled_dspy_qp.query_pipeline.module_dict[\"synthesizer\"].predict_module)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "e3c86e63-f225-4540-b1a2-a48a13e29756", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Answer questions with short factoid answers.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "Context Str: contains relevant facts\n", + "Query Str: ${query_str}\n", + "Answer: often between 1 and 5 words\n", + "\n", + "---\n", + "\n", + "Context Str: What I Worked On February 2021 Before college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep. The first programs I tried writing were on the IBM 1401 that our school district used for what was then called \"data processing.\" This was in 9th grade, so I was 13 or 14. The school district's 1401 happened to be in the basement of our junior high school, and my friend Rich Draves and I got permission to use it. It was like a mini Bond villain's lair down there, with all these alien-looking machines — CPU, disk drives, printer, card reader — sitting up on a raised floor under bright fluorescent lights. The language we used was an early version of Fortran. You had to type programs on punch cards, then stack them in the card reader and press a button to load the program into memory and run it. The result would ordinarily be to print something on the spectacularly loud printer. I was puzzled by the 1401. I couldn't figure out what to do with it. And in retrospect there's not much I could have done with it. The only form of input to programs was data stored on punched cards, and I didn't have any data stored on punched cards. The only other option was to do things that didn't rely on any input, like calculate approximations of pi, but I didn't know enough math to do anything interesting of that type. So I'm not surprised I can't remember any programs I wrote, because they can't have done much. My clearest memory is of the moment I learned it was possible for programs not to terminate, when one of mine didn't. On a machine without time-sharing, this was a social as well as a technical error, as the data center manager's expression made clear. With microcomputers, everything changed. Now you could have a computer sitting right in front of you, on a desk, that could respond to your keystrokes as it was running instead of just churning through a stack of punch cards and then stopping. [1] The first of my friends to get a microcomputer built it himself. It was sold as a kit by Heathkit. I remember vividly how impressed and envious I felt watching him sitting in front of it, typing programs right into the computer. Computers were expensive in those days and it took me years of nagging before I convinced my father to buy one, a TRS-80, in about 1980. The gold standard then was the Apple II, but a TRS-80 was good enough. This was when I really started programming. I wrote simple games, a program to predict how high my model rockets would fly, and a word processor that my father used to write at least one book. There was only room in memory for about 2 pages of text, so he'd write 2 pages at a time and then print them out, but it was a lot better than a typewriter. Though I liked programming, I didn't plan to study it in college. In college I was going to study philosophy, which sounded much more powerful. It seemed, to my naive high school self, to be the study of the ultimate truths, compared to which the things studied in other fields would be mere domain knowledge. What I discovered when I got to college was that the other fields took up so much of the space of ideas that there wasn't much left for these supposed ultimate truths. All that seemed left for philosophy were edge cases that people in other fields felt could safely be ignored. I couldn't have put this into words when I was 18. All I knew at the time was that I kept taking philosophy courses and they kept being boring. So I decided to switch to AI. AI was in the air in the mid 1980s, but there were two things especially that made me want to work on it: a novel by Heinlein called The Moon is a Harsh Mistress, which featured an intelligent computer called Mike, and a PBS documentary that showed Terry Winograd using SHRDLU. I haven't tried rereading The Moon is a Harsh Mistress, so I don't know how well it has aged, but when I read it I was drawn entirely into its world. It seemed only a matter of time before we'd have Mike, and when I saw Winograd using SHRDLU, it seemed like that time would be a few years at most. All you had to do was teach SHRDLU more words. I didn't want to drop out of grad school, but how else was I going to get out? I remember when my friend Robert Morris got kicked out of Cornell for writing the internet worm of 1988, I was envious that he'd found such a spectacular way to get out of grad school. Then one day in April 1990 a crack appeared in the wall. I ran into professor Cheatham and he asked if I was far enough along to graduate that June. I didn't have a word of my dissertation written, but in what must have been the quickest bit of thinking in my life, I decided to take a shot at writing one in the 5 weeks or so that remained before the deadline, reusing parts of On Lisp where I could, and I was able to respond, with no perceptible delay \"Yes, I think so. I'll give you something to read in a few days.\" I picked applications of continuations as the topic. In retrospect I should have written about macros and embedded languages. There's a whole world there that's barely been explored. But all I wanted was to get out of grad school, and my rapidly written dissertation sufficed, just barely. Meanwhile I was applying to art schools. I applied to two: RISD in the US, and the Accademia di Belli Arti in Florence, which, because it was the oldest art school, I imagined would be good. RISD accepted me, and I never heard back from the Accademia, so off to Providence I went. I'd applied for the BFA program at RISD, which meant in effect that I had to go to college again. This was not as strange as it sounds, because I was only 25, and art schools are full of people of different ages. RISD counted me as a transfer sophomore and said I had to do the foundation that summer. The foundation means the classes that everyone has to take in fundamental subjects like drawing, color, and design. Toward the end of the summer I got a big surprise: a letter from the Accademia, which had been delayed because they'd sent it to Cambridge England instead of Cambridge Massachusetts, inviting me to take the entrance exam in Florence that fall. This was now only weeks away. My nice landlady let me leave my stuff in her attic. I had some money saved from consulting work I'd done in grad school; there was probably enough to last a year if I lived cheaply. Now all I had to do was learn Italian. Only stranieri (foreigners) had to take this entrance exam. In retrospect it may well have been a way of excluding them, because there were so many stranieri attracted by the idea of studying art in Florence that the Italian students would otherwise have been outnumbered. I was in decent shape at painting and drawing from the RISD foundation that summer, but I still don't know how I managed to pass the written exam. I remember that I answered the essay question by writing about Cezanne, and that I cranked up the intellectual level as high as I could to make the most of my limited vocabulary. [2] I'm only up to age 25 and already there are such conspicuous patterns. Here I was, yet again about to attend some august institution in the hopes of learning about some prestigious subject, and yet again about to be disappointed. The students and faculty in the painting department at the Accademia were the nicest people you could imagine, but they had long since arrived at an arrangement whereby the students wouldn't require the faculty to teach anything, and in return the faculty wouldn't require the students to learn anything. And at the same time all involved would adhere outwardly to the conventions of a 19th century atelier. We actually had one of those little stoves, fed with kindling, that you see in 19th century studio paintings, and a nude model sitting as close to it as possible without getting burned. Except hardly anyone else painted her besides me. The rest of the students spent their time chatting or occasionally trying to imitate things they'd seen in American art magazines. Our model turned out to live just down the street from me. She made a living from a combination of modelling and making fakes for a local antique dealer. She'd copy an obscure old painting out of a book, and then he'd take the copy and maltreat it to make it look old. [3] While I was a student at the Accademia I started painting still lives in my bedroom at night. These paintings were tiny, because the room was, and because I painted them on leftover scraps of canvas, which was all I could afford at the time.\n", + "Query Str: What did the author do growing up?\n", + "Answer: wrote short stories, programmed on an IBM 1401 in 9th grade, built a microcomputer from a kit, wrote simple games and a word processor on a TRS-80, studied philosophy in college, switched to AI, and painted still lives at the Accademia di Belli Arti\n", + "\n", + "---\n", + "\n", + "Context Str: YC was different from other kinds of work I've done. Instead of deciding for myself what to work on, the problems came to me. Every 6 months there was a new batch of startups, and their problems, whatever they were, became our problems. It was very engaging work, because their problems were quite varied, and the good founders were very effective. If you were trying to learn the most you could about startups in the shortest possible time, you couldn't have picked a better way to do it. There were parts of the job I didn't like. Disputes between cofounders, figuring out when people were lying to us, fighting with people who maltreated the startups, and so on. But I worked hard even at the parts I didn't like. I was haunted by something Kevin Hale once said about companies: \"No one works harder than the boss.\" He meant it both descriptively and prescriptively, and it was the second part that scared me. I wanted YC to be good, so if how hard I worked set the upper bound on how hard everyone else worked, I'd better work very hard. One day in 2010, when he was visiting California for interviews, Robert Morris did something astonishing: he offered me unsolicited advice. I can only remember him doing that once before. One day at Viaweb, when I was bent over double from a kidney stone, he suggested that it would be a good idea for him to take me to the hospital. That was what it took for Rtm to offer unsolicited advice. So I remember his exact words very clearly. \"You know,\" he said, \"you should make sure Y Combinator isn't the last cool thing you do.\" At the time I didn't understand what he meant, but gradually it dawned on me that he was saying I should quit. This seemed strange advice, because YC was doing great. But if there was one thing rarer than Rtm offering advice, it was Rtm being wrong. So this set me thinking. It was true that on my current trajectory, YC would be the last thing I did, because it was only taking up more of my attention. It had already eaten Arc, and was in the process of eating essays too. Either YC was my life's work or I'd have to leave eventually. And it wasn't, so I would. In the summer of 2012 my mother had a stroke, and the cause turned out to be a blood clot caused by colon cancer. The stroke destroyed her balance, and she was put in a nursing home, but she really wanted to get out of it and back to her house, and my sister and I were determined to help her do it. I used to fly up to Oregon to visit her regularly, and I had a lot of time to think on those flights. On one of them I realized I was ready to hand YC over to someone else. I asked Jessica if she wanted to be president, but she didn't, so we decided we'd try to recruit Sam Altman. We talked to Robert and Trevor and we agreed to make it a complete changing of the guard. Up till that point YC had been controlled by the original LLC we four had started. But we wanted YC to last for a long time, and to do that it couldn't be controlled by the founders. So if Sam said yes, we'd let him reorganize YC. Robert and I would retire, and Jessica and Trevor would become ordinary partners. When we asked Sam if he wanted to be president of YC, initially he said no. He wanted to start a startup to make nuclear reactors. But I kept at it, and in October 2013 he finally agreed. We decided he'd take over starting with the winter 2014 batch. For the rest of 2013 I left running YC more and more to Sam, partly so he could learn the job, and partly because I was focused on my mother, whose cancer had returned. She died on January 15, 2014. We knew this was coming, but it was still hard when it did. I kept working on YC till March, to help get that batch of startups through Demo Day, then I checked out pretty completely. (I still talk to alumni and to new startups working on things I'm interested in, but that only takes a few hours a week.) What should I do next? Rtm's advice hadn't included anything about that. I wanted to do something completely different, so I decided I'd paint. I wanted to see how good I could get if I really focused on it. So the day after I stopped working on YC, I started painting. I don't think it was entirely luck that the first batch was so good. You had to be pretty bold to sign up for a weird thing like the Summer Founders Program instead of a summer job at a legit place like Microsoft or Goldman Sachs. The deal for startups was based on a combination of the deal we did with Julian ($10k for 10%) and what Robert said MIT grad students got for the summer ($6k). We invested $6k per founder, which in the typical two-founder case was $12k, in return for 6%. That had to be fair, because it was twice as good as the deal we ourselves had taken. Plus that first summer, which was really hot, Jessica brought the founders free air conditioners. [16] Fairly quickly I realized that we had stumbled upon the way to scale startup funding. Funding startups in batches was more convenient for us, because it meant we could do things for a lot of startups at once, but being part of a batch was better for the startups too. It solved one of the biggest problems faced by founders: the isolation. Now you not only had colleagues, but colleagues who understood the problems you were facing and could tell you how they were solving them. As YC grew, we started to notice other advantages of scale. The alumni became a tight community, dedicated to helping one another, and especially the current batch, whose shoes they remembered being in. We also noticed that the startups were becoming one another's customers. We used to refer jokingly to the \"YC GDP,\" but as YC grows this becomes less and less of a joke. Now lots of startups get their initial set of customers almost entirely from among their batchmates. I had not originally intended YC to be a full-time job. I was going to do three things: hack, write essays, and work on YC. As YC grew, and I grew more excited about it, it started to take up a lot more than a third of my attention. But for the first few years I was still able to work on other things. In the summer of 2006, Robert and I started working on a new version of Arc. This one was reasonably fast, because it was compiled into Scheme. To test this new Arc, I wrote Hacker News in it. It was originally meant to be a news aggregator for startup founders and was called Startup News, but after a few months I got tired of reading about nothing but startups. Plus it wasn't startup founders we wanted to reach. It was future startup founders. So I changed the name to Hacker News and the topic to whatever engaged one's intellectual curiosity. HN was no doubt good for YC, but it was also by far the biggest source of stress for me. If all I'd had to do was select and help founders, life would have been so easy. And that implies that HN was a mistake. Surely the biggest source of stress in one's work should at least be something close to the core of the work. Whereas I was like someone who was in pain while running a marathon not from the exertion of running, but because I had a blister from an ill-fitting shoe. When I was dealing with some urgent problem during YC, there was about a 60% chance it had to do with HN, and a 40% chance it had do with everything else combined. [17] As well as HN, I wrote all of YC's internal software in Arc. But while I continued to work a good deal in Arc, I gradually stopped working on Arc, partly because I didn't have time to, and partly because it was a lot less attractive to mess around with the language now that we had all this infrastructure depending on it. So now my three projects were reduced to two: writing essays and working on YC. YC was different from other kinds of work I've done. Instead of deciding for myself what to work on, the problems came to me. Every 6 months there was a new batch of startups, and their problems, whatever they were, became our problems. It was very engaging work, because their problems were quite varied, and the good founders were very effective. If you were trying to learn the most you could about startups in the shortest possible time, you couldn't have picked a better way to do it. There were parts of the job I didn't like. Disputes between cofounders, figuring out when people were lying to us, fighting with people who maltreated the startups, and so on. But I worked hard even at the parts I didn't like. I was haunted by something Kevin Hale once said about companies: \"No one works harder than the boss.\"\n", + "Query Str: What did the author do during his time at YC?\n", + "Answer: Worked with startups, solved their problems, wrote essays, and developed internal software.\n", + "\n", + "---\n", + "\n", + "Context Str: this is my context\n", + "Query Str: hello?\n", + "Answer:\n" + ] + } + ], + "source": [ + "print(qa_prompt_tmpl.format(query_str=\"hello?\", context_str=\"this is my context\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "f3097d9c-89e3-4d89-ae8d-6a57bd4fda0b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "query_engine = index.as_query_engine(\n", + " text_qa_template=qa_prompt_tmpl\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "df7b8d58-ddb0-468c-854f-bf475134d653", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "response = query_engine.query(\"what did the author do at RISD?\")" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "2a93b21d-2766-404e-833f-5a1cb3bf8891", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What did the author do after dropping out of RISD?\n", + "Answer: Moved to New York, painted, and wrote a book on Lisp.\n" + ] + } + ], + "source": [ + "print(str(response))" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "id": "5687e7f0-36ec-406f-9427-6047b8ca4fde", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'response_synthesizer:text_qa_template': DSPyPromptTemplate(metadata={}, template_vars=['context_str', 'query_str'], kwargs={}, output_parser=None, template_var_mappings=None, function_mappings=None, predict_module=ChainOfThought(GenerateAnswer(context_str, query_str -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " context_str = Field(annotation=str required=True json_schema_extra={'desc': 'contains relevant facts', '__dspy_field_type': 'input', 'prefix': 'Context Str:'})\n", + " query_str = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Query Str:', 'desc': '${query_str}'})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:'})\n", + " ))),\n", + " 'response_synthesizer:refine_template': SelectorPromptTemplate(metadata={'prompt_type': }, template_vars=['query_str', 'existing_answer', 'context_msg'], kwargs={}, output_parser=None, template_var_mappings={}, function_mappings={}, default_template=PromptTemplate(metadata={'prompt_type': }, template_vars=['query_str', 'existing_answer', 'context_msg'], kwargs={}, output_parser=None, template_var_mappings=None, function_mappings=None, template=\"The original query is as follows: {query_str}\\nWe have provided an existing answer: {existing_answer}\\nWe have the opportunity to refine the existing answer (only if needed) with some more context below.\\n------------\\n{context_msg}\\n------------\\nGiven the new context, refine the original answer to better answer the query. If the context isn't useful, return the original answer.\\nRefined Answer: \"), conditionals=[(, ChatPromptTemplate(metadata={'prompt_type': }, template_vars=['context_msg', 'query_str', 'existing_answer'], kwargs={}, output_parser=None, template_var_mappings=None, function_mappings=None, message_templates=[ChatMessage(role=, content=\"You are an expert Q&A system that strictly operates in two modes when refining existing answers:\\n1. **Rewrite** an original answer using the new context.\\n2. **Repeat** the original answer if the new context isn't useful.\\nNever reference the original answer or context directly in your answer.\\nWhen in doubt, just repeat the original answer.\\nNew Context: {context_msg}\\nQuery: {query_str}\\nOriginal Answer: {existing_answer}\\nNew Answer: \", additional_kwargs={})]))])}" + ] + }, + "execution_count": 48, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "query_engine.get_prompts()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fc2b9755-9cfb-4254-b579-8f0bfaebeb7f", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "self.predict_modules = []\n", + "for prompt in query_engine.get_prompts().keys():\n", + " predict_module = LlamaIndexPredict(global_llm, prompt)\n", + " self.predict_modules.append(predict_module)\n", + " \n", + "def forward(self, **kwargs): \n", + " " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "llama_index_v3", + "language": "python", + "name": "llama_index_v3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 0f85bc3cdf06a6133ed36d69ab7b3e8a640f5333 Mon Sep 17 00:00:00 2001 From: Jerry Liu Date: Tue, 18 Jun 2024 09:32:47 -0700 Subject: [PATCH 2/5] cr --- dspy/predict/llamaindex.py | 39 +----- examples/llamaindex/dspy_llamaindex_rag.ipynb | 124 ++---------------- 2 files changed, 13 insertions(+), 150 deletions(-) diff --git a/dspy/predict/llamaindex.py b/dspy/predict/llamaindex.py index 407124248..e0fe0677c 100644 --- a/dspy/predict/llamaindex.py +++ b/dspy/predict/llamaindex.py @@ -190,9 +190,6 @@ def __init__( predict_module=predict_module, predict_template=signature_to_template(predict_module.signature), ) - # self.predict_module = predict_module - # self.predict_template = signature_to_template(predict_module.signature) - # super().__init__() @classmethod def from_prompt( @@ -230,10 +227,6 @@ async def _arun_component(self, **kwargs: Any) -> Any: # TODO: no async predict module yet return self._run_component(**kwargs) - # def forward(self, *args: Any, **kwargs: Any) -> Any: - # """Forward.""" - # return self._run_component(*args, **kwargs)[self.output_key] - @property def input_keys(self) -> InputKeys: """Input keys.""" @@ -247,11 +240,6 @@ def output_keys(self) -> OutputKeys: return OutputKeys.from_keys(output_keys) -class LlamaIndexModuleMeta(ProgramMeta, type(BaseModel)): - pass - - -# class LlamaIndexModule(dspy.Module, QueryComponent, metaclass=LlamaIndexModuleMeta): class LlamaIndexModule(dspy.Module): """A module for LlamaIndex. @@ -276,29 +264,4 @@ def forward(self, **kwargs: Any) -> Dict[str, Any]: """Forward.""" output_dict = self.query_pipeline.run(**kwargs, return_values_direct=False) return dspy.Prediction(**output_dict) - - # def set_callback_manager(self, callback_manager: CallbackManager) -> None: - # """Set callback manager.""" - # self.query_pipeline.set_callback_manager(callback_manager) - - # def _validate_component_inputs(self, input: Dict[str, Any]) -> Dict[str, Any]: - # """Validate component inputs during run_component.""" - # return self.query_pipeline.validate_component_inputs(input) - - # def _run_component(self, **kwargs: Any) -> Dict: - # """Run component.""" - # return self.query_pipeline.run_component(**kwargs) - - # async def _arun_component(self, **kwargs: Any) -> Any: - # """Run component (async).""" - # return await self.query_pipeline.arun_component(**kwargs) - - # @property - # def input_keys(self) -> InputKeys: - # """Input keys.""" - # return self.query_pipeline.input_keys - - # @property - # def output_keys(self) -> OutputKeys: - # """Output keys.""" - # return self.query_pipeline.output_keys \ No newline at end of file + \ No newline at end of file diff --git a/examples/llamaindex/dspy_llamaindex_rag.ipynb b/examples/llamaindex/dspy_llamaindex_rag.ipynb index 630efaa41..7bd1e2050 100644 --- a/examples/llamaindex/dspy_llamaindex_rag.ipynb +++ b/examples/llamaindex/dspy_llamaindex_rag.ipynb @@ -17,6 +17,16 @@ "3. **Port over DSPy-Optimized Prompts to any LlamaIndex Module**: Possible through our `DSPyPromptTemplate` - translate an optimized prompt through DSPy into any module that requires prompts in LlamaIndex." ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "fa558b8d", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install llama-index==0.10.44" + ] + }, { "cell_type": "markdown", "id": "3d223313-af5b-4155-8896-c24aa4cb6925", @@ -717,78 +727,12 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "id": "e2b1b5f7-66d0-49d2-9bae-ca94f12a124d", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "\n", - "\n", - "Essential Instructions: Provide an answer to the query based solely on the context information provided. The response should be written in the style of a Shakespearean sonnet, which typically consists of 14 lines written in iambic pentameter, with a rhyme scheme of ABABCDCDEFEFGG.\n", - "\n", - "---\n", - "\n", - "Follow the following format.\n", - "\n", - "Context Str: ${context_str}\n", - "Query Str: ${query_str}\n", - "Sonnet Answer: ${sonnet_answer}\n", - "\n", - "---\n", - "\n", - "Context Str: What I Worked On February 2021 Before college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep. The first programs I tried writing were on the IBM 1401 that our school district used for what was then called \"data processing.\" This was in 9th grade, so I was 13 or 14. The school district's 1401 happened to be in the basement of our junior high school, and my friend Rich Draves and I got permission to use it. It was like a mini Bond villain's lair down there, with all these alien-looking machines — CPU, disk drives, printer, card reader — sitting up on a raised floor under bright fluorescent lights. The language we used was an early version of Fortran. You had to type programs on punch cards, then stack them in the card reader and press a button to load the program into memory and run it. The result would ordinarily be to print something on the spectacularly loud printer. I was puzzled by the 1401. I couldn't figure out what to do with it. And in retrospect there's not much I could have done with it. The only form of input to programs was data stored on punched cards, and I didn't have any data stored on punched cards. The only other option was to do things that didn't rely on any input, like calculate approximations of pi, but I didn't know enough math to do anything interesting of that type. So I'm not surprised I can't remember any programs I wrote, because they can't have done much. My clearest memory is of the moment I learned it was possible for programs not to terminate, when one of mine didn't. On a machine without time-sharing, this was a social as well as a technical error, as the data center manager's expression made clear. With microcomputers, everything changed. Now you could have a computer sitting right in front of you, on a desk, that could respond to your keystrokes as it was running instead of just churning through a stack of punch cards and then stopping. [1] The first of my friends to get a microcomputer built it himself. It was sold as a kit by Heathkit. I remember vividly how impressed and envious I felt watching him sitting in front of it, typing programs right into the computer. Computers were expensive in those days and it took me years of nagging before I convinced my father to buy one, a TRS-80, in about 1980. The gold standard then was the Apple II, but a TRS-80 was good enough. This was when I really started programming. I wrote simple games, a program to predict how high my model rockets would fly, and a word processor that my father used to write at least one book. There was only room in memory for about 2 pages of text, so he'd write 2 pages at a time and then print them out, but it was a lot better than a typewriter. Though I liked programming, I didn't plan to study it in college. In college I was going to study philosophy, which sounded much more powerful. It seemed, to my naive high school self, to be the study of the ultimate truths, compared to which the things studied in other fields would be mere domain knowledge. What I discovered when I got to college was that the other fields took up so much of the space of ideas that there wasn't much left for these supposed ultimate truths. All that seemed left for philosophy were edge cases that people in other fields felt could safely be ignored. I couldn't have put this into words when I was 18. All I knew at the time was that I kept taking philosophy courses and they kept being boring. So I decided to switch to AI. AI was in the air in the mid 1980s, but there were two things especially that made me want to work on it: a novel by Heinlein called The Moon is a Harsh Mistress, which featured an intelligent computer called Mike, and a PBS documentary that showed Terry Winograd using SHRDLU. I haven't tried rereading The Moon is a Harsh Mistress, so I don't know how well it has aged, but when I read it I was drawn entirely into its world. It seemed only a matter of time before we'd have Mike, and when I saw Winograd using SHRDLU, it seemed like that time would be a few years at most. All you had to do was teach SHRDLU more words. I didn't want to drop out of grad school, but how else was I going to get out? I remember when my friend Robert Morris got kicked out of Cornell for writing the internet worm of 1988, I was envious that he'd found such a spectacular way to get out of grad school. Then one day in April 1990 a crack appeared in the wall. I ran into professor Cheatham and he asked if I was far enough along to graduate that June. I didn't have a word of my dissertation written, but in what must have been the quickest bit of thinking in my life, I decided to take a shot at writing one in the 5 weeks or so that remained before the deadline, reusing parts of On Lisp where I could, and I was able to respond, with no perceptible delay \"Yes, I think so. I'll give you something to read in a few days.\" I picked applications of continuations as the topic. In retrospect I should have written about macros and embedded languages. There's a whole world there that's barely been explored. But all I wanted was to get out of grad school, and my rapidly written dissertation sufficed, just barely. Meanwhile I was applying to art schools. I applied to two: RISD in the US, and the Accademia di Belli Arti in Florence, which, because it was the oldest art school, I imagined would be good. RISD accepted me, and I never heard back from the Accademia, so off to Providence I went. I'd applied for the BFA program at RISD, which meant in effect that I had to go to college again. This was not as strange as it sounds, because I was only 25, and art schools are full of people of different ages. RISD counted me as a transfer sophomore and said I had to do the foundation that summer. The foundation means the classes that everyone has to take in fundamental subjects like drawing, color, and design. Toward the end of the summer I got a big surprise: a letter from the Accademia, which had been delayed because they'd sent it to Cambridge England instead of Cambridge Massachusetts, inviting me to take the entrance exam in Florence that fall. This was now only weeks away. My nice landlady let me leave my stuff in her attic. I had some money saved from consulting work I'd done in grad school; there was probably enough to last a year if I lived cheaply. Now all I had to do was learn Italian. Only stranieri (foreigners) had to take this entrance exam. In retrospect it may well have been a way of excluding them, because there were so many stranieri attracted by the idea of studying art in Florence that the Italian students would otherwise have been outnumbered. I was in decent shape at painting and drawing from the RISD foundation that summer, but I still don't know how I managed to pass the written exam. I remember that I answered the essay question by writing about Cezanne, and that I cranked up the intellectual level as high as I could to make the most of my limited vocabulary. [2] I'm only up to age 25 and already there are such conspicuous patterns. Here I was, yet again about to attend some august institution in the hopes of learning about some prestigious subject, and yet again about to be disappointed. The students and faculty in the painting department at the Accademia were the nicest people you could imagine, but they had long since arrived at an arrangement whereby the students wouldn't require the faculty to teach anything, and in return the faculty wouldn't require the students to learn anything. And at the same time all involved would adhere outwardly to the conventions of a 19th century atelier. We actually had one of those little stoves, fed with kindling, that you see in 19th century studio paintings, and a nude model sitting as close to it as possible without getting burned. Except hardly anyone else painted her besides me. The rest of the students spent their time chatting or occasionally trying to imitate things they'd seen in American art magazines. Our model turned out to live just down the street from me. She made a living from a combination of modelling and making fakes for a local antique dealer. She'd copy an obscure old painting out of a book, and then he'd take the copy and maltreat it to make it look old. [3] While I was a student at the Accademia I started painting still lives in my bedroom at night. These paintings were tiny, because the room was, and because I painted them on leftover scraps of canvas, which was all I could afford at the time.\n", - "Query Str: What did the author do growing up?\n", - "Sonnet Answer: In youth, the author's path was set to write, With tales of characters and feelings deep, Yet lacking plots, his stories took to flight, In IBM's lair, programming did creep. With punch cards stacked, the Fortran language spoke, A world of data processing unfurled, But lacking input, programs barely woke, 'Til microcomputers changed the world. A TRS-80 sparked the flame of code, Simple games and rockets soared on the screen, Philosophy's allure began to erode, AI's promise in the air was keen. From grad school's grasp, a dissertation penned, Art schools and painting, a new chapter penned.\n", - "\n", - "---\n", - "\n", - "Context Str: YC was different from other kinds of work I've done. Instead of deciding for myself what to work on, the problems came to me. Every 6 months there was a new batch of startups, and their problems, whatever they were, became our problems. It was very engaging work, because their problems were quite varied, and the good founders were very effective. If you were trying to learn the most you could about startups in the shortest possible time, you couldn't have picked a better way to do it. There were parts of the job I didn't like. Disputes between cofounders, figuring out when people were lying to us, fighting with people who maltreated the startups, and so on. But I worked hard even at the parts I didn't like. I was haunted by something Kevin Hale once said about companies: \"No one works harder than the boss.\" He meant it both descriptively and prescriptively, and it was the second part that scared me. I wanted YC to be good, so if how hard I worked set the upper bound on how hard everyone else worked, I'd better work very hard. One day in 2010, when he was visiting California for interviews, Robert Morris did something astonishing: he offered me unsolicited advice. I can only remember him doing that once before. One day at Viaweb, when I was bent over double from a kidney stone, he suggested that it would be a good idea for him to take me to the hospital. That was what it took for Rtm to offer unsolicited advice. So I remember his exact words very clearly. \"You know,\" he said, \"you should make sure Y Combinator isn't the last cool thing you do.\" At the time I didn't understand what he meant, but gradually it dawned on me that he was saying I should quit. This seemed strange advice, because YC was doing great. But if there was one thing rarer than Rtm offering advice, it was Rtm being wrong. So this set me thinking. It was true that on my current trajectory, YC would be the last thing I did, because it was only taking up more of my attention. It had already eaten Arc, and was in the process of eating essays too. Either YC was my life's work or I'd have to leave eventually. And it wasn't, so I would. In the summer of 2012 my mother had a stroke, and the cause turned out to be a blood clot caused by colon cancer. The stroke destroyed her balance, and she was put in a nursing home, but she really wanted to get out of it and back to her house, and my sister and I were determined to help her do it. I used to fly up to Oregon to visit her regularly, and I had a lot of time to think on those flights. On one of them I realized I was ready to hand YC over to someone else. I asked Jessica if she wanted to be president, but she didn't, so we decided we'd try to recruit Sam Altman. We talked to Robert and Trevor and we agreed to make it a complete changing of the guard. Up till that point YC had been controlled by the original LLC we four had started. But we wanted YC to last for a long time, and to do that it couldn't be controlled by the founders. So if Sam said yes, we'd let him reorganize YC. Robert and I would retire, and Jessica and Trevor would become ordinary partners. When we asked Sam if he wanted to be president of YC, initially he said no. He wanted to start a startup to make nuclear reactors. But I kept at it, and in October 2013 he finally agreed. We decided he'd take over starting with the winter 2014 batch. For the rest of 2013 I left running YC more and more to Sam, partly so he could learn the job, and partly because I was focused on my mother, whose cancer had returned. She died on January 15, 2014. We knew this was coming, but it was still hard when it did. I kept working on YC till March, to help get that batch of startups through Demo Day, then I checked out pretty completely. (I still talk to alumni and to new startups working on things I'm interested in, but that only takes a few hours a week.) What should I do next? Rtm's advice hadn't included anything about that. I wanted to do something completely different, so I decided I'd paint. I wanted to see how good I could get if I really focused on it. So the day after I stopped working on YC, I started painting. I don't think it was entirely luck that the first batch was so good. You had to be pretty bold to sign up for a weird thing like the Summer Founders Program instead of a summer job at a legit place like Microsoft or Goldman Sachs. The deal for startups was based on a combination of the deal we did with Julian ($10k for 10%) and what Robert said MIT grad students got for the summer ($6k). We invested $6k per founder, which in the typical two-founder case was $12k, in return for 6%. That had to be fair, because it was twice as good as the deal we ourselves had taken. Plus that first summer, which was really hot, Jessica brought the founders free air conditioners. [16] Fairly quickly I realized that we had stumbled upon the way to scale startup funding. Funding startups in batches was more convenient for us, because it meant we could do things for a lot of startups at once, but being part of a batch was better for the startups too. It solved one of the biggest problems faced by founders: the isolation. Now you not only had colleagues, but colleagues who understood the problems you were facing and could tell you how they were solving them. As YC grew, we started to notice other advantages of scale. The alumni became a tight community, dedicated to helping one another, and especially the current batch, whose shoes they remembered being in. We also noticed that the startups were becoming one another's customers. We used to refer jokingly to the \"YC GDP,\" but as YC grows this becomes less and less of a joke. Now lots of startups get their initial set of customers almost entirely from among their batchmates. I had not originally intended YC to be a full-time job. I was going to do three things: hack, write essays, and work on YC. As YC grew, and I grew more excited about it, it started to take up a lot more than a third of my attention. But for the first few years I was still able to work on other things. In the summer of 2006, Robert and I started working on a new version of Arc. This one was reasonably fast, because it was compiled into Scheme. To test this new Arc, I wrote Hacker News in it. It was originally meant to be a news aggregator for startup founders and was called Startup News, but after a few months I got tired of reading about nothing but startups. Plus it wasn't startup founders we wanted to reach. It was future startup founders. So I changed the name to Hacker News and the topic to whatever engaged one's intellectual curiosity. HN was no doubt good for YC, but it was also by far the biggest source of stress for me. If all I'd had to do was select and help founders, life would have been so easy. And that implies that HN was a mistake. Surely the biggest source of stress in one's work should at least be something close to the core of the work. Whereas I was like someone who was in pain while running a marathon not from the exertion of running, but because I had a blister from an ill-fitting shoe. When I was dealing with some urgent problem during YC, there was about a 60% chance it had to do with HN, and a 40% chance it had do with everything else combined. [17] As well as HN, I wrote all of YC's internal software in Arc. But while I continued to work a good deal in Arc, I gradually stopped working on Arc, partly because I didn't have time to, and partly because it was a lot less attractive to mess around with the language now that we had all this infrastructure depending on it. So now my three projects were reduced to two: writing essays and working on YC. YC was different from other kinds of work I've done. Instead of deciding for myself what to work on, the problems came to me. Every 6 months there was a new batch of startups, and their problems, whatever they were, became our problems. It was very engaging work, because their problems were quite varied, and the good founders were very effective. If you were trying to learn the most you could about startups in the shortest possible time, you couldn't have picked a better way to do it. There were parts of the job I didn't like. Disputes between cofounders, figuring out when people were lying to us, fighting with people who maltreated the startups, and so on. But I worked hard even at the parts I didn't like. I was haunted by something Kevin Hale once said about companies: \"No one works harder than the boss.\"\n", - "Query Str: What did the author do during his time at YC?\n", - "Sonnet Answer: In YC's realm, the author did reside, Where problems of startups did swiftly glide. Each batch brought forth a new set of tasks, Engaging work, where founders wore their masks. Disputes and lies, a part of the game, Yet the author worked hard, not seeking fame. Haunted by words of Kevin Hale's decree, \"No one works harder than the boss,\" said he. In 2012, a shift was in the air, The author decided to leave with care. Handing YC to another's command, To focus on painting, a new strand. Through YC's growth, the author did strive, To learn the most about startups alive.\n", - "\n", - "---\n", - "\n", - "Context Str: Over the next several years I wrote lots of essays about all kinds of different topics. O'Reilly reprinted a collection of them as a book, called Hackers & Painters after one of the essays in it. I also worked on spam filters, and did some more painting. I used to have dinners for a group of friends every thursday night, which taught me how to cook for groups. And I bought another building in Cambridge, a former candy factory (and later, twas said, porn studio), to use as an office. One night in October 2003 there was a big party at my house. It was a clever idea of my friend Maria Daniels, who was one of the thursday diners. Three separate hosts would all invite their friends to one party. So for every guest, two thirds of the other guests would be people they didn't know but would probably like. One of the guests was someone I didn't know but would turn out to like a lot: a woman called Jessica Livingston. A couple days later I asked her out. Jessica was in charge of marketing at a Boston investment bank. This bank thought it understood startups, but over the next year, as she met friends of mine from the startup world, she was surprised how different reality was. And how colorful their stories were. So she decided to compile a book of interviews with startup founders. When the bank had financial problems and she had to fire half her staff, she started looking for a new job. In early 2005 she interviewed for a marketing job at a Boston VC firm. It took them weeks to make up their minds, and during this time I started telling her about all the things that needed to be fixed about venture capital. They should make a larger number of smaller investments instead of a handful of giant ones, they should be funding younger, more technical founders instead of MBAs, they should let the founders remain as CEO, and so on. One of my tricks for writing essays had always been to give talks. The prospect of having to stand up in front of a group of people and tell them something that won't waste their time is a great spur to the imagination. When the Harvard Computer Society, the undergrad computer club, asked me to give a talk, I decided I would tell them how to start a startup. Maybe they'd be able to avoid the worst of the mistakes we'd made. So I gave this talk, in the course of which I told them that the best sources of seed funding were successful startup founders, because then they'd be sources of advice too. Whereupon it seemed they were all looking expectantly at me. Horrified at the prospect of having my inbox flooded by business plans (if I'd only known), I blurted out \"But not me!\" and went on with the talk. But afterward it occurred to me that I should really stop procrastinating about angel investing. I'd been meaning to since Yahoo bought us, and now it was 7 years later and I still hadn't done one angel investment. Meanwhile I had been scheming with Robert and Trevor about projects we could work on together. I missed working with them, and it seemed like there had to be something we could collaborate on. As Jessica and I were walking home from dinner on March 11, at the corner of Garden and Walker streets, these three threads converged. Screw the VCs who were taking so long to make up their minds. We'd start our own investment firm and actually implement the ideas we'd been talking about. I'd fund it, and Jessica could quit her job and work for it, and we'd get Robert and Trevor as partners too. [13] Once again, ignorance worked in our favor. We had no idea how to be angel investors, and in Boston in 2005 there were no Ron Conways to learn from. So we just made what seemed like the obvious choices, and some of the things we did turned out to be novel. There are multiple components to Y Combinator, and we didn't figure them all out at once. The part we got first was to be an angel firm. In those days, those two words didn't go together. There were VC firms, which were organized companies with people whose job it was to make investments, but they only did big, million dollar investments. And there were angels, who did smaller investments, but these were individuals who were usually focused on other things and made investments on the side. And neither of them helped founders enough in the beginning. We knew how helpless founders were in some respects, because we remembered how helpless we'd been. For example, one thing Julian had done for us that seemed to us like magic was to get us set up as a company. Nor had I changed my grad student lifestyle significantly since we started. So when Yahoo bought us it felt like going from rags to riches. Since we were going to California, I bought a car, a yellow 1998 VW GTI. I remember thinking that its leather seats alone were by far the most luxurious thing I owned. The next year, from the summer of 1998 to the summer of 1999, must have been the least productive of my life. I didn't realize it at the time, but I was worn out from the effort and stress of running Viaweb. For a while after I got to California I tried to continue my usual m.o. of programming till 3 in the morning, but fatigue combined with Yahoo's prematurely aged culture and grim cube farm in Santa Clara gradually dragged me down. After a few months it felt disconcertingly like working at Interleaf. Yahoo had given us a lot of options when they bought us. At the time I thought Yahoo was so overvalued that they'd never be worth anything, but to my astonishment the stock went up 5x in the next year. I hung on till the first chunk of options vested, then in the summer of 1999 I left. It had been so long since I'd painted anything that I'd half forgotten why I was doing this. My brain had been entirely full of software and men's shirts for 4 years. But I had done this to get rich so I could paint, I reminded myself, and now I was rich, so I should go paint. When I said I was leaving, my boss at Yahoo had a long conversation with me about my plans. I told him all about the kinds of pictures I wanted to paint. At the time I was touched that he took such an interest in me. Now I realize it was because he thought I was lying. My options at that point were worth about $2 million a month. If I was leaving that kind of money on the table, it could only be to go and start some new startup, and if I did, I might take people with me. This was the height of the Internet Bubble, and Yahoo was ground zero of it. My boss was at that moment a billionaire. Leaving then to start a new startup must have seemed to him an insanely, and yet also plausibly, ambitious plan. But I really was quitting to paint, and I started immediately. There was no time to lose. I'd already burned 4 years getting rich. Now when I talk to founders who are leaving after selling their companies, my advice is always the same: take a vacation. That's what I should have done, just gone off somewhere and done nothing for a month or two, but the idea never occurred to me. So I tried to paint, but I just didn't seem to have any energy or ambition. Part of the problem was that I didn't know many people in California. I'd compounded this problem by buying a house up in the Santa Cruz Mountains, with a beautiful view but miles from anywhere. I stuck it out for a few more months, then in desperation I went back to New York, where unless you understand about rent control you'll be surprised to hear I still had my apartment, sealed up like a tomb of my old life. Idelle was in New York at least, and there were other people trying to paint there, even though I didn't know any of them. When I got back to New York I resumed my old life, except now I was rich. It was as weird as it sounds. I resumed all my old patterns, except now there were doors where there hadn't been. Now when I was tired of walking, all I had to do was raise my hand, and (unless it was raining) a taxi would stop to pick me up. Now when I walked past charming little restaurants I could go in and order lunch. It was exciting for a while. Painting started to go better. I experimented with a new kind of still life where I'd paint one painting in the old way, then photograph it and print it, blown up, on canvas, and then use that as the underpainting for a second still life, painted from the same objects (which hopefully hadn't rotted yet). Meanwhile I looked for an apartment to buy. Now I could actually choose what neighborhood to live in. Where, I asked myself and various real estate agents, is the Cambridge of New York? Aided by occasional visits to actual Cambridge, I gradually realized there wasn't one. Huh. Around this time, in the spring of 2000, I had an idea. It was clear from our experience with Viaweb that web apps were the future.\n", - "Query Str: How did PG meet Jessica Livingston?\n", - "Sonnet Answer:\u001b[32m In the midst of a party, bright and gay,\n", - "A clever scheme brought guests together, true.\n", - "Among them, Jessica, in a charming way,\n", - "Caught the author's eye, a friendship grew.\n", - "\n", - "She, a marketer in a bank of old,\n", - "Discovered startup tales, colorful and bold.\n", - "As the bank faced troubles, she sought anew,\n", - "Venture capital's flaws came into view.\n", - "\n", - "Their paths converged on a fateful night,\n", - "At the corner of Garden and Walker streets.\n", - "A decision made, a future bright,\n", - "To start an investment firm, their feats.\n", - "\n", - "Thus, through ignorance and boldness, they began,\n", - "A journey in angel investing, a novel plan.\u001b[0m\n", - "\n", - "\n", - "\n" - ] - }, - { - "data": { - "text/plain": [ - "'\\n\\n\\nEssential Instructions: Provide an answer to the query based solely on the context information provided. The response should be written in the style of a Shakespearean sonnet, which typically consists of 14 lines written in iambic pentameter, with a rhyme scheme of ABABCDCDEFEFGG.\\n\\n---\\n\\nFollow the following format.\\n\\nContext Str: ${context_str}\\nQuery Str: ${query_str}\\nSonnet Answer: ${sonnet_answer}\\n\\n---\\n\\nContext Str: What I Worked On February 2021 Before college the two main things I worked on, outside of school, were writing and programming. I didn\\'t write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep. The first programs I tried writing were on the IBM 1401 that our school district used for what was then called \"data processing.\" This was in 9th grade, so I was 13 or 14. The school district\\'s 1401 happened to be in the basement of our junior high school, and my friend Rich Draves and I got permission to use it. It was like a mini Bond villain\\'s lair down there, with all these alien-looking machines — CPU, disk drives, printer, card reader — sitting up on a raised floor under bright fluorescent lights. The language we used was an early version of Fortran. You had to type programs on punch cards, then stack them in the card reader and press a button to load the program into memory and run it. The result would ordinarily be to print something on the spectacularly loud printer. I was puzzled by the 1401. I couldn\\'t figure out what to do with it. And in retrospect there\\'s not much I could have done with it. The only form of input to programs was data stored on punched cards, and I didn\\'t have any data stored on punched cards. The only other option was to do things that didn\\'t rely on any input, like calculate approximations of pi, but I didn\\'t know enough math to do anything interesting of that type. So I\\'m not surprised I can\\'t remember any programs I wrote, because they can\\'t have done much. My clearest memory is of the moment I learned it was possible for programs not to terminate, when one of mine didn\\'t. On a machine without time-sharing, this was a social as well as a technical error, as the data center manager\\'s expression made clear. With microcomputers, everything changed. Now you could have a computer sitting right in front of you, on a desk, that could respond to your keystrokes as it was running instead of just churning through a stack of punch cards and then stopping. [1] The first of my friends to get a microcomputer built it himself. It was sold as a kit by Heathkit. I remember vividly how impressed and envious I felt watching him sitting in front of it, typing programs right into the computer. Computers were expensive in those days and it took me years of nagging before I convinced my father to buy one, a TRS-80, in about 1980. The gold standard then was the Apple II, but a TRS-80 was good enough. This was when I really started programming. I wrote simple games, a program to predict how high my model rockets would fly, and a word processor that my father used to write at least one book. There was only room in memory for about 2 pages of text, so he\\'d write 2 pages at a time and then print them out, but it was a lot better than a typewriter. Though I liked programming, I didn\\'t plan to study it in college. In college I was going to study philosophy, which sounded much more powerful. It seemed, to my naive high school self, to be the study of the ultimate truths, compared to which the things studied in other fields would be mere domain knowledge. What I discovered when I got to college was that the other fields took up so much of the space of ideas that there wasn\\'t much left for these supposed ultimate truths. All that seemed left for philosophy were edge cases that people in other fields felt could safely be ignored. I couldn\\'t have put this into words when I was 18. All I knew at the time was that I kept taking philosophy courses and they kept being boring. So I decided to switch to AI. AI was in the air in the mid 1980s, but there were two things especially that made me want to work on it: a novel by Heinlein called The Moon is a Harsh Mistress, which featured an intelligent computer called Mike, and a PBS documentary that showed Terry Winograd using SHRDLU. I haven\\'t tried rereading The Moon is a Harsh Mistress, so I don\\'t know how well it has aged, but when I read it I was drawn entirely into its world. It seemed only a matter of time before we\\'d have Mike, and when I saw Winograd using SHRDLU, it seemed like that time would be a few years at most. All you had to do was teach SHRDLU more words. I didn\\'t want to drop out of grad school, but how else was I going to get out? I remember when my friend Robert Morris got kicked out of Cornell for writing the internet worm of 1988, I was envious that he\\'d found such a spectacular way to get out of grad school. Then one day in April 1990 a crack appeared in the wall. I ran into professor Cheatham and he asked if I was far enough along to graduate that June. I didn\\'t have a word of my dissertation written, but in what must have been the quickest bit of thinking in my life, I decided to take a shot at writing one in the 5 weeks or so that remained before the deadline, reusing parts of On Lisp where I could, and I was able to respond, with no perceptible delay \"Yes, I think so. I\\'ll give you something to read in a few days.\" I picked applications of continuations as the topic. In retrospect I should have written about macros and embedded languages. There\\'s a whole world there that\\'s barely been explored. But all I wanted was to get out of grad school, and my rapidly written dissertation sufficed, just barely. Meanwhile I was applying to art schools. I applied to two: RISD in the US, and the Accademia di Belli Arti in Florence, which, because it was the oldest art school, I imagined would be good. RISD accepted me, and I never heard back from the Accademia, so off to Providence I went. I\\'d applied for the BFA program at RISD, which meant in effect that I had to go to college again. This was not as strange as it sounds, because I was only 25, and art schools are full of people of different ages. RISD counted me as a transfer sophomore and said I had to do the foundation that summer. The foundation means the classes that everyone has to take in fundamental subjects like drawing, color, and design. Toward the end of the summer I got a big surprise: a letter from the Accademia, which had been delayed because they\\'d sent it to Cambridge England instead of Cambridge Massachusetts, inviting me to take the entrance exam in Florence that fall. This was now only weeks away. My nice landlady let me leave my stuff in her attic. I had some money saved from consulting work I\\'d done in grad school; there was probably enough to last a year if I lived cheaply. Now all I had to do was learn Italian. Only stranieri (foreigners) had to take this entrance exam. In retrospect it may well have been a way of excluding them, because there were so many stranieri attracted by the idea of studying art in Florence that the Italian students would otherwise have been outnumbered. I was in decent shape at painting and drawing from the RISD foundation that summer, but I still don\\'t know how I managed to pass the written exam. I remember that I answered the essay question by writing about Cezanne, and that I cranked up the intellectual level as high as I could to make the most of my limited vocabulary. [2] I\\'m only up to age 25 and already there are such conspicuous patterns. Here I was, yet again about to attend some august institution in the hopes of learning about some prestigious subject, and yet again about to be disappointed. The students and faculty in the painting department at the Accademia were the nicest people you could imagine, but they had long since arrived at an arrangement whereby the students wouldn\\'t require the faculty to teach anything, and in return the faculty wouldn\\'t require the students to learn anything. And at the same time all involved would adhere outwardly to the conventions of a 19th century atelier. We actually had one of those little stoves, fed with kindling, that you see in 19th century studio paintings, and a nude model sitting as close to it as possible without getting burned. Except hardly anyone else painted her besides me. The rest of the students spent their time chatting or occasionally trying to imitate things they\\'d seen in American art magazines. Our model turned out to live just down the street from me. She made a living from a combination of modelling and making fakes for a local antique dealer. She\\'d copy an obscure old painting out of a book, and then he\\'d take the copy and maltreat it to make it look old. [3] While I was a student at the Accademia I started painting still lives in my bedroom at night. These paintings were tiny, because the room was, and because I painted them on leftover scraps of canvas, which was all I could afford at the time.\\nQuery Str: What did the author do growing up?\\nSonnet Answer: In youth, the author\\'s path was set to write, With tales of characters and feelings deep, Yet lacking plots, his stories took to flight, In IBM\\'s lair, programming did creep. With punch cards stacked, the Fortran language spoke, A world of data processing unfurled, But lacking input, programs barely woke, \\'Til microcomputers changed the world. A TRS-80 sparked the flame of code, Simple games and rockets soared on the screen, Philosophy\\'s allure began to erode, AI\\'s promise in the air was keen. From grad school\\'s grasp, a dissertation penned, Art schools and painting, a new chapter penned.\\n\\n---\\n\\nContext Str: YC was different from other kinds of work I\\'ve done. Instead of deciding for myself what to work on, the problems came to me. Every 6 months there was a new batch of startups, and their problems, whatever they were, became our problems. It was very engaging work, because their problems were quite varied, and the good founders were very effective. If you were trying to learn the most you could about startups in the shortest possible time, you couldn\\'t have picked a better way to do it. There were parts of the job I didn\\'t like. Disputes between cofounders, figuring out when people were lying to us, fighting with people who maltreated the startups, and so on. But I worked hard even at the parts I didn\\'t like. I was haunted by something Kevin Hale once said about companies: \"No one works harder than the boss.\" He meant it both descriptively and prescriptively, and it was the second part that scared me. I wanted YC to be good, so if how hard I worked set the upper bound on how hard everyone else worked, I\\'d better work very hard. One day in 2010, when he was visiting California for interviews, Robert Morris did something astonishing: he offered me unsolicited advice. I can only remember him doing that once before. One day at Viaweb, when I was bent over double from a kidney stone, he suggested that it would be a good idea for him to take me to the hospital. That was what it took for Rtm to offer unsolicited advice. So I remember his exact words very clearly. \"You know,\" he said, \"you should make sure Y Combinator isn\\'t the last cool thing you do.\" At the time I didn\\'t understand what he meant, but gradually it dawned on me that he was saying I should quit. This seemed strange advice, because YC was doing great. But if there was one thing rarer than Rtm offering advice, it was Rtm being wrong. So this set me thinking. It was true that on my current trajectory, YC would be the last thing I did, because it was only taking up more of my attention. It had already eaten Arc, and was in the process of eating essays too. Either YC was my life\\'s work or I\\'d have to leave eventually. And it wasn\\'t, so I would. In the summer of 2012 my mother had a stroke, and the cause turned out to be a blood clot caused by colon cancer. The stroke destroyed her balance, and she was put in a nursing home, but she really wanted to get out of it and back to her house, and my sister and I were determined to help her do it. I used to fly up to Oregon to visit her regularly, and I had a lot of time to think on those flights. On one of them I realized I was ready to hand YC over to someone else. I asked Jessica if she wanted to be president, but she didn\\'t, so we decided we\\'d try to recruit Sam Altman. We talked to Robert and Trevor and we agreed to make it a complete changing of the guard. Up till that point YC had been controlled by the original LLC we four had started. But we wanted YC to last for a long time, and to do that it couldn\\'t be controlled by the founders. So if Sam said yes, we\\'d let him reorganize YC. Robert and I would retire, and Jessica and Trevor would become ordinary partners. When we asked Sam if he wanted to be president of YC, initially he said no. He wanted to start a startup to make nuclear reactors. But I kept at it, and in October 2013 he finally agreed. We decided he\\'d take over starting with the winter 2014 batch. For the rest of 2013 I left running YC more and more to Sam, partly so he could learn the job, and partly because I was focused on my mother, whose cancer had returned. She died on January 15, 2014. We knew this was coming, but it was still hard when it did. I kept working on YC till March, to help get that batch of startups through Demo Day, then I checked out pretty completely. (I still talk to alumni and to new startups working on things I\\'m interested in, but that only takes a few hours a week.) What should I do next? Rtm\\'s advice hadn\\'t included anything about that. I wanted to do something completely different, so I decided I\\'d paint. I wanted to see how good I could get if I really focused on it. So the day after I stopped working on YC, I started painting. I don\\'t think it was entirely luck that the first batch was so good. You had to be pretty bold to sign up for a weird thing like the Summer Founders Program instead of a summer job at a legit place like Microsoft or Goldman Sachs. The deal for startups was based on a combination of the deal we did with Julian ($10k for 10%) and what Robert said MIT grad students got for the summer ($6k). We invested $6k per founder, which in the typical two-founder case was $12k, in return for 6%. That had to be fair, because it was twice as good as the deal we ourselves had taken. Plus that first summer, which was really hot, Jessica brought the founders free air conditioners. [16] Fairly quickly I realized that we had stumbled upon the way to scale startup funding. Funding startups in batches was more convenient for us, because it meant we could do things for a lot of startups at once, but being part of a batch was better for the startups too. It solved one of the biggest problems faced by founders: the isolation. Now you not only had colleagues, but colleagues who understood the problems you were facing and could tell you how they were solving them. As YC grew, we started to notice other advantages of scale. The alumni became a tight community, dedicated to helping one another, and especially the current batch, whose shoes they remembered being in. We also noticed that the startups were becoming one another\\'s customers. We used to refer jokingly to the \"YC GDP,\" but as YC grows this becomes less and less of a joke. Now lots of startups get their initial set of customers almost entirely from among their batchmates. I had not originally intended YC to be a full-time job. I was going to do three things: hack, write essays, and work on YC. As YC grew, and I grew more excited about it, it started to take up a lot more than a third of my attention. But for the first few years I was still able to work on other things. In the summer of 2006, Robert and I started working on a new version of Arc. This one was reasonably fast, because it was compiled into Scheme. To test this new Arc, I wrote Hacker News in it. It was originally meant to be a news aggregator for startup founders and was called Startup News, but after a few months I got tired of reading about nothing but startups. Plus it wasn\\'t startup founders we wanted to reach. It was future startup founders. So I changed the name to Hacker News and the topic to whatever engaged one\\'s intellectual curiosity. HN was no doubt good for YC, but it was also by far the biggest source of stress for me. If all I\\'d had to do was select and help founders, life would have been so easy. And that implies that HN was a mistake. Surely the biggest source of stress in one\\'s work should at least be something close to the core of the work. Whereas I was like someone who was in pain while running a marathon not from the exertion of running, but because I had a blister from an ill-fitting shoe. When I was dealing with some urgent problem during YC, there was about a 60% chance it had to do with HN, and a 40% chance it had do with everything else combined. [17] As well as HN, I wrote all of YC\\'s internal software in Arc. But while I continued to work a good deal in Arc, I gradually stopped working on Arc, partly because I didn\\'t have time to, and partly because it was a lot less attractive to mess around with the language now that we had all this infrastructure depending on it. So now my three projects were reduced to two: writing essays and working on YC. YC was different from other kinds of work I\\'ve done. Instead of deciding for myself what to work on, the problems came to me. Every 6 months there was a new batch of startups, and their problems, whatever they were, became our problems. It was very engaging work, because their problems were quite varied, and the good founders were very effective. If you were trying to learn the most you could about startups in the shortest possible time, you couldn\\'t have picked a better way to do it. There were parts of the job I didn\\'t like. Disputes between cofounders, figuring out when people were lying to us, fighting with people who maltreated the startups, and so on. But I worked hard even at the parts I didn\\'t like. I was haunted by something Kevin Hale once said about companies: \"No one works harder than the boss.\"\\nQuery Str: What did the author do during his time at YC?\\nSonnet Answer: In YC\\'s realm, the author did reside, Where problems of startups did swiftly glide. Each batch brought forth a new set of tasks, Engaging work, where founders wore their masks. Disputes and lies, a part of the game, Yet the author worked hard, not seeking fame. Haunted by words of Kevin Hale\\'s decree, \"No one works harder than the boss,\" said he. In 2012, a shift was in the air, The author decided to leave with care. Handing YC to another\\'s command, To focus on painting, a new strand. Through YC\\'s growth, the author did strive, To learn the most about startups alive.\\n\\n---\\n\\nContext Str: Over the next several years I wrote lots of essays about all kinds of different topics. O\\'Reilly reprinted a collection of them as a book, called Hackers & Painters after one of the essays in it. I also worked on spam filters, and did some more painting. I used to have dinners for a group of friends every thursday night, which taught me how to cook for groups. And I bought another building in Cambridge, a former candy factory (and later, twas said, porn studio), to use as an office. One night in October 2003 there was a big party at my house. It was a clever idea of my friend Maria Daniels, who was one of the thursday diners. Three separate hosts would all invite their friends to one party. So for every guest, two thirds of the other guests would be people they didn\\'t know but would probably like. One of the guests was someone I didn\\'t know but would turn out to like a lot: a woman called Jessica Livingston. A couple days later I asked her out. Jessica was in charge of marketing at a Boston investment bank. This bank thought it understood startups, but over the next year, as she met friends of mine from the startup world, she was surprised how different reality was. And how colorful their stories were. So she decided to compile a book of interviews with startup founders. When the bank had financial problems and she had to fire half her staff, she started looking for a new job. In early 2005 she interviewed for a marketing job at a Boston VC firm. It took them weeks to make up their minds, and during this time I started telling her about all the things that needed to be fixed about venture capital. They should make a larger number of smaller investments instead of a handful of giant ones, they should be funding younger, more technical founders instead of MBAs, they should let the founders remain as CEO, and so on. One of my tricks for writing essays had always been to give talks. The prospect of having to stand up in front of a group of people and tell them something that won\\'t waste their time is a great spur to the imagination. When the Harvard Computer Society, the undergrad computer club, asked me to give a talk, I decided I would tell them how to start a startup. Maybe they\\'d be able to avoid the worst of the mistakes we\\'d made. So I gave this talk, in the course of which I told them that the best sources of seed funding were successful startup founders, because then they\\'d be sources of advice too. Whereupon it seemed they were all looking expectantly at me. Horrified at the prospect of having my inbox flooded by business plans (if I\\'d only known), I blurted out \"But not me!\" and went on with the talk. But afterward it occurred to me that I should really stop procrastinating about angel investing. I\\'d been meaning to since Yahoo bought us, and now it was 7 years later and I still hadn\\'t done one angel investment. Meanwhile I had been scheming with Robert and Trevor about projects we could work on together. I missed working with them, and it seemed like there had to be something we could collaborate on. As Jessica and I were walking home from dinner on March 11, at the corner of Garden and Walker streets, these three threads converged. Screw the VCs who were taking so long to make up their minds. We\\'d start our own investment firm and actually implement the ideas we\\'d been talking about. I\\'d fund it, and Jessica could quit her job and work for it, and we\\'d get Robert and Trevor as partners too. [13] Once again, ignorance worked in our favor. We had no idea how to be angel investors, and in Boston in 2005 there were no Ron Conways to learn from. So we just made what seemed like the obvious choices, and some of the things we did turned out to be novel. There are multiple components to Y Combinator, and we didn\\'t figure them all out at once. The part we got first was to be an angel firm. In those days, those two words didn\\'t go together. There were VC firms, which were organized companies with people whose job it was to make investments, but they only did big, million dollar investments. And there were angels, who did smaller investments, but these were individuals who were usually focused on other things and made investments on the side. And neither of them helped founders enough in the beginning. We knew how helpless founders were in some respects, because we remembered how helpless we\\'d been. For example, one thing Julian had done for us that seemed to us like magic was to get us set up as a company. Nor had I changed my grad student lifestyle significantly since we started. So when Yahoo bought us it felt like going from rags to riches. Since we were going to California, I bought a car, a yellow 1998 VW GTI. I remember thinking that its leather seats alone were by far the most luxurious thing I owned. The next year, from the summer of 1998 to the summer of 1999, must have been the least productive of my life. I didn\\'t realize it at the time, but I was worn out from the effort and stress of running Viaweb. For a while after I got to California I tried to continue my usual m.o. of programming till 3 in the morning, but fatigue combined with Yahoo\\'s prematurely aged culture and grim cube farm in Santa Clara gradually dragged me down. After a few months it felt disconcertingly like working at Interleaf. Yahoo had given us a lot of options when they bought us. At the time I thought Yahoo was so overvalued that they\\'d never be worth anything, but to my astonishment the stock went up 5x in the next year. I hung on till the first chunk of options vested, then in the summer of 1999 I left. It had been so long since I\\'d painted anything that I\\'d half forgotten why I was doing this. My brain had been entirely full of software and men\\'s shirts for 4 years. But I had done this to get rich so I could paint, I reminded myself, and now I was rich, so I should go paint. When I said I was leaving, my boss at Yahoo had a long conversation with me about my plans. I told him all about the kinds of pictures I wanted to paint. At the time I was touched that he took such an interest in me. Now I realize it was because he thought I was lying. My options at that point were worth about $2 million a month. If I was leaving that kind of money on the table, it could only be to go and start some new startup, and if I did, I might take people with me. This was the height of the Internet Bubble, and Yahoo was ground zero of it. My boss was at that moment a billionaire. Leaving then to start a new startup must have seemed to him an insanely, and yet also plausibly, ambitious plan. But I really was quitting to paint, and I started immediately. There was no time to lose. I\\'d already burned 4 years getting rich. Now when I talk to founders who are leaving after selling their companies, my advice is always the same: take a vacation. That\\'s what I should have done, just gone off somewhere and done nothing for a month or two, but the idea never occurred to me. So I tried to paint, but I just didn\\'t seem to have any energy or ambition. Part of the problem was that I didn\\'t know many people in California. I\\'d compounded this problem by buying a house up in the Santa Cruz Mountains, with a beautiful view but miles from anywhere. I stuck it out for a few more months, then in desperation I went back to New York, where unless you understand about rent control you\\'ll be surprised to hear I still had my apartment, sealed up like a tomb of my old life. Idelle was in New York at least, and there were other people trying to paint there, even though I didn\\'t know any of them. When I got back to New York I resumed my old life, except now I was rich. It was as weird as it sounds. I resumed all my old patterns, except now there were doors where there hadn\\'t been. Now when I was tired of walking, all I had to do was raise my hand, and (unless it was raining) a taxi would stop to pick me up. Now when I walked past charming little restaurants I could go in and order lunch. It was exciting for a while. Painting started to go better. I experimented with a new kind of still life where I\\'d paint one painting in the old way, then photograph it and print it, blown up, on canvas, and then use that as the underpainting for a second still life, painted from the same objects (which hopefully hadn\\'t rotted yet). Meanwhile I looked for an apartment to buy. Now I could actually choose what neighborhood to live in. Where, I asked myself and various real estate agents, is the Cambridge of New York? Aided by occasional visits to actual Cambridge, I gradually realized there wasn\\'t one. Huh. Around this time, in the spring of 2000, I had an idea. It was clear from our experience with Viaweb that web apps were the future.\\nQuery Str: How did PG meet Jessica Livingston?\\nSonnet Answer:\\x1b[32m In the midst of a party, bright and gay,\\nA clever scheme brought guests together, true.\\nAmong them, Jessica, in a charming way,\\nCaught the author\\'s eye, a friendship grew.\\n\\nShe, a marketer in a bank of old,\\nDiscovered startup tales, colorful and bold.\\nAs the bank faced troubles, she sought anew,\\nVenture capital\\'s flaws came into view.\\n\\nTheir paths converged on a fateful night,\\nAt the corner of Garden and Walker streets.\\nA decision made, a future bright,\\nTo start an investment firm, their feats.\\n\\nThus, through ignorance and boldness, they began,\\nA journey in angel investing, a novel plan.\\x1b[0m\\n\\n\\n'" - ] - }, - "execution_count": 43, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# [optional]: inspect the optimized prompt \n", "turbo.inspect_history(n=1)" @@ -915,50 +859,6 @@ "source": [ "print(str(response))" ] - }, - { - "cell_type": "code", - "execution_count": 48, - "id": "5687e7f0-36ec-406f-9427-6047b8ca4fde", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'response_synthesizer:text_qa_template': DSPyPromptTemplate(metadata={}, template_vars=['context_str', 'query_str'], kwargs={}, output_parser=None, template_var_mappings=None, function_mappings=None, predict_module=ChainOfThought(GenerateAnswer(context_str, query_str -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " context_str = Field(annotation=str required=True json_schema_extra={'desc': 'contains relevant facts', '__dspy_field_type': 'input', 'prefix': 'Context Str:'})\n", - " query_str = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Query Str:', 'desc': '${query_str}'})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:'})\n", - " ))),\n", - " 'response_synthesizer:refine_template': SelectorPromptTemplate(metadata={'prompt_type': }, template_vars=['query_str', 'existing_answer', 'context_msg'], kwargs={}, output_parser=None, template_var_mappings={}, function_mappings={}, default_template=PromptTemplate(metadata={'prompt_type': }, template_vars=['query_str', 'existing_answer', 'context_msg'], kwargs={}, output_parser=None, template_var_mappings=None, function_mappings=None, template=\"The original query is as follows: {query_str}\\nWe have provided an existing answer: {existing_answer}\\nWe have the opportunity to refine the existing answer (only if needed) with some more context below.\\n------------\\n{context_msg}\\n------------\\nGiven the new context, refine the original answer to better answer the query. If the context isn't useful, return the original answer.\\nRefined Answer: \"), conditionals=[(, ChatPromptTemplate(metadata={'prompt_type': }, template_vars=['context_msg', 'query_str', 'existing_answer'], kwargs={}, output_parser=None, template_var_mappings=None, function_mappings=None, message_templates=[ChatMessage(role=, content=\"You are an expert Q&A system that strictly operates in two modes when refining existing answers:\\n1. **Rewrite** an original answer using the new context.\\n2. **Repeat** the original answer if the new context isn't useful.\\nNever reference the original answer or context directly in your answer.\\nWhen in doubt, just repeat the original answer.\\nNew Context: {context_msg}\\nQuery: {query_str}\\nOriginal Answer: {existing_answer}\\nNew Answer: \", additional_kwargs={})]))])}" - ] - }, - "execution_count": 48, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "query_engine.get_prompts()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fc2b9755-9cfb-4254-b579-8f0bfaebeb7f", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "self.predict_modules = []\n", - "for prompt in query_engine.get_prompts().keys():\n", - " predict_module = LlamaIndexPredict(global_llm, prompt)\n", - " self.predict_modules.append(predict_module)\n", - " \n", - "def forward(self, **kwargs): \n", - " " - ] } ], "metadata": { From 07d8e1d1ee5dc43be09e9ff009251f196fb57cc0 Mon Sep 17 00:00:00 2001 From: arnavsinghvi11 <54859892+arnavsinghvi11@users.noreply.github.com> Date: Wed, 19 Jun 2024 12:15:58 -0700 Subject: [PATCH 3/5] Update llamaindex.py ruff fix --- dspy/predict/llamaindex.py | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/dspy/predict/llamaindex.py b/dspy/predict/llamaindex.py index e0fe0677c..d5c000250 100644 --- a/dspy/predict/llamaindex.py +++ b/dspy/predict/llamaindex.py @@ -1,26 +1,22 @@ -from llama_index.core.prompts import BasePromptTemplate -from dspy import Predict -import dspy -from abc import abstractmethod -from typing import Any, Optional, List, Dict, Callable +import re +from copy import deepcopy +from typing import Any, Callable, Dict, List, Optional + from llama_index.core.base.llms.base import BaseLLM -from llama_index.core.base.llms.types import ChatMessage from llama_index.core.base.llms.generic_utils import ( prompt_to_messages, ) -from llama_index.core.base.query_pipeline.query import QueryComponent, InputKeys, OutputKeys -from llama_index.core.query_pipeline import QueryPipeline -from dspy.signatures.signature import ensure_signature, signature_to_template, infer_prefix, make_signature -from dspy.signatures.field import InputField, OutputField -from dspy.primitives import ProgramMeta -import dsp -from copy import deepcopy -import re +from llama_index.core.base.llms.types import ChatMessage +from llama_index.core.base.query_pipeline.query import InputKeys, OutputKeys, QueryComponent from llama_index.core.callbacks.base import CallbackManager -from llama_index.core.bridge.pydantic import BaseModel, create_model -from llama_index.core.prompts import PromptTemplate - +from llama_index.core.prompts import BasePromptTemplate, PromptTemplate +from llama_index.core.query_pipeline import QueryPipeline +import dsp +import dspy +from dspy import Predict +from dspy.signatures.field import InputField, OutputField +from dspy.signatures.signature import ensure_signature, make_signature, signature_to_template def get_formatted_template(predict_module: Predict, kwargs: Dict[str, Any]) -> str: @@ -78,7 +74,7 @@ def __init__( metadata: Optional[Dict[str, Any]] = None, template_var_mappings: Optional[Dict[str, Any]] = None, function_mappings: Optional[Dict[str, Callable]] = None, - **kwargs: Any + **kwargs: Any, ) -> None: template = signature_to_template(predict_module.signature) template_vars = _input_keys_from_template(template) @@ -116,7 +112,7 @@ def format(self, llm: Optional[BaseLLM] = None, **kwargs: Any) -> str: return get_formatted_template(self.predict_module, mapped_kwargs) def format_messages( - self, llm: Optional[BaseLLM] = None, **kwargs: Any + self, llm: Optional[BaseLLM] = None, **kwargs: Any, ) -> List[ChatMessage]: """Formats the prompt template into chat messages.""" del llm # unused @@ -126,7 +122,7 @@ def format_messages( def get_template(self, llm: Optional[BaseLLM] = None) -> str: """Get template.""" # get kwarg templates - kwarg_tmpl_map = {k: f"{{k}}" for k in self.template_vars} + kwarg_tmpl_map = {k: "{k}" for k in self.template_vars} # get "raw" template with all the values filled in with {var_name} template0 = get_formatted_template(self.predict_module, kwarg_tmpl_map) @@ -264,4 +260,4 @@ def forward(self, **kwargs: Any) -> Dict[str, Any]: """Forward.""" output_dict = self.query_pipeline.run(**kwargs, return_values_direct=False) return dspy.Prediction(**output_dict) - \ No newline at end of file + From 95547f3744400e28a1ea8de9755d4335f74818aa Mon Sep 17 00:00:00 2001 From: Jerry Liu Date: Wed, 19 Jun 2024 17:56:47 -0700 Subject: [PATCH 4/5] cr --- dspy/predict/__init__.py | 1 + dspy/predict/llamaindex.py | 4 +-- examples/llamaindex/dspy_llamaindex_rag.ipynb | 25 +++---------------- 3 files changed, 6 insertions(+), 24 deletions(-) diff --git a/dspy/predict/__init__.py b/dspy/predict/__init__.py index b176d2a0e..2f6397bb6 100644 --- a/dspy/predict/__init__.py +++ b/dspy/predict/__init__.py @@ -7,3 +7,4 @@ from .program_of_thought import ProgramOfThought from .react import ReAct from .retry import Retry +from .llamaindex import LlamaIndexModule diff --git a/dspy/predict/llamaindex.py b/dspy/predict/llamaindex.py index d5c000250..6b120437e 100644 --- a/dspy/predict/llamaindex.py +++ b/dspy/predict/llamaindex.py @@ -50,8 +50,8 @@ def _input_keys_from_template(template: dsp.Template) -> InputKeys: ] def _output_keys_from_template(template: dsp.Template) -> InputKeys: - """Get input keys from template.""" - # get only fields that are marked OldInputField and NOT OldOutputField + """Get output keys from template.""" + # get only fields that are marked OldOutputField and NOT OldInputField # template_vars = list(template.kwargs.keys()) return [ k for k, v in template.kwargs.items() if isinstance(v, dspy.signatures.OldOutputField) diff --git a/examples/llamaindex/dspy_llamaindex_rag.ipynb b/examples/llamaindex/dspy_llamaindex_rag.ipynb index 7bd1e2050..1dfb6aaef 100644 --- a/examples/llamaindex/dspy_llamaindex_rag.ipynb +++ b/examples/llamaindex/dspy_llamaindex_rag.ipynb @@ -170,7 +170,7 @@ "outputs": [], "source": [ "from llama_index.core.query_pipeline import QueryPipeline as QP, InputComponent, FnComponent\n", - "from llamaindex import DSPyComponent, LlamaIndexModule\n", + "from dspy.predict.llamaindex import DSPyComponent, LlamaIndexModule\n", "\n", "dspy_component = DSPyComponent(\n", " dspy.ChainOfThought(GenerateAnswer)\n", @@ -455,25 +455,6 @@ "turbo.inspect_history(n=1)" ] }, - { - "cell_type": "code", - "execution_count": 25, - "id": "ab29598c-2bf5-4472-ab13-43772e678a9f", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# from dspy.signatures.signature import signature_to_template\n", - "# from llamaindex import DSPyPromptTemplate\n", - "\n", - "# # tmpl = signature_to_template(compiled_dspy_qp.query_pipeline.module_dict[\"synthesizer\"].predict_module.signature)\n", - "\n", - "# # qa_prompt_tmpl = DSPyPromptTemplate(RAG().generate_answer)\n", - "# qa_prompt_tmpl = DSPyPromptTemplate(compiled_dspy_qp.query_pipeline.module_dict[\"synthesizer\"].predict_module)\n", - "# print(qa_prompt_tmpl.format(query_str=\"hello?\", context_str=\"this is my context\"))" - ] - }, { "cell_type": "markdown", "id": "9895ceff-5aeb-4285-8ede-99bb35a50a45", @@ -523,7 +504,7 @@ "outputs": [], "source": [ "from llama_index.core.query_pipeline import QueryPipeline as QP, InputComponent, FnComponent\n", - "from llamaindex import DSPyComponent, LlamaIndexModule\n", + "from dspy.predict.llamaindex import DSPyComponent, LlamaIndexModule\n", "\n", "dspy_component = DSPyComponent.from_prompt(qa_prompt_template)\n", "\n", @@ -761,7 +742,7 @@ }, "outputs": [], "source": [ - "from llamaindex import DSPyPromptTemplate\n", + "from dspy.predict.llamaindex import DSPyPromptTemplate\n", "\n", "# NOTE: you cannot do DSPyPromptTemplate(dspy_component.predict_module) - the predict_module is replaced.\n", "qa_prompt_tmpl = DSPyPromptTemplate(compiled_dspy_qp.query_pipeline.module_dict[\"synthesizer\"].predict_module)" From e6373de484da7cc808134ca086a82040446655fd Mon Sep 17 00:00:00 2001 From: Arnav Singhvi Date: Thu, 20 Jun 2024 18:57:24 -0700 Subject: [PATCH 5/5] avoid llama_index dependency breaking tests --- dspy/predict/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dspy/predict/__init__.py b/dspy/predict/__init__.py index 2f6397bb6..b176d2a0e 100644 --- a/dspy/predict/__init__.py +++ b/dspy/predict/__init__.py @@ -7,4 +7,3 @@ from .program_of_thought import ProgramOfThought from .react import ReAct from .retry import Retry -from .llamaindex import LlamaIndexModule