From 6e04993c3026b4501cb5db3e3aefca32444809cc Mon Sep 17 00:00:00 2001 From: hnyls2002 Date: Thu, 1 Feb 2024 04:06:19 +0000 Subject: [PATCH 1/6] upload build_dataset --- benchmark/json_fast_forward/build_dataset.py | 58 ++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 benchmark/json_fast_forward/build_dataset.py diff --git a/benchmark/json_fast_forward/build_dataset.py b/benchmark/json_fast_forward/build_dataset.py new file mode 100644 index 00000000000..1396e5edecc --- /dev/null +++ b/benchmark/json_fast_forward/build_dataset.py @@ -0,0 +1,58 @@ +import json + +import transformers +import wikipedia + +model_path = "meta-llama/Llama-2-7b-chat-hf" +t = transformers.AutoTokenizer.from_pretrained(model_path) +city_names = [ + "los angles", + "london", + "tokyo", + "beijing", + "singapore", + "paris", + "dubai", + "sydney", + "moscow", + "rome", + "toronto", + "rio de janeiro", + "istanbul", + "berlin", + "auckland", + "buenos aires", + "mexico city", + "mumbai", + "seoul", + "bangkok", + "cairo", + "athens", + "jerusalem", +] + + +def get_content(city_name): + content = str(wikipedia.page(city_name).content) + content = content.replace("\n\n", "\n") + + tokens = t.encode(content) + + expected_tokens = 3000 + truncate_len = int((expected_tokens / len(tokens)) * len(content)) + truncate_content = content[:truncate_len] + truncate_tokens = t.encode(truncate_content) + + # Count token + print( + f"city_name: {city_name}, #tokens: {len(tokens)}, #truncate tokens: {len(truncate_tokens)}" + ) + + return truncate_content + + +if __name__ == "__main__": + with open("questions.jsonl", "w") as fout: + for city_name in city_names: + truncate_content = get_content(city_name) + fout.write(json.dumps({"document": truncate_content}) + "\n") From eb3835b503eb8caa0633d0e0c0eef5f404218429 Mon Sep 17 00:00:00 2001 From: hnyls2002 Date: Thu, 1 Feb 2024 04:07:06 +0000 Subject: [PATCH 2/6] update with bench_sglang --- benchmark/json_fast_forward/bench_sglang.py | 58 +++++++++++++++++++-- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/benchmark/json_fast_forward/bench_sglang.py b/benchmark/json_fast_forward/bench_sglang.py index d0ef786fbae..ebe2153a729 100644 --- a/benchmark/json_fast_forward/bench_sglang.py +++ b/benchmark/json_fast_forward/bench_sglang.py @@ -7,7 +7,8 @@ add_common_sglang_args_and_parse, select_sglang_backend, ) -from sglang.utils import dump_state_text +from sglang.utils import dump_state_text, read_jsonl + # there are some FSM bugs with json regex converted from pydantic model # here use a string regex instead @@ -29,6 +30,16 @@ + r"""\}""" ) +city_regex = ( + r"""\{\n""" + + r""" "name": "[\w\d\s]{1,16}",\n""" + + r""" "country": "[\w\d\s]{1,16}",\n""" + + r""" "latitude": [-+]?[0-9]*\.?[0-9]{0,2},\n""" + + r""" "population": [-+]?[0-9]{1,9},\n""" + + r""" "top 3 landmarks": \["[\w\d\s]{1,16}", "[\w\d\s]{1,16}", "[\w\d\s]{1,16}"\]\n""" + + r"""\}""" +) + # fmt: off @sgl.function def character_gen(s, name): @@ -36,6 +47,38 @@ def character_gen(s, name): s += sgl.gen("json_output", max_tokens=256, regex=character_regex) # fmt: on +# fmt: off +@sgl.function +def city_gen(s, document): + s += "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + s += sgl.gen("json_output",max_tokens=256, regex=city_regex) +# fmt: on + + +def bench_city_doc(args): + arguments = [] + for line in read_jsonl(args.data_path): + arguments.append({"document": line["document"]}) + arguments = arguments[: args.num_jsons] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.time() + states = city_gen.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=(args.parallel == 1), + ) + latency = time.time() - tic + + return states, latency + def bench_character(args): arguments = [] @@ -62,14 +105,19 @@ def bench_character(args): def main(args): - states, latency = bench_character(args) + if args.mode == "character": + args.data_path = "dataset.txt" + states, latency = bench_character(args) + elif args.mode == "city": + args.data_path = "questions.jsonl" + states, latency = bench_city_doc(args) # Compute accuracy print(f"Latency: {latency:.3f}") # Write results dump_state_text(f"tmp_output_{args.backend}.txt", states) - with open(f"{args.backend}.json", "w") as fout: + with open(f"{args.backend}_{args.mode}.json", "w") as fout: for state in states: fout.write(state["json_output"] + "\n") @@ -79,6 +127,7 @@ def main(args): "backend": args.backend, "latency": round(latency, 3), "num_jsons": args.num_jsons, + "mode": args.mode, "parallel": args.parallel, } fout.write(json.dumps(value) + "\n") @@ -86,7 +135,8 @@ def main(args): if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--data-path", type=str, default="dataset.txt") + parser.add_argument("--data-path", type=str) parser.add_argument("--num-jsons", type=int, default=50) + parser.add_argument("--mode", type=str, default="character", choices=["character", "city"]) args = add_common_sglang_args_and_parse(parser) main(args) From fb3b555822df1d88b59a4a8022a2e492e1fe518e Mon Sep 17 00:00:00 2001 From: hnyls2002 Date: Thu, 1 Feb 2024 05:06:31 +0000 Subject: [PATCH 3/6] update bench_other --- benchmark/json_fast_forward/bench_other.py | 117 +++++++++++++++++++- benchmark/json_fast_forward/bench_sglang.py | 5 +- 2 files changed, 116 insertions(+), 6 deletions(-) diff --git a/benchmark/json_fast_forward/bench_other.py b/benchmark/json_fast_forward/bench_other.py index 7052c148a93..c8126fec1a4 100644 --- a/benchmark/json_fast_forward/bench_other.py +++ b/benchmark/json_fast_forward/bench_other.py @@ -9,7 +9,7 @@ add_common_other_args_and_parse, call_generate_outlines, ) -from sglang.utils import dump_state_text +from sglang.utils import dump_state_text, read_jsonl from tqdm import tqdm # there are some FSM bugs with json regex converted from pydantic model @@ -32,6 +32,16 @@ + r"""\}""" ) +city_regex = ( + r"""\{\n""" + + r""" "name": "[\w\d\s]{1,16}",\n""" + + r""" "country": "[\w\d\s]{1,16}",\n""" + + r""" "latitude": [-+]?[0-9]*\.?[0-9]{0,2},\n""" + + r""" "population": [-+]?[0-9]{1,9},\n""" + + r""" "top 3 landmarks": \["[\w\d\s]{1,16}", "[\w\d\s]{1,16}", "[\w\d\s]{1,16}"\]\n""" + + r"""\}""" +) + # fmt: off def character_gen(name, generate): s = name + " is a character in Harry Potter. Please fill in the following information about this character.\n" @@ -39,6 +49,15 @@ def character_gen(name, generate): return s # fmt: on +# fmt: off +def city_gen(document, generate): + s = "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + s += generate(s, max_tokens=256, regex=city_regex) + return s +# fmt: on + @guidance def character_maker(lm, name): @@ -65,7 +84,31 @@ def character_maker(lm, name): return lm -def main(args): +@guidance +def city_maker(lm, document): + regex_str_no_quote = r"[\w\d\s]+" + regex_float = r"[0-9]+\.[0-9]+" + lm += f"""\ + Please extract the information of a city from the following wikipedia page. + Page begin. + {document} + Page end. + Here is the name, country, and symbol of the city in JSON format. + {{ + "name": "{guidance.gen("name", max_tokens=16, regex=regex_str_no_quote)}", + "country": "{guidance.gen("country", max_tokens=16, regex=regex_str_no_quote)}", + "latitude": {guidance.gen("latitude", max_tokens=10, regex=regex_float)}, + "population": {guidance.gen("population", max_tokens=10, regex=r"[0-9]+")}, + "top 3 landmarks": [ + "{guidance.gen("landmark1", max_tokens=16, regex=regex_str_no_quote)}", "{guidance.gen("landmark2", max_tokens=16, regex=regex_str_no_quote)}", "{guidance.gen("landmark3", max_tokens=16, regex=regex_str_no_quote)}" + ] + }} + """ + + return lm + + +def bench_character(args): arguments = [] with open(args.data_path, "r") as f: for line in f: @@ -85,7 +128,7 @@ def func(i): get_one_answer = func elif args.backend == "guidance": model = guidance.models.LlamaCpp( - "/home/ubuntu/model_weights/Llama-2-7b-chat-hf/ggml-model-f16.gguf", + args.llama_cpp_model_path, n_gpu_layers=-1, n_ctx=4096, ) @@ -110,11 +153,69 @@ def func(i): latency = time.time() - tic + return states, latency + + +def bench_city_doc(args): + arguments = [] + for line in read_jsonl(args.data_path): + arguments.append({"document": line["document"]}) + arguments = arguments[: args.num_jsons] + + states = [None] * len(arguments) + + # Select backend + if args.backend == "vllm": + url = f"{args.host}:{args.port}/generate" + generate = partial(call_generate_outlines, url=url, temperature=0) + + def func(i): + states[i] = city_gen(**arguments[i], generate=generate) + + get_one_answer = func + elif args.backend == "guidance": + model = guidance.models.LlamaCpp( + args.llama_cpp_model_path, + n_gpu_layers=-1, + n_ctx=4096, + ) + + def func(i): + lm = model + city_maker(**arguments[i]) + states[i] = lm + + get_one_answer = func + else: + raise ValueError(f"Invalid backend: {args.backend}") + + tic = time.time() + if args.parallel == 1: + for i in tqdm(range(len(arguments))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + rets = executor.map(get_one_answer, list(range(len(arguments)))) + for _ in rets: + pass + + latency = time.time() - tic + + return states, latency + + +def main(args): + if args.mode == "character": + args.data_path = "dataset.txt" + states, latency = bench_character(args) + elif args.mode == "city": + args.data_path = "questions.jsonl" + states, latency = bench_city_doc(args) + # Compute accuracy print(f"Latency: {latency:.3f}") # Write results - dump_state_text(f"tmp_output_{args.backend}.txt", states) + dump_state_text(f"tmp_output_{args.backend}_{args.mode}.txt", states) with open(args.result_file, "a") as fout: value = { @@ -131,5 +232,13 @@ def func(i): parser = argparse.ArgumentParser() parser.add_argument("--data-path", type=str, default="dataset.txt") parser.add_argument("--num-jsons", type=int, default=50) + parser.add_argument( + "--mode", type=str, default="character", choices=["character", "city"] + ) + parser.add_argument( + "--llama-cpp-model-path", + type=str, + default="/home/ubuntu/model_weights/Llama-2-7b-chat-hf/ggml-model-f16.gguf", + ) args = add_common_other_args_and_parse(parser) main(args) diff --git a/benchmark/json_fast_forward/bench_sglang.py b/benchmark/json_fast_forward/bench_sglang.py index ebe2153a729..1ea9afb7d8e 100644 --- a/benchmark/json_fast_forward/bench_sglang.py +++ b/benchmark/json_fast_forward/bench_sglang.py @@ -9,7 +9,6 @@ ) from sglang.utils import dump_state_text, read_jsonl - # there are some FSM bugs with json regex converted from pydantic model # here use a string regex instead # regex_string = build_regex_from_object(HarryPoterRole) @@ -137,6 +136,8 @@ def main(args): parser = argparse.ArgumentParser() parser.add_argument("--data-path", type=str) parser.add_argument("--num-jsons", type=int, default=50) - parser.add_argument("--mode", type=str, default="character", choices=["character", "city"]) + parser.add_argument( + "--mode", type=str, default="character", choices=["character", "city"] + ) args = add_common_sglang_args_and_parse(parser) main(args) From 5a333d4e8090d52c23dfb01ea3c80ef97578a40e Mon Sep 17 00:00:00 2001 From: hnyls2002 Date: Thu, 1 Feb 2024 05:27:44 +0000 Subject: [PATCH 4/6] update readme --- benchmark/json_fast_forward/README.md | 50 +++++++++++++++++++++------ 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/benchmark/json_fast_forward/README.md b/benchmark/json_fast_forward/README.md index 30643d745fc..95737cf8f52 100644 --- a/benchmark/json_fast_forward/README.md +++ b/benchmark/json_fast_forward/README.md @@ -3,44 +3,72 @@ ### Dependencies ``` -llama_cpp_python 0.2.32 +llama_cpp_python 0.2.38 guidance 0.1.10 vllm 0.2.7 -outlines 0.0.24 +outlines 0.0.25 +``` + +### Build dataset + +When benchmarking long document information retrieval, run the following command to build the dataset: + +```bash +pip install wikipedia +python3 build_dataset.py ``` ### Benchmark sglang Run Llama-7B -``` +```bash python3 -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 ``` -Benchmark +Benchmark Character Generation +```bash +python3 bench_sglang.py --mode character ``` -python3 bench_sglang.py + +Benchmark City Information Retrieval + +```bash +python3 bench_sglang.py --mode city ``` + ### Benchmark vllm Run Llama-7B -``` +```bash python3 -m outlines.serve.serve --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 ``` -Benchmark +Benchmark Character Generation +```bash +python3 bench_other.py --mode character --backend vllm ``` -python3 bench_other.py --backend vllm + +Benchmark City Information Retrieval + +```bash +python3 bench_other.py --mode city --backend vllm ``` -### Benchmark guidance (seems not supported) +### Benchmark guidance -Run Llama-7B and benchmark +Run Llama-7B and benchmark character generation +```bash +python3 bench_other.py --mode character --backend guidance --parallel 1 ``` -python3 bench_other.py --backend guidance --parallel 1 + +Run Llama-7B and benchmark city information retrieval + +```bash +python3 bench_other.py --mode city --backend guidance --parallel 1 ``` From 5cb22d0b7e8c67967406ae9baff2a96074258b21 Mon Sep 17 00:00:00 2001 From: hnyls2002 Date: Thu, 1 Feb 2024 05:31:45 +0000 Subject: [PATCH 5/6] minor --- benchmark/json_fast_forward/bench_other.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/json_fast_forward/bench_other.py b/benchmark/json_fast_forward/bench_other.py index c8126fec1a4..f2fb06ccfe9 100644 --- a/benchmark/json_fast_forward/bench_other.py +++ b/benchmark/json_fast_forward/bench_other.py @@ -230,7 +230,7 @@ def main(args): if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--data-path", type=str, default="dataset.txt") + parser.add_argument("--data-path", type=str) parser.add_argument("--num-jsons", type=int, default=50) parser.add_argument( "--mode", type=str, default="character", choices=["character", "city"] From 8e532fc009669ee30a596eeb3bfecd7b8082e828 Mon Sep 17 00:00:00 2001 From: hnyls2002 Date: Thu, 1 Feb 2024 05:33:32 +0000 Subject: [PATCH 6/6] minor --- benchmark/json_fast_forward/bench_sglang.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/json_fast_forward/bench_sglang.py b/benchmark/json_fast_forward/bench_sglang.py index 1ea9afb7d8e..7f92a704aae 100644 --- a/benchmark/json_fast_forward/bench_sglang.py +++ b/benchmark/json_fast_forward/bench_sglang.py @@ -115,7 +115,7 @@ def main(args): print(f"Latency: {latency:.3f}") # Write results - dump_state_text(f"tmp_output_{args.backend}.txt", states) + dump_state_text(f"tmp_output_{args.backend}_{args.mode}.txt", states) with open(f"{args.backend}_{args.mode}.json", "w") as fout: for state in states: fout.write(state["json_output"] + "\n")