-
Notifications
You must be signed in to change notification settings - Fork 21
/
server.py
91 lines (73 loc) · 2.5 KB
/
server.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
import argparse
import time
import uuid
import fastapi
import uvicorn
import pydantic
import sillm
import sillm.utils as utils
app = fastapi.FastAPI()
@app.get("/")
def index():
return {
"message": "Welcome to the SiLLM API!"
}
@app.get("/v1/models")
def list_models():
return {
"object": "list",
"data": [app.model.description()]
}
class ChatCompletion(pydantic.BaseModel):
messages: list
model: str
logprobs: bool | None = False
max_tokens: int | None = 4096
temperature: float | None = 0.7
@app.post("/v1/chat/completions")
def chat_completions(completion: ChatCompletion):
result = {
"id": str(uuid.uuid4()),
"object": "chat.completion",
"created": int(time.time()),
"model": app.model.id,
"system_fingerprint": app.fingerprint
}
prompt = app.template.apply_chat_template(completion.messages, add_generation_prompt=True)
generate_args = {
"max_tokens": completion.max_tokens,
"temperature": completion.temperature
}
content, metadata = app.model.completion(prompt, **generate_args)
result["choices"] = [{
"index": 0,
"message": {
"role": "assistant",
"content": content
},
"logprobs": None,
"finish_reason": metadata["finish_reason"]
}]
result["usage"] = metadata["usage"]
return result
if __name__ == "__main__":
# Parse commandline arguments
parser = argparse.ArgumentParser(description="")
parser.add_argument("model", type=str, help="The directory that contains the models")
parser.add_argument("--host", type=str, default="0.0.0.0", help="Port to listen on")
parser.add_argument("--port", type=int, default=8000, help="Port to listen on")
parser.add_argument("--template", type=str, default=None, help="Chat template (chatml, llama2, alpaca, etc.)")
parser.add_argument("-v", "--verbose", default=1, action="count", help="Increase output verbosity")
args = parser.parse_args()
# Initialize logging
log_level = 40 - (10 * args.verbose) if args.verbose > 0 else 0
logger = utils.init_logger(log_level)
app.logger = logger
# Load model
model = sillm.load(args.model)
app.model = model
# Initialize system fingerprint
app.fingerprint = "sillm_" + str(uuid.uuid4())
# Init conversation template
app.template = sillm.init_template(model.tokenizer, model.args, args.template)
uvicorn.run(app, host=args.host, port=args.port)