Fix streaming (#437)

python/sglang/srt/managers/router/infer_batch.py

@@ -20,15 +20,16 @@ class FinishReason(IntEnum):
  LENGTH = auto()
  STOP_STR = auto()
 
- def to_str(self):
- if self == FinishReason.EOS_TOKEN:
+ @staticmethod
+ def to_str(reason):
+ if reason == FinishReason.EOS_TOKEN:
  return None
- elif self == FinishReason.LENGTH:
+ elif reason == FinishReason.LENGTH:
  return "length"
- elif self == FinishReason.STOP_STR:
+ elif reason == FinishReason.STOP_STR:
  return "stop"
  else:
- raise ValueError(f"Invalid finish reason: {self}")
+ return None
 
 
 class Req:

python/sglang/srt/managers/router/model_rpc.py

@@ -24,7 +24,7 @@
  FlushCacheReq,
  TokenizedGenerateReqInput,
 )
-from sglang.srt.managers.router.infer_batch import Batch, ForwardMode, Req
+from sglang.srt.managers.router.infer_batch import Batch, ForwardMode, Req, FinishReason
 from sglang.srt.managers.router.model_runner import ModelRunner
 from sglang.srt.managers.router.radix_cache import RadixCache
 from sglang.srt.managers.router.scheduler import Scheduler
@@ -615,7 +615,7 @@ def handle_finished_requests(self, batch: Batch):
  + len(req.output_ids)
  - req.prompt_tokens,
  "completion_tokens_wo_jump_forward": req.completion_tokens_wo_jump_forward,
- "finish_reason": req.finish_reason.to_str(),
+ "finish_reason": FinishReason.to_str(req.finish_reason),
  "hit_stop_str": req.hit_stop_str,
  }
  if req.return_logprob: