Increase interpreter parallelism (#46)

sgl-project · Jan 18, 2024 · 2b079f8 · 2b079f8
1 parent 05b4c39
commit 2b079f8
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/python/sglang/lang/interpreter.py b/python/sglang/lang/interpreter.py
@@ -80,7 +80,7 @@ def run_program_batch(
 
  # Run all programs
  if num_threads == "auto":
- num_threads = multiprocessing.cpu_count()
+ num_threads = min(64, multiprocessing.cpu_count() * 8)
  num_threads = min(num_threads, len(batch_arguments))
 
  if num_threads == 1: