tcsenpai/agenticSeek
1
0
Fork 0
mirror of https://github.com/tcsenpai/agenticSeek.git synced 2025-06-06 11:05:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

server gpu percentage for vllm

Browse Source
This commit is contained in:
martin legrand 2025-04-08 20:58:32 +02:00
parent 9a1b2b93f6
commit ed2a9cc204
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
server/sources/vllm_handler.py
View File

@ -51,6 +51,7 @@ class Vllm(GeneratorLLM):
sampling_params = SamplingParams( sampling_params = SamplingParams(
temperature=0.7, temperature=0.7,
max_tokens=512, max_tokens=512,
gpu_memory_utilization=0.5,
stream=True # Enable streaming stream=True # Enable streaming
) )
outputs = self.llm.generate(prompt, sampling_params, use_tqdm=False) outputs = self.llm.generate(prompt, sampling_params, use_tqdm=False)