wassname/vllm
1
0
Fork 0
mirror of https://github.com/wassname/vllm.git synced 2026-09-11 12:51:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Misc] Include matched stop string/token in responses (#2976)

Browse Source 
Co-authored-by: Sahil Suneja <sahilsuneja@gmail.com>
This commit is contained in:
Nick Hill
2024-03-25 17:31:32 -07:00
committed by GitHub
co-authored by Sahil Suneja
parent 3a243095e5
commit dfeb2ecc3a
7 changed files with 97 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vllm/engine/llm_engine.py
+5 -2
View File
@@ -740,12 +740,15 @@ class LLMEngine:
if seq.output_text.endswith(stop_str):
self._finalize_sequence(seq, sampling_params, stop_str)
seq.status = SequenceStatus.FINISHED_STOPPED
seq.stop_reason = stop_str
return
if seq.get_last_token_id() in sampling_params.stop_token_ids:
last_token_id = seq.get_last_token_id()
if last_token_id in sampling_params.stop_token_ids:
stop_str = self.get_tokenizer_for_seq(seq).convert_ids_to_tokens(
seq.get_last_token_id())
last_token_id)
self._finalize_sequence(seq, sampling_params, stop_str)
seq.status = SequenceStatus.FINISHED_STOPPED
seq.stop_reason = last_token_id
return
# Check if the sequence has generated the EOS token.