wassname/vllm
1
0
Fork 0
mirror of https://github.com/wassname/vllm.git synced 2026-09-10 12:41:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Core] Optimize evictor-v2 performance (#7193)

Browse Source
This commit is contained in:
xiaobochen123
2024-08-06 12:34:25 -07:00
committed by GitHub
parent 8d59dbb000
commit 660470e5a3
1 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vllm/core/evictor_v2.py
+4 -2
View File
@@ -91,8 +91,9 @@ class LRUEvictor(Evictor):
# at the start of OrderedDict. Loop through all these blocks to
# find the one with maximum number of hashed tokens.
for _id, block in self.free_table.items():
if evicted_block.last_accessed > block.last_accessed or (
evicted_block.last_accessed == block.last_accessed and
if evicted_block.last_accessed < block.last_accessed:
break
if (evicted_block.last_accessed == block.last_accessed and
evicted_block.num_hashed_tokens < block.num_hashed_tokens):
evicted_block = block
evicted_block_id = _id
@@ -109,6 +110,7 @@ class LRUEvictor(Evictor):
def update(self, block_id: int, last_accessed: float):
self.free_table[block_id].last_accessed = last_accessed
self.free_table.move_to_end(block_id)
def remove(self, block_id: int):
if block_id not in self.free_table: