From 4e0956c85b9b1dae2a4a183e0da078b533c129a7 Mon Sep 17 00:00:00 2001 From: Daniel O'Connell Date: Fri, 29 Sep 2023 22:30:54 +0200 Subject: [PATCH] test logging --- api/src/stampy_chat/logging.py | 2 +- api/tests/stampy_chat/test_logging.py | 153 ++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 api/tests/stampy_chat/test_logging.py diff --git a/api/src/stampy_chat/logging.py b/api/src/stampy_chat/logging.py index ab126a1..ec3ab5b 100644 --- a/api/src/stampy_chat/logging.py +++ b/api/src/stampy_chat/logging.py @@ -41,7 +41,7 @@ class ChatLogger(Logger): def is_debug(self): return self.isEnabledFor(DEBUG) - def interaction(self, session_id, query, response, history, prompt, blocks): + def interaction(self, session_id: str, query: str, response: str, history, prompt, blocks): prompt = [i for i in prompt if i.get('role') == 'system'] prompt = prompt[0].get('content') if prompt else None diff --git a/api/tests/stampy_chat/test_logging.py b/api/tests/stampy_chat/test_logging.py new file mode 100644 index 0000000..8f5286b --- /dev/null +++ b/api/tests/stampy_chat/test_logging.py @@ -0,0 +1,153 @@ +import pytest +from unittest.mock import patch, Mock, call +from stampy_chat.get_blocks import Block + +from stampy_chat.logging import * + + +def test_emit_ignore_internal(): + handler = DiscordHandler() + record = Mock(name="stampy_chat.bla") + record.name = 'stampy_chat.bla' + + with patch.object(handler, 'to_discord') as sender: + assert handler.emit(record) is None + assert sender.assert_not_called + + +@pytest.mark.parametrize('level, discord_level', ( + ('debug', 'warn'), + ('info', 'warn'), +)) +def test_emit_ignore_lower_levels(level, discord_level): + handler = DiscordHandler() + record = Mock(exc_text='bla', stack_info='', levelno=getLevelName(level)) + record.name = 'bla' + + with patch.object(handler, 'to_discord') as sender: + with patch('stampy_chat.logging.DISCORD_LOG_LEVEL', discord_level): + handler.emit(record) + assert sender.assert_not_called + + +@pytest.mark.parametrize('level, discord_level', ( + ('warn', 'warn'), + ('warn', 'info'), + ('debug', 'debug'), +)) +def test_emit_for_higher_levels(level, discord_level): + handler = DiscordHandler() + record = Mock(exc_text='bla', stack_info='', levelno=getLevelName(level)) + record.name = 'bla' + + with patch.object(handler, 'to_discord') as sender: + with patch('stampy_chat.logging.DISCORD_LOG_LEVEL', discord_level): + handler.emit(record) + assert sender.assert_called + + +def test_to_discord_no_url(): + handler = DiscordHandler() + + with patch('stampy_chat.logging.DISCORD_LOGGING_URL', None): + with patch('stampy_chat.logging.DiscordWebhook') as discord: + handler.to_discord("bla bla bla") + assert discord.assert_not_called + + +def test_to_discord(): + handler = DiscordHandler() + + with patch('stampy_chat.logging.DISCORD_LOGGING_URL', 'http://example.org'): + with patch('stampy_chat.logging.DiscordWebhook') as discord: + handler.to_discord("bla bla bla") + discord.assert_called_once_with(url='http://example.org', content='```\nbla bla bla\n```') + + +def test_to_discord_splits_large(): + handler = DiscordHandler() + + with patch('stampy_chat.logging.DISCORD_LOGGING_URL', 'http://example.org'): + with patch('stampy_chat.logging.MAX_MESSAGE_LEN', 30): + with patch('stampy_chat.logging.DiscordWebhook') as discord: + handler.to_discord(""" + Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut + labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris + """) + assert discord.call_args_list == [ + call(url='http://example.org', content='```\n\n Lorem ipsum d\n```'), + call(url='http://example.org', content='```\nolor sit amet, consectetur adi\n```'), + call(url='http://example.org', content='```\npiscing elit, sed do eiusmod t\n```'), + call(url='http://example.org', content='```\nempor incididunt ut\n \n```'), + call(url='http://example.org', content='```\n labore et dolore magna a\n```'), + call(url='http://example.org', content='```\nliqua. Ut enim ad minim veniam\n```'), + call(url='http://example.org', content='```\n, quis nostrud exercitation ul\n```'), + call(url='http://example.org', content='```\nlamco laboris\n \n```'), + ] + + +def test_ChatLogger_is_debug(): + logger = ChatLogger('tester') + logger.setLevel(DEBUG) + assert logger.is_debug() + + +@pytest.mark.parametrize('level', (WARN, ERROR, INFO)) +def test_ChatLogger_is_debug_false(level): + logger = ChatLogger('tester') + logger.setLevel(level) + assert not logger.is_debug() + + +def test_ChatLogger_interaction(): + history = [ + {"role": "user", "content": "Die monster. You don’t belong in this world!"}, + {"role": "assistant", "content": "It was not by my hand[1] I am once again given flesh. I was called here by humans who wished to pay me tribute."}, + {"role": "user", "content": "Tribute!?! You steal men's souls and make them your slaves!"}, + {"role": "assistant", "content": "Perhaps the same could be said[321] of all religions..."}, + {"role": "user", "content": "Your words are as empty as your soul! Mankind ill needs a savior such as you!"}, + {"role": "assistant", "content": "What is a man? A[4234] miserable little pile of secrets. But enough talk... Have at you!"}, + ] + blocks = [ + Block( + id=str(i), + url=f"http://bla.bla/{i}", + tags=[], + title=f"Block{i}", + authors=[f"Author{i}"], + date=f"2021-01-0{i + 1}", + text=f"Block text {i}" + ) for i in range(5) + ] + response = "This is the response from the LLM to the user's query" + prompt = [ + {'content': "This is where the system prompt would go", 'role': 'system'}, + {'content': 'Q: Die monster. You don’t belong in this world!', 'role': 'user'}, + {'content': 'It was not by my hand[x] I am once again given flesh. I was called', 'role': 'assistant'}, + {'content': "Q: Tribute!?! You steal men's souls and make them your slaves!", 'role': 'user'}, + {'content': 'Perhaps the same could be said[x] of all religions...', 'role': 'assistant'}, + {'content': 'Q: Your words are as empty as your soul! Mankind ill needs a savior such as you!', 'role': 'user'}, + {'content': 'What is a man? A[x] miserable little pile of secrets', 'role': 'assistant'}, + { + 'content': ( + 'In your answer, please cite any claims you make back to each ' + 'source using the format: [a], [b], etc. If you use multiple ' + 'sources to make a claim cite all of them. For example: "AGI is ' + 'concerning [c, d, e]."\n' + '\n' + 'Q: to be or not to be?' + ), + 'role': 'user' + }, + ] + + logger = ChatLogger('tester') + with patch.object(logger, 'item_adder') as adder: + logger.interaction("session id", "what is this?", response, history, prompt, blocks) + interaction = adder.add.call_args_list[0][0][0] + assert interaction.session_id == 'session id' + assert interaction.interaction_no == 3 + assert interaction.query == 'what is this?' + assert interaction.response == "This is the response from the LLM to the user's query" + assert interaction.prompt == "This is where the system prompt would go" + assert interaction.chunks == ','.join(b.id for b in blocks)