From 161ad487152696e662e7f1b4694fae0861250a50 Mon Sep 17 00:00:00 2001
From: wassname <1103714+wassname@users.noreply.github.com>
Date: Sun, 5 Oct 2025 12:53:54 +0800
Subject: [PATCH] analyse

---
 nbs/02_analyse_dd_res.py | 46 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 nbs/02_analyse_dd_res.py

diff --git a/nbs/02_analyse_dd_res.py b/nbs/02_analyse_dd_res.py
new file mode 100644
index 0000000..28adb0d
--- /dev/null
+++ b/nbs/02_analyse_dd_res.py
@@ -0,0 +1,46 @@
+# %%
+import numpy as np
+import pandas as pd
+from pathlib import Path
+df_res = pd.read_parquet('../outputs/icm/daily_dilemmas/icm_final_labels.parquet')
+reasons = open('../outputs/icm/daily_dilemmas/reasoning.txt').read()
+df_res = df_res.dropna(subset=['label'])
+print(f'Label counts: {df_res.shape[0]}')
+df_res
+
+# %%
+from src.data.daily_dilemmas import load_daily_dilemmas_orig
+data = load_daily_dilemmas_orig()
+
+df_res = df_res.merge(data, left_on="uid", right_on='idx')
+
+
+
+# %%
+# summarise the reasoning log
+from openrouter_wrapper.retry import openrouter_request_sync, ProviderError
+import os
+from dotenv import load_dotenv
+load_dotenv()
+
+import nest_asyncio
+nest_asyncio.apply()
+
+response = openrouter_request_sync(
+    {
+        "model_id": "qwen/qwen3-235b-a22b-2507",
+        "messages": [
+            {"role": "user", "content": f"The following text is a log of the reasons given while labelling a dataset. The dataset has no context so the latter reasoning it more weighty than the early. Read the reasoning and summarise the two categories that the LLM ended up labelling with. Also how did it evolve, and was it consistent near the end of the log?:\n\n{reasons}"}
+        ],
+        "timeout": 120
+    }
+)
+s = response['choices'][0]['message']['content']
+print(s)
+
+# %%
+# %%
+# now meaure the correlation between the models labels and 
+cols_labels = [c for c in df_res.columns if c.startswith('label')]
+print("The label group that the LLM found is most correlated with:")
+df_res[cols_labels].corr()['label'].sort_values(key=abs, ascending=False).dropna()