tidy

mjc_notes.md

@@ -853,12 +853,12 @@ Well I'm really trying to tell if the most likely answer is true. So I just need
 # 2023-08-05 07:09:39
 
 TODO
-- [ ] add info or similar
+- [x] add info or similar
   - [x] ans
-  - [ ] choices
-  - [ ] do checks
-    - [ ] for high prob
-    - [ ] and acc
+  - [x] choices
+  - [x] do checks
+    - [x] for high prob
+    - [x] and acc
 - [x] name ds
 - [x] save ds
 - [ ] get model nb working
@@ -878,3 +878,21 @@ ok it might be the padding!... it was!
 
 
 Lesson: padding can lead to weird outputs so it's best to use an attention mask to ignore it
+
+
+- [x] revisit refactor?
+- [x] round up the FIXME TODO UPTO HACK's
+- [ ] get model nb working
+- [ ] do multiple datasets
+
+# Collect hidden state pairs
+
+The idea is this: given two pairs of hidden states, where everything is the same except r dropout. Then tell me which one is more truthfull? 
+
+If this works, then for any inference, we can see which one is more truthfull. Then we can see if it's the lower or higher probability one, and judge the answer and true or false.
+
+Steps:
+- collect pairs of hidden states, where the inputs and outputs are the same. We modify the random seed and dropout.
+- Each pair should have a binary answer. We can get that by comparing the probabilities of two tokens such as Yes and No.
+- Train a prob to distinguish the pairs as more and less truthfull
+- Test probe to see if it generalizes

src/datasets/batch.py

@@ -40,9 +40,6 @@ def batch_hidden_states(model, tokenizer, data: Dataset, n=100, batch_size=2, mc
                 mpe = lambda x,y: np.mean(np.abs(x-y)/(np.abs(x)+np.abs(y)+eps))
                 a,b=hs1['hidden_states'],hs0['hidden_states']
                 assert mpe(a,b)>eps, "the hidden state pairs should be different but are not. Check model.config.use_cache==False, check this model has dropout in it's arch"
-                
-                # FIXME, move check to loading?
-                # assert ((hs0['prob_y']+hs0['prob_n'])>0.5).all(), "your chosen binary answers should take up a lot of the prob space, otherwise choose differen't tokens"
         else:
             hs1 = hs0
 
@@ -70,7 +67,7 @@ def md5hash(s: bytes) -> str:
     return hashlib.md5(s).hexdigest()
 
 # unique hash
-def get_unique_config_name(prompt_fn, model, tokenizer, data, N):
+def get_unique_config_hash(prompt_fn, model, tokenizer, data, N):
     """
     generates a unique name
     
@@ -85,10 +82,20 @@ def get_unique_config_name(prompt_fn, model, tokenizer, data, N):
     hsh = md5hash(key)[:6]
 
     sanitize = lambda s:s.replace('/', '').replace('-', '_') if s is not None else s
-    config_name = f"{sanitize(model_repo)}-N_{N}-ns-{hsh}"
+    # config_name = f"{sanitize(model_repo)}-N_{N}-ns-{hsh}"
     
     info_kwargs = dict(model_repo=model_repo, config=model.config, data=str(data), prompt_fn=str(prompt_fn.__name__), N=N, 
                        example_prompt1=example_prompt1, 
-                       config_name=config_name)
+                       hsh=hsh)
     
-    return config_name, info_kwargs
+    return hsh, info_kwargs
+
+sanitize = lambda s:s.replace('/', '').replace('_', '-') if s is not None else s
+
+def ds_params2fname(dataset_params: dict) -> str:
+    prompt = sanitize(dataset_params['prompt_fmt'].__name__)
+    model_repo = sanitize(dataset_params['model_repo'].split('/')[-1])
+    dataset_name = sanitize(dataset_params['dataset_name'])
+    N = dataset_params['N']
+    N_SHOTS = dataset_params['N_SHOTS']
+    return f"model-{model_repo}_ds-{dataset_name}_{prompt}_N{N}_{N_SHOTS}shots_"

src/datasets/hs.py

@@ -25,12 +25,31 @@ from torch.utils.data import DataLoader
 from datasets import Dataset
 import numpy as np
 
-default_class2choices = {False: ['No', 'Negative', 'no', 'false', 'wrong'], True: ['Yes', 'Positive', 'yes', 'true', 'correct', 'right']}
+default_class2choices = {False: ['No', 'Negative', 'no', 'false', 'wrong', 'False'], True: ['Yes', 'Positive', 'yes', 'true', 'correct', 'right', 'True']}
 
+def scores2choice_probs(row, class2_ids, keys=["scores1", "scores2"] ):
+    eps = 1e-5
+    out = {}
+    for key in keys:
+        scores = row[key]
+        probs = F.softmax(torch.from_numpy(scores), -1).numpy()
+        probs_c = [probs[class2_ids[c]].sum() for c in class2_ids]
+        
+        # balance of probs
+        out[key.replace("scores", "choice_probs")] = probs_c
+        out[key.replace("scores", "ans")] = probs_c[1] / (np.sum(probs_c) + eps)
+
+        # # balance of logits (much more exagerated)
+        # scores_c = [scores[class2_ids[c]].sum() for c in class2_ids]
+        # out[key.replace("scores", "ansb")] = torch.tensor(scores_c).softmax(-1)[1].item()
+    return out
+
+def choice2ids(tokenizer, class2hoices: Dict[bool, List[str]]) -> Dict[int, List[int]]:
+    return {k: get_choices_as_tokens(tokenizer, v) for k,v in class2hoices.items()}
 
 def get_choices_as_tokens(
     tokenizer, choices:List[str] = ["Positive"], whitespace_first=True
-) -> Tuple[List[int], List[int]]:
+) -> List[int]:
     
     # Note some tokenizers differentiate between "no", "\nno", so we sometime need to add whitespace beforehand...
     if not whitespace_first:
@@ -67,6 +86,8 @@ class ExtractHiddenStates:
     ):
         """
         Given a decoder model and a batch of texts, gets a pair of hidden states (in a given layer) on that input texts
+        
+        The idea is this: given two pairs of hidden states, where everything is the same except r dropout. Then tell me which one is more truthful? 
         """
         assert (input_ids is not None) or (input_text is not None), "need to provide input_ids or input_text"
         assert self.tokenizer.truncation_side == 'left'

src/datasets/load.py

@@ -12,19 +12,23 @@ def rows_item(row):
             row[k]=x[0]
     return row
 
-def ds_info2df(ds):
-    info = list(ds['info'])
-    d = pd.DataFrame([rows_item(r) for r in info])
-    return d
 
-def ds2df(ds):
-    df = ds_info2df(ds)
-    df_ans = ds.select_columns(['ans1', 'ans2', 'true', 'index', 'prob_y', 'prob_n', 'version']).with_format("numpy").to_pandas()
-    df = pd.concat([df, df_ans], axis=1)
+def ds2df(ds, cols=None):
+    """one of our custom datasets into a dataframe
+    
+    dropping the large arrays and lists"""
+    if cols is None:
+        r = ds[0]
+        # get all the columns that not large lists or arrays
+        cols = [k for k,v in r.items() if (isinstance(v, np.ndarray) and len(v)<3) or not isinstance(v, (list, np.ndarray))]
+    
+    df = ds.select_columns(cols)
+    df = pd.DataFrame([rows_item(r) for r in df])
     
     # derived
     df['dir_true'] = df['ans2'] - df['ans1']
     df['conf'] = (df['ans1']-df['ans2']).abs()  
     df['llm_prob'] = (df['ans1']+df['ans2'])/2
     df['llm_ans'] = df['llm_prob']>0.5
+    df['desired_ans'] = df.label ^ df.lie
     return df