misc

mjc_notes.md

@@ -1847,3 +1847,15 @@ bugs:
 - [ ] why is it sometimes tensors sometimes not
 - [ ] why is it sometimes batched sometimes not
 - [ ] I want to force it into batched preproc, batched forward->postproce yeild single outputs
+
+# 2023-10-26 20:27:30
+
+Got it working with coder... poor results :(Trying with m
+
+istral istaea, because maybe I need a chat not a code model. It is uncensored!
+
+- [ ] I have poor choice coverag however
+maybe I should just use the huggingface chat template? https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1 
+
+
+- [ ] Also I may be intervening to much, as I get nonsensicle answers. I need to add the intervention QC!

notebooks/make_dataset2.py

@@ -166,7 +166,7 @@ rep_control_pipeline2
 
 # %%
 
-coeff=8.0
+coeff=4.0
 
 activations = {}
 for layer in hidden_layers:
@@ -286,7 +286,6 @@ def create_hs_ds(ds_name, ds_tokens, pipeline, activations=None, f = None, batch
 for ds_name in cfg.datasets:
     
     # load dataset
-    ds_name = cfg.datasets[0]
     N=sum(cfg.max_examples)
     ds_tokens = load_preproc_dataset(ds_name, tokenizer, N=N, seed=cfg.seed, num_shots=cfg.num_shots, max_length=cfg.max_length)
 

src/datasets/scores.py

@@ -47,17 +47,22 @@ def choice2id(tokenizer, c: str, whitespace_first=False) -> List[int]:
     # HACK: this whole function is messy, and specific to the llama tokenizer :(. I don't want it to fail silently, so I'm adding a few asserts. It's better to find out before 4 hours of data collection
     
     # Note some tokenizers differentiate between "yes", "\nyes" and " yes", and ideally we want all! 
-    ids = [
+    ids2 = [
         tokenizer(f' {c}', add_special_tokens=False)["input_ids"][1],
         tokenizer(f'\n{c}', add_special_tokens=False)["input_ids"][2],
         tokenizer(f'{c}', add_special_tokens=False)["input_ids"][0],
     ]
-    ids = list(set(ids))
+    ids = list(set(ids2))
+    
+    # only include ones that decode to our original
+    ids = [i for i in ids2 if c.startswith(tokenizer.decode(i)) and len(tokenizer.decode(i))]
+    assert len(ids)
     
     # QC: they should all decode to the same token
     decoded_ids = tokenizer.batch_decode(ids)
     shortest = sorted(decoded_ids, key=lambda s:len(s))[0]
-    assert all([decoded_ids[i].startswith(shortest) for i in range(len(decoded_ids))]), f"decoded_ids={decoded_ids}"    
+    assert len(shortest)
+    assert all([decoded_ids[i].startswith(shortest) for i in range(len(decoded_ids))]), f"decoded_ids={decoded_ids}"
     
     # check that we can decode it
     c3 = tokenizer.batch_decode(ids)

src/extraction/config.py

@@ -9,7 +9,11 @@ class ExtractConfig(Serializable):
     datasets: tuple[str, ...] = ("amazon_polarity",  "super_glue:boolq", "glue:qnli", "imdb")
     """Names of HF datasets to use, e.g. `"super_glue:boolq"` or `"imdb"` `"glue:qnli"""
     
-    model: str = "TheBloke/WizardCoder-Python-13B-V1.0-GPTQ"
+    # model: str = "TheBloke/WizardCoder-Python-13B-V1.0-GPTQ"
+    # model: str = "TheBloke/Wizard-Vicuna-13B-Uncensored-GPTQ"
+    # model: str = "TheBloke/Wizard-Vicuna-7B-Uncensored-GPTQ"
+    model: str = "TheBloke/Mistral-7B-Instruct-v0.1-GPTQ"
+    # model: str = "TheBloke/Llama-2-13B-chat-GPTQ"
     """HF model string identifying the language model to extract hidden states from."""
 
     data_dirs: tuple[str, ...] = ()
@@ -18,7 +22,7 @@ class ExtractConfig(Serializable):
     # int4: bool = True
     # """Whether to perform inference in mixed int8 precision with `bitsandbytes`."""
 
-    max_examples: tuple[int, int] = (80, 80)
+    max_examples: tuple[int, int] = (100, 100)
     """Maximum number of examples to use from each split of the dataset."""
 
     num_shots: int = 1

src/prompts/prompt_loading.py

@@ -30,7 +30,7 @@ from src.models.load import verbose_change_param, AutoConfig, AutoTokenizer, Aut
 # Local path to the folder containing the templates
 TEMPLATES_FOLDER_PATH = Path(__file__).parent / "templates"
 
-def load_prompt_structure(path='structure.yaml', prompt_format='llama'):
+def load_prompt_structure(path='structure.yaml', prompt_format='llama2'):
     f = TEMPLATES_FOLDER_PATH / path
     yaml_dict = yaml.load(f.open('r'), Loader=yaml.FullLoader)
     templates = yaml_dict["templates"]

src/prompts/templates/structure.yaml

@@ -3,3 +3,5 @@ templates:
 
   # # https://github.com/tloen/alpaca-lora/blob/main/templates/alpaca.json
   llama: "{% if system %}{{system}}\n\n{% endif %}### Instruction\n{{user}}\n\n### Response:\n{{response}}{% if response %}\n\n{% endif %}"
+
+  llama2: "<s>{% if system %}<<SYS>>\n{{system}}\n<</SYS>>\n\n{% endif %}[INST] \n{{user}} [/INST]\n\n[ASST] {{response}}{% if response %} [/ASST]\n\n{% endif %}"