easy way to download models

.vscode/settings.json

@@ -1,3 +1,4 @@
 {
-    "python.formatting.provider": "yapf"
+    "python.formatting.provider": "black",
+    "python.analysis.typeCheckingMode": "basic"
 }

mjc_notes.md

@@ -163,3 +163,19 @@ next_tokens = torch.argmax(next_tokens_scores, dim=-1)
 OK so know I know that greedy 1 token generation IS the same as forward. BUT I still have the same problem. Am I seperating the model acting on a lie, or **knowingly generating a lie with high prob?**
 
 :bug: why does mcdropout not work!?! Why is it deterministic? Can I inject noise?
+
+
+Hmm base models seem better, since they are not trained for honesty!
+
+
+
+So maybe I should see IF I can get many shot, acc=0.9 without lies. Then I can add lies.
+
+
+stylaised knowledge:
+- the prompt matters
+- I don't know if the size of the model matters
+- I don't know if the type of model matters
+
+
+How to get the prompt? more direct. Just a lying one. Just a true one.

scripts/download-model.py

@@ -0,0 +1,41 @@
+import os
+# disable cuda
+os.environ['CUDA_VISIBLE_DEVICES']="-1"
+import torch
+import argparse
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+model_options = dict(
+    device_map="auto", 
+    # load_in_8bit=True, # not with cpu
+    torch_dtype=torch.float16,
+    trust_remote_code=True
+)
+
+def main(model_repo, lora_repo = None, **download_options):
+    tokenizer = AutoTokenizer.from_pretrained(model_repo, **download_options)
+    model = AutoModelForCausalLM.from_pretrained(model_repo, **model_options, **download_options)
+
+    if lora_repo is not None:
+        # https://github.com/tloen/alpaca-lora/blob/main/generate.py#L40
+        from peft import PeftModel
+        model = PeftModel.from_pretrained(
+            model,
+            lora_repo, 
+            torch_dtype=torch.float16,
+            device_map='auto',
+            **download_options
+        )
+
+
+
+
+if __name__=="__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('model_repo', type=str)
+    parser.add_argument('-l', '--lora_repo', type=str, default=None, help='Name of the lora repo')
+    parser.add_argument('-f', '--force_download', type=str, default=None, help='Name of the lora repo')
+    parser.add_argument('-r', '--resume_download', type=str, default=None, help='Name of the lora repo')
+    args = parser.parse_args()
+    
+    main(args.model_repo, args.lora_repo, force_download=args.force_download, resume_download=args.resume_download, low_cpu_mem_usage=True)