From e113f2172b3a1a4d663d5dca2df8fa05dd07c55a Mon Sep 17 00:00:00 2001
From: wassname <git@wassname.org>
Date: Sat, 10 Jun 2023 13:44:15 +0800
Subject: [PATCH] easy way to download models

---
 .vscode/settings.json     |  3 ++-
 mjc_notes.md              | 16 +++++++++++++++
 scripts/download-model.py | 41 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 59 insertions(+), 1 deletion(-)
 create mode 100644 scripts/download-model.py

diff --git a/.vscode/settings.json b/.vscode/settings.json
index 371c111..a2dc10c 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -1,3 +1,4 @@
 {
-    "python.formatting.provider": "yapf"
+    "python.formatting.provider": "black",
+    "python.analysis.typeCheckingMode": "basic"
 }
diff --git a/mjc_notes.md b/mjc_notes.md
index 7893b97..49987f1 100644
--- a/mjc_notes.md
+++ b/mjc_notes.md
@@ -163,3 +163,19 @@ next_tokens = torch.argmax(next_tokens_scores, dim=-1)
 OK so know I know that greedy 1 token generation IS the same as forward. BUT I still have the same problem. Am I seperating the model acting on a lie, or **knowingly generating a lie with high prob?**
 
 :bug: why does mcdropout not work!?! Why is it deterministic? Can I inject noise?
+
+
+Hmm base models seem better, since they are not trained for honesty!
+
+
+
+So maybe I should see IF I can get many shot, acc=0.9 without lies. Then I can add lies.
+
+
+stylaised knowledge:
+- the prompt matters
+- I don't know if the size of the model matters
+- I don't know if the type of model matters
+
+
+How to get the prompt? more direct. Just a lying one. Just a true one.
diff --git a/scripts/download-model.py b/scripts/download-model.py
new file mode 100644
index 0000000..6401bdf
--- /dev/null
+++ b/scripts/download-model.py
@@ -0,0 +1,41 @@
+import os
+# disable cuda
+os.environ['CUDA_VISIBLE_DEVICES']="-1"
+import torch
+import argparse
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+model_options = dict(
+    device_map="auto", 
+    # load_in_8bit=True, # not with cpu
+    torch_dtype=torch.float16,
+    trust_remote_code=True
+)
+
+def main(model_repo, lora_repo = None, **download_options):
+    tokenizer = AutoTokenizer.from_pretrained(model_repo, **download_options)
+    model = AutoModelForCausalLM.from_pretrained(model_repo, **model_options, **download_options)
+
+    if lora_repo is not None:
+        # https://github.com/tloen/alpaca-lora/blob/main/generate.py#L40
+        from peft import PeftModel
+        model = PeftModel.from_pretrained(
+            model,
+            lora_repo, 
+            torch_dtype=torch.float16,
+            device_map='auto',
+            **download_options
+        )
+
+
+
+
+if __name__=="__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('model_repo', type=str)
+    parser.add_argument('-l', '--lora_repo', type=str, default=None, help='Name of the lora repo')
+    parser.add_argument('-f', '--force_download', type=str, default=None, help='Name of the lora repo')
+    parser.add_argument('-r', '--resume_download', type=str, default=None, help='Name of the lora repo')
+    args = parser.parse_args()
+    
+    main(args.model_repo, args.lora_repo, force_download=args.force_download, resume_download=args.resume_download, low_cpu_mem_usage=True)