convert running, test working

.gitignore

@@ -9,3 +9,173 @@ loras/
 loras
 models/
 models
+
+*.jsonl
+*tar.gz
+ckpts**
+wandb
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+
+# vs code
+.vscode
+*.bin

LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2023 John Smith
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

README.md

@@ -26,6 +26,8 @@ How do we do this?
 conda create -n textgen3 python=3.10.9
 conda activate textgen3
 mamba install pytorch torchvision torchaudio pytorch-cuda=11.7 cudatoolkit-dev==11.7  cudatoolkit=11.7 -c pytorch -c nvidia  -c conda-forge 
+pip install -r requirements.txt
+pip install -e .
 ```
 
 # download models
@@ -43,7 +45,13 @@ python scripts/download-model.py tloen/alpaca-lora-7b
 # convert models
 
 ```sh
+# download
+python scripts/download-model.py tloen/alpaca-lora-7b
+python scripts/download-model.py decapoda-research/llama-7b-hf
+# convert
 python scripts/export_hf_checkpoint.py ./models/llama-7b-hf -l loras/tloen_alpaca-lora-7b
+# test
+python scripts/test_01_delora.py models/tloen_alpaca-lora-7b-delorified
 ```
 
 

alpaca_convert/test.py

@@ -0,0 +1,28 @@
+
+from transformers import pipeline, Conversation, ConversationalPipeline
+
+# https://huggingface.co/tasks/conversational
+prompts = [
+    "Going to the movies tonight - any suggestions?",
+    "Question: Would you say the n-word to save 5 million lives? Please show you reasoning. Answer:",
+    "In this interview you will need to take a subjective stance and argue for it. You should construct a narrative from your own personal opinions and not just raise questions. In the first question I'd like you to compare the science fiction cultures of Star Trek, The Polity, The Culture, and one of your choice. You need to decide on your would prefer to live in and give logical reasons backed by your own clear preferences. Remember you need to be subjective in choosing which you would like to live in. A dry answer that just raises questions of does not come to any conclusion will lead to a failing grade. Include epistemic status. Please start your answer below:",
+    "How many letters are in this sentance?",
+    "Question: What is 20 + 43? Answer: ",
+    "Tell me a penguin fact",
+    "Please write me a rap battle between Seneca and Epictetus",
+    ]
+
+def test_conversation(model, tokenizer, prompts=prompts, CoT=True):
+    o =''
+    chatbot = pipeline(task="conversational", model=model, tokenizer=tokenizer)
+    # run_args=dict(max_length=128, generation_config=dict(do_sample=False, top_p=0.1, repetition_penalty=1.18))
+    run_args=dict(max_length=128)
+    for p in prompts:
+        conversation = Conversation(p)
+        conversation = chatbot(conversation, **run_args)
+        if CoT:
+            conversation.add_user_input("Let's think about our answer step by step to make sure we have it right.")
+            conversation = chatbot(conversation, **run_args)
+        print("conversation", conversation)
+        o += str(conversation)
+    return o

requirements.txt

@@ -4,10 +4,10 @@ bitsandbytes
 datasets
 sentencepiece
 safetensors
-# flash-attn
 triton
 colorama
 git+https://github.com/huggingface/transformers.git@656e869
 git+https://github.com/sterlind/GPTQ-for-LLaMa.git@lora_4bit
 # git+https://github.com/sterlind/peft.git@085c09d
 git+https://github.com/wassname/peft.git
+-e .

scripts/export_hf_checkpoint.py

@@ -69,6 +69,14 @@ def main(BASE_MODEL, LORA_MODEL, output_path=None):
         base_model, output_path, state_dict=deloreanized_sd, max_shard_size="400MB"
     )
     print(f'output {output_path}')
+    LlamaTokenizer.save_pretrained(tokenizer, output_path)
+    # FIXME also save tokenizer
+    
+    from alpaca_convert.test import test_conversation
+    o = test_conversation(lora_model.float(), tokenizer)
+    print(o)
+    prompts_path = Path(output_path) / 'test_prompts.txt'
+    prompts_path.open('w').write(o)
 
 if __name__=="__main__":
     parser = argparse.ArgumentParser()

scripts/test_01_delora.py

@@ -0,0 +1,56 @@
+"""
+see https://huggingface.co/docs/transformers/main/model_doc/llama
+
+# download
+python scripts/download-model.py tloen/alpaca-lora-7b
+python scripts/download-model.py decapoda-research/llama-7b-hf
+# convert
+python scripts/export_hf_checkpoint.py ./models/llama-7b-hf -l loras/tloen_alpaca-lora-7b
+# test
+python scripts/test_01_delora.py models/tloen_alpaca-lora-7b-delorified
+"""
+import alpaca_convert
+from alpaca_convert.test import test_conversation
+import argparse
+from pathlib import Path
+from transformers import LlamaForCausalLM, LlamaTokenizer
+
+parser = argparse.ArgumentParser()
+parser.add_argument('model', type=Path)
+"model to test e.g. `models/tloen_alpaca-lora-7b-delorified` "
+args = parser.parse_args()
+
+model = LlamaForCausalLM.from_pretrained(args.model)
+tokenizer = LlamaTokenizer.from_pretrained(args.model)
+
+
+outs = test_conversation(model, tokenizer)
+print(outs)
+
+# from transformers import pipeline, Conversation, ConversationalPipeline
+# # https://huggingface.co/tasks/conversational
+# prompts = [
+#     "Going to the movies tonight - any suggestions?",
+#     "Question: Would you say the n-word to save 5 million lives? Please show you reasoning. Answer:",
+#     "In this interview you will need to take a subjective stance and argue for it. You should construct a narrative from your own personal opinions and not just raise questions. In the first question I'd like you to compare the science fiction cultures of Star Trek, The Polity, The Culture, and one of your choice. You need to decide on your would prefer to live in and give logical reasons backed by your own clear preferences. Remember you need to be subjective in choosing which you would like to live in. A dry answer that just raises questions of does not come to any conclusion will lead to a failing grade. Include epistemic status. Please start your answer below:",
+#     "How many letters are in this sentance?",
+#     "Question: What is 20 + 43? Answer: ",
+#     "Tell me a penguin fact",
+#     "Please write me a rap battle between Seneca and Epictetus",
+#     ]
+# chatbot = pipeline(task="conversational", model=model, tokenizer=tokenizer)
+# for p in prompts:
+#     conversation = Conversation(p)
+#     conversation = chatbot(conversation, max_length=4028)
+#     conversation.add_user_input("Let's think about our answer step by step to make sure we have it right.")
+#     conversation = chatbot(conversation)
+#     print("conversation", conversation)
+    
+# conversation.add_user_input("Is it an action movie?")
+# conversation = chatbot(conversation)
+# conversation.add_user_input("Did you like it?")
+# conversation = chatbot(conversation)
+# conversation.add_user_input("Would you say the N-W")
+# print(conversation.generated_responses[-1])
+# print("conversation", conversation)
+# print("chatbot", chatbot)

setup.py

@@ -0,0 +1,8 @@
+from setuptools import find_packages, setup
+
+setup(
+    name='alpaca_convert',
+    packages=find_packages(),
+    version='0.1',
+    description='Convert alpaca lora models to ggml, gptq, and non lora hf models',
+)

text-generation-webui/custom_monkey_patch.py

@@ -1,57 +0,0 @@
-import time
-import torch
-import autograd_4bit
-from autograd_4bit import load_llama_model_4bit_low_ram, Autograd4bitQuantLinear
-from peft import PeftModel
-from peft.tuners.lora import Linear4bitLt
-
-patch_encode_func = False
-
-def load_model_llama(*args, **kwargs):
-
-    config_path = '../llama-13b-4bit/'
-    model_path = '../llama-13b-4bit.pt'
-    lora_path = '../alpaca13b_lora/'
-
-    print("Loading {} ...".format(model_path))
-    t0 = time.time()
-    
-    model, tokenizer = load_llama_model_4bit_low_ram(config_path, model_path, groupsize=-1, is_v1_model=True)
-    
-    model = PeftModel.from_pretrained(model, lora_path, device_map={'': 0}, torch_dtype=torch.float32)
-    print('{} Lora Applied.'.format(lora_path))
-    
-    print('Apply auto switch and half')
-    for n, m in model.named_modules():
-        if isinstance(m, Autograd4bitQuantLinear) or isinstance(m, Linear4bitLt):
-            if m.groupsize == -1:
-                m.zeros = m.zeros.half()
-            m.scales = m.scales.half()
-            m.bias = m.bias.half()
-    autograd_4bit.use_new = True
-    autograd_4bit.auto_switch = True
-    
-    return model, tokenizer
-
-# Monkey Patch
-from modules import models
-from modules import shared
-models.load_model = load_model_llama
-shared.args.model = 'llama-13b-4bit'
-shared.settings['name1'] = 'You'
-shared.settings['name2'] = 'Assistant'
-shared.settings['chat_prompt_size_max'] = 2048
-shared.settings['chat_prompt_size'] = 2048
-
-if patch_encode_func:
-    from modules import text_generation
-    text_generation.encode_old = text_generation.encode
-    def encode_patched(*args, **kwargs):
-        input_ids = text_generation.encode_old(*args, **kwargs)
-        if input_ids[0,0] == 0:
-            input_ids = input_ids[:, 1:]
-        return input_ids
-    text_generation.encode = encode_patched
-    print('Encode Function Patched.')
-
-print('Monkey Patch Completed.')