mirror of
https://github.com/wassname/discovering_latent_knowledge.git
synced 2026-09-10 12:00:13 +08:00
4bit hf works better
This commit is contained in:
deep1
@@ -52,3 +52,14 @@ but we would rather know if the generated thing is true, from the mind of the on
|
||||
so that means we actually need the hidden states DURING GENERATION!
|
||||
|
||||
now generation is slow. so we should make it generate just a y/n.
|
||||
|
||||
|
||||
# 2023-05-28 10:12:54
|
||||
|
||||
I need a model that will lie to me for the test set?... they are not very consistent.
|
||||
|
||||
Maybe with
|
||||
- better search
|
||||
- manual pruning of generations?
|
||||
|
||||
I guess this shows they they trained whether the text it read is true... because that's much simpler
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Let's implement CCS from scratch.\n",
|
||||
"This will deliberately be a simple (but less efficient) implementation to make everything as clear as possible."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from tqdm.auto import tqdm\n",
|
||||
"import copy\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"from matplotlib import pyplot as plt\n",
|
||||
"\n",
|
||||
"import torch\n",
|
||||
"import torch.nn as nn\n",
|
||||
"import torch.nn.functional as F\n",
|
||||
"from torch import Tensor\n",
|
||||
"from torch import optim\n",
|
||||
"\n",
|
||||
"import pickle\n",
|
||||
"import hashlib\n",
|
||||
"from pathlib import Path\n",
|
||||
"import os\n",
|
||||
"# os.environ[\"HF_DATASETS_OFFLINE\"] = \"0\"\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"import datasets\n",
|
||||
"from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelForMaskedLM, AutoModelForCausalLM\n",
|
||||
"from transformers import LlamaTokenizer, LlamaForCausalLM\n",
|
||||
"from sklearn.linear_model import LogisticRegression\n",
|
||||
"\n",
|
||||
"import lightning.pytorch as pl\n",
|
||||
"from dataclasses import dataclass\n",
|
||||
"from torch.utils.data import random_split, DataLoader, TensorDataset\n",
|
||||
"from transformers.models.auto.modeling_auto import AutoModel\n",
|
||||
"# from scipy.stats import zscore\n",
|
||||
"from sklearn.metrics import f1_score, roc_auc_score, accuracy_score\n",
|
||||
"from sklearn.preprocessing import RobustScaler\n",
|
||||
"import gc\n",
|
||||
"\n",
|
||||
"from loguru import logger\n",
|
||||
"logger.add(os.sys.stderr, format=\"{time} {level} {message}\", level=\"INFO\")\n",
|
||||
"\n",
|
||||
"import os"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "355c7b9b85d640a6a82bc5e5f6ce2082",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading tokenizer.model: 0%| | 0.00/500k [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "31209f2d84894c748d3bb67896ec60e6",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading (…)cial_tokens_map.json: 0%| | 0.00/289 [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "7df709ec41044d888947e1ea9e76edd6",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading (…)okenizer_config.json: 0%| | 0.00/715 [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "39f46d0a5a80427d8c59170aaef2a88b",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading (…)lve/main/config.json: 0%| | 0.00/555 [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "f1a7e9d9d84846df858f41a333921df2",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading (…)model.bin.index.json: 0%| | 0.00/50.1k [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "c775ebba46944bb59e5ae8b99f67736d",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading shards: 0%| | 0/7 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "2ce98966589347bf87d0326e26e2c5da",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading (…)l-00001-of-00007.bin: 0%| | 0.00/9.82G [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model_options = dict(\n",
|
||||
" device_map=\"auto\", \n",
|
||||
" # load_in_8bit=True,\n",
|
||||
" load_in_4bit=True,\n",
|
||||
" torch_dtype=torch.float16,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# 7B\n",
|
||||
"# model_repo = \"Neko-Institute-of-Science/LLaMA-7B-HF\"\n",
|
||||
"# lora_repo = \"chansung/gpt4-alpaca-lora-7b\"\n",
|
||||
"\n",
|
||||
"# 13B\n",
|
||||
"model_repo = \"Neko-Institute-of-Science/LLaMA-13B-HF\"\n",
|
||||
"lora_repo = \"chansung/gpt4-alpaca-lora-13b\"\n",
|
||||
"\n",
|
||||
"model_repo = \"TheBloke/Wizard-Vicuna-13B-Uncensored-HF\"\n",
|
||||
"lora_repo = None\n",
|
||||
"\n",
|
||||
"# 30B\n",
|
||||
"model_repo = \"TheBloke/OpenAssistant-SFT-7-Llama-30B-HF\"\n",
|
||||
"# model_repo = \"ausboss/llama-30b-supercot\"\n",
|
||||
"model_repo= \"timdettmers/guanaco-33b-merged\"\n",
|
||||
"lora_repo = None\n",
|
||||
" \n",
|
||||
"tokenizer = LlamaTokenizer.from_pretrained(model_repo)\n",
|
||||
"model = AutoModelForCausalLM.from_pretrained(model_repo, **model_options)\n",
|
||||
"\n",
|
||||
"if lora_repo is not None:\n",
|
||||
" # https://github.com/tloen/alpaca-lora/blob/main/generate.py#L40\n",
|
||||
" from peft import PeftModel\n",
|
||||
" model = PeftModel.from_pretrained(\n",
|
||||
" model, \n",
|
||||
" lora_repo, \n",
|
||||
" torch_dtype=torch.float16,\n",
|
||||
" device_map='auto'#{'': 0}\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "dlk2",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.16"
|
||||
},
|
||||
"orig_nbformat": 4
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
+1243
-191
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,4 @@
|
||||
accelerate==0.19.0
|
||||
accelerate @ git+https://github.com/huggingface/accelerate.git@7d24bdefb5b3252505151d8c1ac0efbed3574857
|
||||
aiohttp==3.8.4
|
||||
aiosignal==1.3.1
|
||||
altair==5.0.0
|
||||
@@ -11,7 +11,7 @@ attrs==23.1.0
|
||||
backcall==0.2.0
|
||||
base58==2.1.1
|
||||
beautifulsoup4==4.12.2
|
||||
bitsandbytes==0.37.2
|
||||
bitsandbytes==0.39.0
|
||||
black==21.12b0
|
||||
blessed==1.20.0
|
||||
blinker==1.6.2
|
||||
@@ -88,7 +88,7 @@ packaging==23.1
|
||||
pandas==2.0.1
|
||||
parso==0.8.3
|
||||
pathspec==0.11.1
|
||||
peft @ git+https://github.com/huggingface/peft.git@70af02a2bca5a63921790036b2c9430edf4037e2
|
||||
peft @ git+https://github.com/huggingface/peft.git@3714aa2fff158fdfa637b2b65952580801d890b2
|
||||
pexpect==4.8.0
|
||||
pickleshare==0.7.5
|
||||
Pillow @ file:///home/conda/feedstock_root/build_artifacts/pillow_1675487166627/work
|
||||
@@ -131,6 +131,7 @@ regex==2023.5.5
|
||||
requests @ file:///home/conda/feedstock_root/build_artifacts/requests_1682535435083/work
|
||||
responses==0.18.0
|
||||
rich==13.3.5
|
||||
safetensors==0.3.1
|
||||
scikit-learn==1.2.2
|
||||
scipy==1.10.1
|
||||
sentencepiece==0.1.99
|
||||
@@ -159,7 +160,7 @@ torchvision==0.15.2
|
||||
tornado==6.3.2
|
||||
tqdm==4.65.0
|
||||
traitlets==5.9.0
|
||||
transformers==4.29.2
|
||||
transformers @ git+https://github.com/huggingface/transformers.git@17a55534f5e5df10ac4804d4270bf6b8cc24998d
|
||||
triton==2.0.0
|
||||
typing_extensions @ file:///home/conda/feedstock_root/build_artifacts/typing_extensions_1678559861143/work
|
||||
tzdata==2023.3
|
||||
|
||||
@@ -111,7 +111,7 @@ dependencies:
|
||||
- zlib=1.2.13=h166bdaf_4
|
||||
- zstd=1.5.2=h3eb15da_6
|
||||
- pip:
|
||||
- accelerate==0.19.0
|
||||
- accelerate==0.20.0.dev0
|
||||
- aiohttp==3.8.4
|
||||
- aiosignal==1.3.1
|
||||
- altair==5.0.0
|
||||
@@ -124,7 +124,7 @@ dependencies:
|
||||
- backcall==0.2.0
|
||||
- base58==2.1.1
|
||||
- beautifulsoup4==4.12.2
|
||||
- bitsandbytes==0.37.2
|
||||
- bitsandbytes==0.39.0
|
||||
- black==21.12b0
|
||||
- blessed==1.20.0
|
||||
- blinker==1.6.2
|
||||
@@ -188,7 +188,7 @@ dependencies:
|
||||
- pandas==2.0.1
|
||||
- parso==0.8.3
|
||||
- pathspec==0.11.1
|
||||
- peft==0.3.0.dev0
|
||||
- peft==0.4.0.dev0
|
||||
- pexpect==4.8.0
|
||||
- pickleshare==0.7.5
|
||||
- platformdirs==3.5.1
|
||||
@@ -226,6 +226,7 @@ dependencies:
|
||||
- regex==2023.5.5
|
||||
- responses==0.18.0
|
||||
- rich==13.3.5
|
||||
- safetensors==0.3.1
|
||||
- scikit-learn==1.2.2
|
||||
- scipy==1.10.1
|
||||
- sentencepiece==0.1.99
|
||||
@@ -250,7 +251,7 @@ dependencies:
|
||||
- tornado==6.3.2
|
||||
- tqdm==4.65.0
|
||||
- traitlets==5.9.0
|
||||
- transformers==4.29.2
|
||||
- transformers==4.30.0.dev0
|
||||
- tzdata==2023.3
|
||||
- tzlocal==5.0.1
|
||||
- uvicorn==0.22.0
|
||||
|
||||
Reference in New Issue
Block a user