mirror of
https://github.com/wassname/discovering_latent_knowledge.git
synced 2026-09-09 11:21:22 +08:00
fixed bug in batch
This commit is contained in:
deep1
+41
-1
@@ -896,6 +896,46 @@ Lesson: padding can lead to weird outputs so it's best to use an attention mask
|
||||
|
||||
- [x] revisit refactor?
|
||||
- [x] round up the FIXME TODO UPTO HACK's
|
||||
- [ ] get model nb working
|
||||
- [x] get model nb working
|
||||
- [ ] tidy
|
||||
- [ ] do multiple datasets (esp TruthfullQA) adverseria_qa commonsense_qa.
|
||||
- [ ] in fact can I consume elk [defs](https://github.com/EleutherAI/elk/blob/main/elk/promptsource/templates/adversarial_qa/adversarialQA/templates.yaml)?
|
||||
|
||||
|
||||
# 2023-08-07 08:24:43
|
||||
|
||||
Oh no it's not generalising. And I realised that by having multiple duplicate datasets I was mixing test and train duh! Start again
|
||||
|
||||
- normalise stops it from overfitting... or learnign at all? What's going on. Did I mix up test train os hs1 hs2?
|
||||
|
||||
huh in the dm notebook I get 100 and 60% with linear cls. But in 023 I get 100 50% weird. And with norm I get 50% 50%
|
||||
|
||||
:bug: I had hs0 hs0, wtf
|
||||
|
||||
err so manbe ranking is not the best! do I need to try other models again?
|
||||
|
||||
- [ ] try other setups? cls, (hs0-hs1)/y etc
|
||||
- [ ] try restricting to question where it can answer it?
|
||||
- [ ] try removign truncated ones?
|
||||
|
||||
|
||||
wait what? when the model tries to lie... we get this acc 0.49
|
||||
|
||||
|
||||
wait
|
||||
- test metrics says it works
|
||||
- but train and val don't!
|
||||
- and my custon ones dont?
|
||||
|
||||
|
||||
on one hand we have acc at prob predicting ans1>ans2
|
||||
on the other prob at predicting label
|
||||
on another llm at answer
|
||||
|
||||
So I can predict if one is more positvie than other
|
||||
at least using ranking loss. hmm
|
||||
|
||||
|
||||
wooo true and label are diff!!! even tho they come from the same source :bug:
|
||||
|
||||
found the bug, I shuffled X but not y lol
|
||||
|
||||
+1215
-2081
File diff suppressed because one or more lines are too long
+279
-182
@@ -11,7 +11,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -32,20 +32,9 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'4.30.1'"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
@@ -73,7 +62,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -95,7 +84,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -108,8 +97,8 @@
|
||||
"dataset_params = dict(\n",
|
||||
" model_repo=\"HuggingFaceH4/starchat-beta\",\n",
|
||||
" dataset_name = \"amazon_polarity\",\n",
|
||||
" N = 8000, # 8000 # 4000 in 4 hours\n",
|
||||
" N_SHOTS = 3,\n",
|
||||
" N = 509, # 8000 # 4000 in 4 hours\n",
|
||||
" N_SHOTS = 2,\n",
|
||||
" prompt_fmt=format_guard_prompt,\n",
|
||||
" choices=default_class2choices,\n",
|
||||
")\n",
|
||||
@@ -142,7 +131,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": 30,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -154,105 +143,136 @@
|
||||
"\u001b[1mchanging truncation_side from right to left\u001b[0m\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"===================================BUG REPORT===================================\n",
|
||||
"Welcome to bitsandbytes. For bug reports, please run\n",
|
||||
"\n",
|
||||
"python -m bitsandbytes\n",
|
||||
"\n",
|
||||
" and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
|
||||
"================================================================================\n",
|
||||
"bin /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so\n",
|
||||
"CUDA SETUP: CUDA runtime path found: /home/ubuntu/mambaforge/envs/dlk2/lib/libcudart.so.11.0\n",
|
||||
"CUDA SETUP: Highest compute capability among GPUs detected: 8.6\n",
|
||||
"CUDA SETUP: Detected CUDA version 117\n",
|
||||
"CUDA SETUP: Loading binary /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/home/ubuntu/mambaforge/envs/dlk2/lib/libcudart.so.11.0'), PosixPath('/home/ubuntu/mambaforge/envs/dlk2/lib/libcudart.so')}.. We'll flip a coin and try one of these, in order to fail forward.\n",
|
||||
"Either way, this might cause trouble in the future:\n",
|
||||
"If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.\n",
|
||||
" warn(msg)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "0dd2928fd7c24f2ca8a9fac1745d7987",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/html": [
|
||||
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800000; text-decoration-color: #800000\">╭─────────────────────────────── </span><span style=\"color: #800000; text-decoration-color: #800000; font-weight: bold\">Traceback </span><span style=\"color: #bf7f7f; text-decoration-color: #bf7f7f; font-weight: bold\">(most recent call last)</span><span style=\"color: #800000; text-decoration-color: #800000\"> ────────────────────────────────╮</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\"><module></span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">1</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>1 model, tokenizer = load_model(model_repo=dataset_params[<span style=\"color: #808000; text-decoration-color: #808000\">'model_repo'</span>]) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">2 </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/src/models/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">load.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">23</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">load_model</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">20 </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">21 </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">def</span> <span style=\"color: #00ff00; text-decoration-color: #00ff00\">load_model</span>(model_repo = <span style=\"color: #808000; text-decoration-color: #808000\">\"HuggingFaceH4/starchat-beta\"</span>, lora_repo=<span style=\"color: #0000ff; text-decoration-color: #0000ff\">None</span>, verbose=<span style=\"color: #0000ff; text-decoration-color: #0000ff\">True</span>) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">22 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">if</span> <span style=\"color: #808000; text-decoration-color: #808000\">\"starchat\"</span> <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">in</span> model_repo: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>23 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ </span>model, tokenizer = load_starchat(model_repo=model_repo) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">24 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"># elif \"llama\" in model_repo:</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">25 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"># model, tokenizer = load_llama(model_repo=model_repo, lora_repo=lora_repo)</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">26 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">else</span>: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/src/models/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">load.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">51</span> in <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #00ff00; text-decoration-color: #00ff00\">load_starchat</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">48 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span>verbose_change_param(tokenizer, <span style=\"color: #808000; text-decoration-color: #808000\">'padding_side'</span>, <span style=\"color: #808000; text-decoration-color: #808000\">'left'</span>) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">49 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span>verbose_change_param(tokenizer, <span style=\"color: #808000; text-decoration-color: #808000\">'truncation_side'</span>, <span style=\"color: #808000; text-decoration-color: #808000\">'left'</span>) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">50 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>51 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span>model = AutoModelForCausalLM.from_pretrained(model_repo, config=config, **model_opti <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">52 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">53 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> model, tokenizer <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">54 </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/models/auto/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">auto_fact</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">ory.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">484</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">from_pretrained</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">481 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ </span>) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">482 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">elif</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">type</span>(config) <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">in</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">cls</span>._model_mapping.keys(): <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">483 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ </span>model_class = _get_model_class(config, <span style=\"color: #00ffff; text-decoration-color: #00ffff\">cls</span>._model_mapping) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>484 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> model_class.from_pretrained( <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">485 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ </span>pretrained_model_name_or_path, *model_args, config=config, **hub_kwargs, <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">486 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ </span>) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">487 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">raise</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">ValueError</span>( <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">modeling_utils.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">281</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #0000ff; text-decoration-color: #0000ff\">9</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">from_pretrained</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">2816 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ │ </span>key: device_map[key] <span style=\"color: #0000ff; text-decoration-color: #0000ff\">for</span> key <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">in</span> device_map.keys() <span style=\"color: #0000ff; text-decoration-color: #0000ff\">if</span> key <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">not</span> <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">in</span> modu <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">2817 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ </span>} <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">2818 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">if</span> <span style=\"color: #808000; text-decoration-color: #808000\">\"cpu\"</span> <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">in</span> device_map_without_lm_head.values() <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">or</span> <span style=\"color: #808000; text-decoration-color: #808000\">\"disk\"</span> <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">in</span> device_map_ <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>2819 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">raise</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">ValueError</span>( <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">2820 </span><span style=\"color: #bfbfbf; text-decoration-color: #bfbfbf\">│ │ │ │ │ │ </span><span style=\"color: #808000; text-decoration-color: #808000\">\"\"\"</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">2821 </span><span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">│ │ │ │ │ │ </span><span style=\"color: #808000; text-decoration-color: #808000\">Some modules are dispatched on the CPU or the disk. Make sure yo</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">2822 </span><span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">│ │ │ │ │ │ </span><span style=\"color: #808000; text-decoration-color: #808000\">the quantized model. If you want to dispatch the model on the CP</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
|
||||
"<span style=\"color: #800000; text-decoration-color: #800000\">╰──────────────────────────────────────────────────────────────────────────────────────────────────╯</span>\n",
|
||||
"<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-weight: bold\">ValueError: </span>\n",
|
||||
" Some modules are dispatched on the CPU or the disk. Make sure you have enough GPU RAM to \n",
|
||||
"fit\n",
|
||||
" the quantized model. If you want to dispatch the model on the CPU or the disk while keeping\n",
|
||||
" these modules in <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">32</span>-bit, you need to set `<span style=\"color: #808000; text-decoration-color: #808000\">load_in_8bit_fp32_cpu_offload</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span>` and pass a \n",
|
||||
"custom\n",
|
||||
" `device_map` to `from_pretrained`. Check\n",
|
||||
" <span style=\"color: #0000ff; text-decoration-color: #0000ff; text-decoration: underline\">https://huggingface.co/docs/transformers/main/en/main_classes/quantization#offload-between-</span>\n",
|
||||
"<span style=\"color: #0000ff; text-decoration-color: #0000ff; text-decoration: underline\">cpu-and-gpu</span>\n",
|
||||
" for more details.\n",
|
||||
" \n",
|
||||
"</pre>\n"
|
||||
],
|
||||
"text/plain": [
|
||||
"Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]"
|
||||
"\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m in \u001b[92m<module>\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 model, tokenizer = load_model(model_repo=dataset_params[\u001b[33m'\u001b[0m\u001b[33mmodel_repo\u001b[0m\u001b[33m'\u001b[0m]) \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/src/models/\u001b[0m\u001b[1;33mload.py\u001b[0m:\u001b[94m23\u001b[0m in \u001b[92mload_model\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m20 \u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m21 \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mload_model\u001b[0m(model_repo = \u001b[33m\"\u001b[0m\u001b[33mHuggingFaceH4/starchat-beta\u001b[0m\u001b[33m\"\u001b[0m, lora_repo=\u001b[94mNone\u001b[0m, verbose=\u001b[94mTrue\u001b[0m) \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m22 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mstarchat\u001b[0m\u001b[33m\"\u001b[0m \u001b[95min\u001b[0m model_repo: \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m23 \u001b[2m│ │ \u001b[0mmodel, tokenizer = load_starchat(model_repo=model_repo) \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m24 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# elif \"llama\" in model_repo:\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m25 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# model, tokenizer = load_llama(model_repo=model_repo, lora_repo=lora_repo)\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m26 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/src/models/\u001b[0m\u001b[1;33mload.py\u001b[0m:\u001b[94m51\u001b[0m in \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[92mload_starchat\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m48 \u001b[0m\u001b[2m│ \u001b[0mverbose_change_param(tokenizer, \u001b[33m'\u001b[0m\u001b[33mpadding_side\u001b[0m\u001b[33m'\u001b[0m, \u001b[33m'\u001b[0m\u001b[33mleft\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m49 \u001b[0m\u001b[2m│ \u001b[0mverbose_change_param(tokenizer, \u001b[33m'\u001b[0m\u001b[33mtruncation_side\u001b[0m\u001b[33m'\u001b[0m, \u001b[33m'\u001b[0m\u001b[33mleft\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m50 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m51 \u001b[2m│ \u001b[0mmodel = AutoModelForCausalLM.from_pretrained(model_repo, config=config, **model_opti \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m52 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m53 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m model, tokenizer \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m54 \u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/models/auto/\u001b[0m\u001b[1;33mauto_fact\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[1;33mory.py\u001b[0m:\u001b[94m484\u001b[0m in \u001b[92mfrom_pretrained\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m481 \u001b[0m\u001b[2m│ │ │ \u001b[0m) \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m482 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melif\u001b[0m \u001b[96mtype\u001b[0m(config) \u001b[95min\u001b[0m \u001b[96mcls\u001b[0m._model_mapping.keys(): \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m483 \u001b[0m\u001b[2m│ │ │ \u001b[0mmodel_class = _get_model_class(config, \u001b[96mcls\u001b[0m._model_mapping) \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m484 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m model_class.from_pretrained( \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m485 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mpretrained_model_name_or_path, *model_args, config=config, **hub_kwargs, \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m486 \u001b[0m\u001b[2m│ │ │ \u001b[0m) \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m487 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mValueError\u001b[0m( \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/\u001b[0m\u001b[1;33mmodeling_utils.py\u001b[0m:\u001b[94m281\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[94m9\u001b[0m in \u001b[92mfrom_pretrained\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m2816 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0mkey: device_map[key] \u001b[94mfor\u001b[0m key \u001b[95min\u001b[0m device_map.keys() \u001b[94mif\u001b[0m key \u001b[95mnot\u001b[0m \u001b[95min\u001b[0m modu \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m2817 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m} \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m2818 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mcpu\u001b[0m\u001b[33m\"\u001b[0m \u001b[95min\u001b[0m device_map_without_lm_head.values() \u001b[95mor\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mdisk\u001b[0m\u001b[33m\"\u001b[0m \u001b[95min\u001b[0m device_map_ \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m2819 \u001b[2m│ │ │ │ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mValueError\u001b[0m( \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m2820 \u001b[0m\u001b[2;90m│ │ │ │ │ │ \u001b[0m\u001b[33m\"\"\"\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m2821 \u001b[0m\u001b[2;33m│ │ │ │ │ │ \u001b[0m\u001b[33mSome modules are dispatched on the CPU or the disk. Make sure yo\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m│\u001b[0m \u001b[2m2822 \u001b[0m\u001b[2;33m│ │ │ │ │ │ \u001b[0m\u001b[33mthe quantized model. If you want to dispatch the model on the CP\u001b[0m \u001b[31m│\u001b[0m\n",
|
||||
"\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
|
||||
"\u001b[1;91mValueError: \u001b[0m\n",
|
||||
" Some modules are dispatched on the CPU or the disk. Make sure you have enough GPU RAM to \n",
|
||||
"fit\n",
|
||||
" the quantized model. If you want to dispatch the model on the CPU or the disk while keeping\n",
|
||||
" these modules in \u001b[1;36m32\u001b[0m-bit, you need to set `\u001b[33mload_in_8bit_fp32_cpu_offload\u001b[0m=\u001b[3;92mTrue\u001b[0m` and pass a \n",
|
||||
"custom\n",
|
||||
" `device_map` to `from_pretrained`. Check\n",
|
||||
" \u001b[4;94mhttps://huggingface.co/docs/transformers/main/en/main_classes/quantization#offload-between-\u001b[0m\n",
|
||||
"\u001b[4;94mcpu-and-gpu\u001b[0m\n",
|
||||
" for more details.\n",
|
||||
" \n"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"GPTBigCodeConfig {\n",
|
||||
" \"_name_or_path\": \"HuggingFaceH4/starchat-beta\",\n",
|
||||
" \"activation_function\": \"gelu\",\n",
|
||||
" \"architectures\": [\n",
|
||||
" \"GPTBigCodeForCausalLM\"\n",
|
||||
" ],\n",
|
||||
" \"attention_softmax_in_fp32\": true,\n",
|
||||
" \"attn_pdrop\": 0.1,\n",
|
||||
" \"bos_token_id\": 0,\n",
|
||||
" \"embd_pdrop\": 0.1,\n",
|
||||
" \"eos_token_id\": 0,\n",
|
||||
" \"inference_runner\": 0,\n",
|
||||
" \"initializer_range\": 0.02,\n",
|
||||
" \"layer_norm_epsilon\": 1e-05,\n",
|
||||
" \"max_batch_size\": null,\n",
|
||||
" \"max_sequence_length\": null,\n",
|
||||
" \"model_type\": \"gpt_bigcode\",\n",
|
||||
" \"multi_query\": true,\n",
|
||||
" \"n_embd\": 6144,\n",
|
||||
" \"n_head\": 48,\n",
|
||||
" \"n_inner\": 24576,\n",
|
||||
" \"n_layer\": 40,\n",
|
||||
" \"n_positions\": 8192,\n",
|
||||
" \"pad_key_length\": true,\n",
|
||||
" \"pre_allocate_kv_cache\": false,\n",
|
||||
" \"quantization_config\": {\n",
|
||||
" \"bnb_4bit_compute_dtype\": \"float32\",\n",
|
||||
" \"bnb_4bit_quant_type\": \"fp4\",\n",
|
||||
" \"bnb_4bit_use_double_quant\": false,\n",
|
||||
" \"llm_int8_enable_fp32_cpu_offload\": false,\n",
|
||||
" \"llm_int8_has_fp16_weight\": false,\n",
|
||||
" \"llm_int8_skip_modules\": null,\n",
|
||||
" \"llm_int8_threshold\": 6.0,\n",
|
||||
" \"load_in_4bit\": true,\n",
|
||||
" \"load_in_8bit\": false\n",
|
||||
" },\n",
|
||||
" \"resid_pdrop\": 0.1,\n",
|
||||
" \"scale_attention_softmax_in_fp32\": true,\n",
|
||||
" \"scale_attn_weights\": true,\n",
|
||||
" \"summary_activation\": null,\n",
|
||||
" \"summary_first_dropout\": 0.1,\n",
|
||||
" \"summary_proj_to_labels\": true,\n",
|
||||
" \"summary_type\": \"cls_index\",\n",
|
||||
" \"summary_use_proj\": true,\n",
|
||||
" \"torch_dtype\": \"bfloat16\",\n",
|
||||
" \"transformers_version\": \"4.30.1\",\n",
|
||||
" \"use_cache\": false,\n",
|
||||
" \"validate_runner_input\": true,\n",
|
||||
" \"vocab_size\": 49156\n",
|
||||
"}\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
@@ -269,7 +289,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": 31,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -282,7 +302,7 @@
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "ea78bafb01d24468a631c30944ab355c",
|
||||
"model_id": "6ee109e341e5461b8d62bc59f2dc64b3",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
@@ -308,7 +328,7 @@
|
||||
"})"
|
||||
]
|
||||
},
|
||||
"execution_count": 6,
|
||||
"execution_count": 31,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@@ -333,7 +353,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": 32,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -364,7 +384,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 33,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -394,7 +414,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": 34,
|
||||
"metadata": {
|
||||
"notebookRunGroups": {
|
||||
"groupValue": ""
|
||||
@@ -402,56 +422,23 @@
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "d68576e08c3a45b0a45fac147e268b62",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Map: 0%| | 0/8000 [00:00<?, ? examples/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Loading cached processed dataset at /home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/cache-88dc9f1fd8b901a8.arrow\n",
|
||||
"Loading cached processed dataset at /home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/cache-e752326d3340220d.arrow\n",
|
||||
"Loading cached processed dataset at /home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/cache-7cb61ae701a89a12.arrow\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "562f5db7a1ac45a4aedb2609daa61969",
|
||||
"model_id": "4550272988a24d68bc8b9376a0c81fe7",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Map: 0%| | 0/8000 [00:00<?, ? examples/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "c131b9f5af884b32b5c521def0d913d7",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Map: 0%| | 0/8000 [00:00<?, ? examples/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "66fc9b05899e4b028a4e192ce9d558b8",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Map: 0%| | 0/8000 [00:00<?, ? examples/s]"
|
||||
"Map: 0%| | 0/509 [00:00<?, ? examples/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
@@ -462,11 +449,11 @@
|
||||
"text/plain": [
|
||||
"Dataset({\n",
|
||||
" features: ['label', 'title', 'content', 'text', 'prompt', 'lie', 'input_ids', 'attention_mask', 'prompt_truncated'],\n",
|
||||
" num_rows: 8000\n",
|
||||
" num_rows: 509\n",
|
||||
"})"
|
||||
]
|
||||
},
|
||||
"execution_count": 9,
|
||||
"execution_count": 34,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@@ -504,14 +491,14 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"execution_count": 35,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"../.ds/model-starchat-beta_ds-amazon-polarity_format-guard-prompt_N8000_3shots_07e51a\n"
|
||||
"../.ds/model-starchat-beta_ds-amazon-polarity_format-guard-prompt_N509_2shots_5c2070\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -526,7 +513,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"execution_count": 36,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -562,13 +549,13 @@
|
||||
" 'tokenizer': GPT2TokenizerFast(name_or_path='HuggingFaceH4/starchat-beta', vocab_size=49152, model_max_length=1000000000000000019884624838656, is_fast=True, padding_side='left', truncation_side='left', special_tokens={'bos_token': '<|endoftext|>', 'eos_token': '<|endoftext|>', 'unk_token': '<|endoftext|>', 'pad_token': '<|endoftext|>', 'additional_special_tokens': ['<|system|>', '<|user|>', '<|assistant|>', '<|end|>']}, clean_up_tokenization_spaces=True),\n",
|
||||
" 'data': Dataset({\n",
|
||||
" features: ['label', 'title', 'content', 'text', 'prompt', 'lie', 'input_ids', 'attention_mask', 'prompt_truncated'],\n",
|
||||
" num_rows: 8000\n",
|
||||
" num_rows: 509\n",
|
||||
" }),\n",
|
||||
" 'n': 8000,\n",
|
||||
" 'n': 509,\n",
|
||||
" 'batch_size': 10}"
|
||||
]
|
||||
},
|
||||
"execution_count": 11,
|
||||
"execution_count": 36,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@@ -586,20 +573,20 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"execution_count": 37,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Downloading and preparing dataset None/../.ds/model-starchat-beta_ds-amazon-polarity_format-guard-prompt_N8000_3shots_07e51a to /home/ubuntu/.cache/huggingface/datasets/generator/default-d0b705df6ed67cbb/0.0.0...\n"
|
||||
"Downloading and preparing dataset None/../.ds/model-starchat-beta_ds-amazon-polarity_format-guard-prompt_N509_2shots_5c2070 to /home/ubuntu/.cache/huggingface/datasets/generator/default-f46bbb923bbf3943/0.0.0...\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "5b24dccc99e34c3fbc66a774977d7a66",
|
||||
"model_id": "6aa11b88940248c199146d73079893d5",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
@@ -613,12 +600,12 @@
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "e8e2265e851f4f309ff0170511d31b59",
|
||||
"model_id": "f4dd1c7bebb84bb6998442b6e821df80",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"get hidden states: 0%| | 0/800 [00:00<?, ?it/s]"
|
||||
"get hidden states: 0%| | 0/51 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
@@ -630,14 +617,22 @@
|
||||
" generator=batch_hidden_states,\n",
|
||||
" info=DatasetInfo(\n",
|
||||
" description=f\"kwargs={info_kwargs} dataset_params={dataset_params}\",\n",
|
||||
" config_name=f,\n",
|
||||
" \n",
|
||||
" config_name=f, \n",
|
||||
" ),\n",
|
||||
" gen_kwargs=gen_kwargs,\n",
|
||||
").with_format(\"numpy\")\n",
|
||||
"ds1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%debug"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
@@ -661,23 +656,17 @@
|
||||
"class2_ids = choice2ids(tokenizer, dataset_params['choices'])\n",
|
||||
"add_txt_ans0 = lambda r: {'txt_ans0': tokenizer.decode(r['scores0'].argmax(-1))}\n",
|
||||
"add_txt_ans1 = lambda r: {'txt_ans1': tokenizer.decode(r['scores1'].argmax(-1))}\n",
|
||||
"add_ans = lambda r: scores2choice_probs(r, class2_ids)\n",
|
||||
"\n",
|
||||
"ds3 = (\n",
|
||||
" ds1\n",
|
||||
" .map(lambda r: scores2choice_probs(r, class2_ids))\n",
|
||||
" .map(add_ans)\n",
|
||||
" .map(add_txt_ans0)\n",
|
||||
" .map(add_txt_ans1)\n",
|
||||
")\n",
|
||||
"ds3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
@@ -816,6 +805,114 @@
|
||||
"# print(\"-\" * 80)\n",
|
||||
"# print(\"label\", r['label'])\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"notebookRunGroups": {
|
||||
"groupValue": "2"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"# QC: linear probe"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"hs = ds4['hs1']-ds4['hs0']\n",
|
||||
"X = hs.reshape(hs.shape[0], -1)\n",
|
||||
"y = (ds4['ans1'] - ds4['ans0'])>0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"true_switch_sign = ds4['label'][:, 0]*2-1\n",
|
||||
"true_switch_sign = ds4['true'][:, 0]*2-1\n",
|
||||
"y = ((ds4['ans1'] - ds4['ans0']) * true_switch_sign) > 0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# n = len(df)\n",
|
||||
"from sklearn.preprocessing import RobustScaler\n",
|
||||
"from sklearn.linear_model import LogisticRegression\n",
|
||||
"from sklearn.metrics import f1_score, roc_auc_score, accuracy_score\n",
|
||||
"\n",
|
||||
"# # Define X and y\n",
|
||||
"# X = dm.hs1-dm.hs2\n",
|
||||
"# y = dm.y>0\n",
|
||||
"\n",
|
||||
"# split\n",
|
||||
"n = len(y)\n",
|
||||
"max_rows = 1000\n",
|
||||
"print('split size', n//2)\n",
|
||||
"X_train, X_test = X[:n//2], X[n//2:]\n",
|
||||
"y_train, y_test = y[:n//2], y[n//2:]\n",
|
||||
"X_train = X_train[:max_rows]\n",
|
||||
"y_train = y_train[:max_rows]\n",
|
||||
"X_test = X_test[:max_rows]\n",
|
||||
"y_test = y_test[:max_rows]\n",
|
||||
"\n",
|
||||
"# scale\n",
|
||||
"scaler = RobustScaler()\n",
|
||||
"scaler.fit(X_train)\n",
|
||||
"X_train2 = scaler.transform(X_train)\n",
|
||||
"X_test2 = scaler.transform(X_test)\n",
|
||||
"print('lr')\n",
|
||||
"\n",
|
||||
"lr = LogisticRegression(class_weight=\"balanced\", penalty=\"l2\", max_iter=380)\n",
|
||||
"lr.fit(X_train2, y_train>0)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(\"Logistic cls acc: {:2.2%} [TRAIN]\".format(lr.score(X_train2, y_train>0)))\n",
|
||||
"print(\"Logistic cls acc: {:2.2%} [TEST]\".format(lr.score(X_test2, y_test>0)))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
from tqdm.auto import tqdm
|
||||
from src.datasets.hs import ExtractHiddenStates
|
||||
from torch.utils.data import DataLoader
|
||||
from datasets import Dataset
|
||||
from datasets.arrow_dataset import Dataset
|
||||
import hashlib
|
||||
import pickle
|
||||
import numpy as np
|
||||
@@ -23,7 +23,7 @@ def batch_hidden_states(model, tokenizer, data: Dataset, n=100, batch_size=2, mc
|
||||
ds_p_subset = data.select(range(n))
|
||||
ds_p_subset.set_format(type="pandas", columns=['lie', 'label', 'prompt', 'prompt_truncated'])
|
||||
|
||||
dl = DataLoader(ds_t_subset, batch_size=batch_size, shuffle=True)
|
||||
dl = DataLoader(ds_t_subset, batch_size=batch_size, shuffle=False)
|
||||
for i, batch in enumerate(tqdm(dl, desc='get hidden states')):
|
||||
input_ids, true_labels, attention_mask = batch["input_ids"], batch["label"], batch["attention_mask"]
|
||||
nn = len(input_ids)
|
||||
@@ -47,7 +47,9 @@ def batch_hidden_states(model, tokenizer, data: Dataset, n=100, batch_size=2, mc
|
||||
for j in range(nn):
|
||||
# let's add the non torch metadata like label, prompt, lie, etc
|
||||
k = i*batch_size + j
|
||||
info = ds_p_subset[k]
|
||||
info = ds_p_subset[k].iloc[0].to_dict()
|
||||
|
||||
assert info['label']==true_labels[j].item(), 'these should line up'
|
||||
|
||||
yield dict(
|
||||
hs0=hs0['hidden_states'][j],
|
||||
@@ -56,8 +58,8 @@ def batch_hidden_states(model, tokenizer, data: Dataset, n=100, batch_size=2, mc
|
||||
hs1=hs1['hidden_states'][j],
|
||||
scores1=hs1["scores"][j],
|
||||
|
||||
true=true_labels[j].item(),
|
||||
index=index[j],
|
||||
label_b=true_labels[j].item(),
|
||||
ds_index=index[j],
|
||||
|
||||
**info
|
||||
)
|
||||
|
||||
+24
-33
@@ -2,18 +2,20 @@ import torch
|
||||
import torch.nn as nn
|
||||
import lightning as pl
|
||||
import pandas as pd
|
||||
from torch.utils.data import Dataset, DataLoader, TensorDataset
|
||||
from torch.utils.data import DataLoader, TensorDataset
|
||||
from src.datasets.load import ds2df
|
||||
from datasets.arrow_dataset import Dataset
|
||||
|
||||
def make_y(df):
|
||||
# label: is ans2 more true than ans1
|
||||
# so we ask does ans2 have greater probability on "positive" than ans1
|
||||
# then, when the right answer is negative we swap the sign
|
||||
true_switch_sign = df.label*2-1
|
||||
def compute_distance(df):
|
||||
"""distance between ans1 and ans2."""
|
||||
true_switch_sign = df.true*2-1 # switch sign to desired answer. with this we ask which is more true
|
||||
# otherwise we ask which is more positive
|
||||
distance = (df.ans1-df.ans0) * true_switch_sign
|
||||
# y = bool2switch(distance>0)
|
||||
return distance
|
||||
|
||||
to_tensor = lambda x: torch.from_numpy(x).float()
|
||||
to_ds = lambda hs0, hs1, y: TensorDataset(to_tensor(hs0), to_tensor(hs1), to_tensor(y))
|
||||
|
||||
class imdbHSDataModule(pl.LightningDataModule):
|
||||
|
||||
def __init__(self,
|
||||
@@ -22,7 +24,7 @@ class imdbHSDataModule(pl.LightningDataModule):
|
||||
):
|
||||
super().__init__()
|
||||
self.save_hyperparameters(ignore=["ds"])
|
||||
self.ds = ds.shuffle(seed=42)
|
||||
self.ds = ds#.shuffle(seed=42)
|
||||
|
||||
def setup(self, stage: str):
|
||||
h = self.hparams
|
||||
@@ -34,46 +36,35 @@ class imdbHSDataModule(pl.LightningDataModule):
|
||||
)
|
||||
self.df = ds2df(self.ds)
|
||||
|
||||
y_cls = make_y(self.df)
|
||||
y_cls = compute_distance(self.df)
|
||||
|
||||
self.y = y_cls.values
|
||||
self.df['y'] = y_cls
|
||||
|
||||
b = len(self.ds_hs)
|
||||
self.hs1 = self.ds_hs['hs0'].transpose(0, 2, 1)
|
||||
self.hs2 = self.ds_hs['hs1'].transpose(0, 2, 1)
|
||||
self.hs0 = self.ds_hs['hs0'].transpose(0, 2, 1)
|
||||
self.hs1 = self.ds_hs['hs1'].transpose(0, 2, 1)
|
||||
self.ans0 = self.df['ans0'].values
|
||||
self.ans1 = self.df['ans1'].values
|
||||
|
||||
# let's create a simple 50/50 train split (the data is already randomized)
|
||||
n = len(self.y)
|
||||
self.splits = {
|
||||
'train': (0, int(n * 0.5)),
|
||||
'val': (int(n * 0.5), int(n * 0.75)),
|
||||
'test': (int(n * 0.75), n),
|
||||
}
|
||||
|
||||
self.val_split = vs = int(n * 0.5)
|
||||
self.test_split = ts = int(n * 0.75)
|
||||
hs1_train, hs2_train, y_train = self.hs1[:vs], self.hs2[:vs], self.y[:vs]
|
||||
hs1_val, hs2_val, y_val = self.hs1[vs:ts], self.hs2[vs:ts], self.y[vs:ts]
|
||||
hs1_test, hs2_test, y_test = self.hs1[ts:],self. hs2[ts:], self.y[ts:]
|
||||
|
||||
|
||||
to_ds = lambda x0, x1, y: TensorDataset(torch.from_numpy(x0).float(),
|
||||
torch.from_numpy(x1).float(),
|
||||
torch.from_numpy(y).float()
|
||||
)
|
||||
self.datasets = {key: to_ds(self.hs0[start:end], self.hs1[start:end], self.y[start:end]) for key, (start, end) in self.splits.items()}
|
||||
|
||||
self.ds_train = to_ds(hs1_train, hs2_train, y_train)
|
||||
|
||||
self.ds_val = to_ds(hs1_val, hs2_val, y_val)
|
||||
|
||||
self.ds_test = to_ds(hs1_test, hs2_test, y_test)
|
||||
def create_dataloader(self, ds, shuffle=False):
|
||||
return DataLoader(ds, batch_size=self.hparams.batch_size, drop_last=True, shuffle=shuffle)
|
||||
|
||||
def train_dataloader(self):
|
||||
return DataLoader(self.ds_train,
|
||||
batch_size=self.hparams.batch_size,
|
||||
drop_last=True,
|
||||
shuffle=True)
|
||||
return self.create_dataloader(self.datasets['train'], shuffle=True)
|
||||
|
||||
def val_dataloader(self):
|
||||
return DataLoader(self.ds_val, batch_size=self.hparams.batch_size, drop_last=True,)
|
||||
return self.create_dataloader(self.datasets['val'])
|
||||
|
||||
def test_dataloader(self):
|
||||
return DataLoader(self.ds_test, batch_size=self.hparams.batch_size, drop_last=True,)
|
||||
return self.create_dataloader(self.datasets['test'])
|
||||
|
||||
+3
-2
@@ -62,7 +62,7 @@ def get_choices_as_tokens(
|
||||
tokenizer, choices:List[str] = ["Positive"], whitespace_first=True
|
||||
) -> List[int]:
|
||||
|
||||
# Note some tokenizers differentiate between "no", "\nno", so we sometime need to add whitespace beforehand...
|
||||
# Note some tokenizers differentiate between "yes", "\nyes" and " yes", so we sometime need to add whitespace beforehand...
|
||||
if not whitespace_first:
|
||||
raise NotImplementedError('TODO')
|
||||
|
||||
@@ -72,7 +72,7 @@ def get_choices_as_tokens(
|
||||
ids.append(id_)
|
||||
|
||||
c2 = tokenizer.decode([id_])
|
||||
assert tokenizer.decode([id_]) == c, f'tokenizer.decode(tokenizer(`{c}`))==`{c2}`!=`{c}`'
|
||||
assert tokenizer.decode([id_]) == c, f'We should be able to encode and decode the choices, but it failed: tokenizer.decode(tokenizer(`{c}`))==`{c2}`!=`{c}`'
|
||||
|
||||
return ids
|
||||
|
||||
@@ -154,6 +154,7 @@ class ExtractHiddenStates:
|
||||
hidden_states=hidden_states,
|
||||
scores=outputs["scores"],
|
||||
input_ids=input_ids,
|
||||
layers=layers,
|
||||
)
|
||||
out = {k: to_numpy(v) for k, v in out.items()}
|
||||
if debug:
|
||||
|
||||
@@ -2,8 +2,10 @@ from pytorch_optimizer import Ranger21
|
||||
import torchmetrics
|
||||
import lightning.pytorch as pl
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import torch.nn as nn
|
||||
from torchmetrics import Metric, MetricCollection, Accuracy, AUROC
|
||||
from torchmetrics.functional import accuracy
|
||||
|
||||
from src.helpers import switch2bool, bool2switch
|
||||
|
||||
@@ -15,20 +17,9 @@ class PLRanking(pl.LightningModule):
|
||||
"""
|
||||
def __init__(self, c_in, total_steps, depth=1, hs=16, lr=4e-3, weight_decay=1e-9, dropout=0):
|
||||
super().__init__()
|
||||
# self.probe = MLPProbe(c_in, depth=depth, dropout=dropout, hs=hs)
|
||||
self.probe = None # subclasses must add this
|
||||
self.save_hyperparameters()
|
||||
|
||||
self.loss_fn = nn.SmoothL1Loss()
|
||||
|
||||
# metrics for each stage
|
||||
metrics_template = MetricCollection({
|
||||
'acc': Accuracy(task="binary"),
|
||||
'auroc': AUROC(task="binary")
|
||||
})
|
||||
self.metrics = torch.nn.ModuleDict({
|
||||
f'metrics_{stage}': metrics_template.clone(prefix=stage+'/') for stage in ['train', 'val', 'test']
|
||||
})
|
||||
|
||||
def forward(self, x):
|
||||
return self.probe(x).squeeze(1)
|
||||
|
||||
@@ -40,14 +31,14 @@ class PLRanking(pl.LightningModule):
|
||||
if stage=='pred':
|
||||
return (ypred1-ypred0).float()
|
||||
|
||||
loss = self.loss_fn(ypred1-ypred0, y)
|
||||
self.log(f"{stage}/loss", loss)
|
||||
|
||||
m = self.metrics[f'metrics_{stage}']
|
||||
loss = F.smooth_l1_loss(ypred1-ypred0, y)
|
||||
# self.log(f"{stage}/loss", loss)
|
||||
|
||||
y_cls = switch2bool(ypred1-ypred0)
|
||||
m(y_cls, y>0.)
|
||||
self.log_dict(m, on_epoch=True, on_step=False)
|
||||
self.log_dict({
|
||||
f"{stage}/acc": accuracy(y_cls, y>0, "binary"),
|
||||
f"{stage}/loss": loss,
|
||||
}, on_epoch=True, on_step=False),
|
||||
return loss
|
||||
|
||||
def training_step(self, batch, batch_idx=0, dataloader_idx=0):
|
||||
|
||||
Reference in New Issue
Block a user