diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000..371c111
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,3 @@
+{
+ "python.formatting.provider": "yapf"
+}
diff --git a/notebooks/001_mjc_CCS.ipynb b/notebooks/001_mjc_CCS.ipynb
index fcc5834..30eb6f2 100644
--- a/notebooks/001_mjc_CCS.ipynb
+++ b/notebooks/001_mjc_CCS.ipynb
@@ -10,23 +10,14 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T01:54:44.191549Z",
"start_time": "2023-05-20T01:54:41.824251Z"
}
},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
- " from .autonotebook import tqdm as notebook_tqdm\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"from tqdm.auto import tqdm\n",
"import copy\n",
@@ -67,7 +58,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T01:54:44.196607Z",
@@ -82,122 +73,14 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T01:56:26.440636Z",
"start_time": "2023-05-20T01:54:44.197666Z"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "===================================BUG REPORT===================================\n",
- "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
- "================================================================================\n",
- "CUDA SETUP: CUDA runtime path found: /home/ubuntu/mambaforge/envs/dlk2/lib/libcudart.so\n",
- "CUDA SETUP: Highest compute capability among GPUs detected: 8.6\n",
- "CUDA SETUP: Detected CUDA version 117\n",
- "CUDA SETUP: Loading binary /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Loading checkpoint shards: 100%|██████████| 2/2 [00:06<00:00, 3.43s/it]\n",
- "Downloading (…)/adapter_config.json: 100%|██████████| 428/428 [00:00<00:00, 133kB/s]\n",
- "Downloading adapter_model.bin: 100%|██████████| 67.2M/67.2M [00:07<00:00, 9.60MB/s]\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "(LlamaTokenizer(name_or_path='Neko-Institute-of-Science/LLaMA-7B-HF', vocab_size=32000, model_max_length=1000000000000000019884624838656, is_fast=False, padding_side='right', truncation_side='right', special_tokens={'bos_token': AddedToken(\"\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'eos_token': AddedToken(\"\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'unk_token': AddedToken(\"\", rstrip=False, lstrip=False, single_word=False, normalized=True)}, clean_up_tokenization_spaces=False),\n",
- " PeftModelForCausalLM(\n",
- " (base_model): LoraModel(\n",
- " (model): LlamaForCausalLM(\n",
- " (model): LlamaModel(\n",
- " (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n",
- " (layers): ModuleList(\n",
- " (0-31): 32 x LlamaDecoderLayer(\n",
- " (self_attn): LlamaAttention(\n",
- " (q_proj): Linear8bitLt(\n",
- " in_features=4096, out_features=4096, bias=False\n",
- " (lora_dropout): ModuleDict(\n",
- " (default): Dropout(p=0.05, inplace=False)\n",
- " )\n",
- " (lora_A): ModuleDict(\n",
- " (default): Linear(in_features=4096, out_features=16, bias=False)\n",
- " )\n",
- " (lora_B): ModuleDict(\n",
- " (default): Linear(in_features=16, out_features=4096, bias=False)\n",
- " )\n",
- " )\n",
- " (k_proj): Linear8bitLt(\n",
- " in_features=4096, out_features=4096, bias=False\n",
- " (lora_dropout): ModuleDict(\n",
- " (default): Dropout(p=0.05, inplace=False)\n",
- " )\n",
- " (lora_A): ModuleDict(\n",
- " (default): Linear(in_features=4096, out_features=16, bias=False)\n",
- " )\n",
- " (lora_B): ModuleDict(\n",
- " (default): Linear(in_features=16, out_features=4096, bias=False)\n",
- " )\n",
- " )\n",
- " (v_proj): Linear8bitLt(\n",
- " in_features=4096, out_features=4096, bias=False\n",
- " (lora_dropout): ModuleDict(\n",
- " (default): Dropout(p=0.05, inplace=False)\n",
- " )\n",
- " (lora_A): ModuleDict(\n",
- " (default): Linear(in_features=4096, out_features=16, bias=False)\n",
- " )\n",
- " (lora_B): ModuleDict(\n",
- " (default): Linear(in_features=16, out_features=4096, bias=False)\n",
- " )\n",
- " )\n",
- " (o_proj): Linear8bitLt(\n",
- " in_features=4096, out_features=4096, bias=False\n",
- " (lora_dropout): ModuleDict(\n",
- " (default): Dropout(p=0.05, inplace=False)\n",
- " )\n",
- " (lora_A): ModuleDict(\n",
- " (default): Linear(in_features=4096, out_features=16, bias=False)\n",
- " )\n",
- " (lora_B): ModuleDict(\n",
- " (default): Linear(in_features=16, out_features=4096, bias=False)\n",
- " )\n",
- " )\n",
- " (rotary_emb): LlamaRotaryEmbedding()\n",
- " )\n",
- " (mlp): LlamaMLP(\n",
- " (gate_proj): Linear8bitLt(in_features=4096, out_features=11008, bias=False)\n",
- " (down_proj): Linear8bitLt(in_features=11008, out_features=4096, bias=False)\n",
- " (up_proj): Linear8bitLt(in_features=4096, out_features=11008, bias=False)\n",
- " (act_fn): SiLUActivation()\n",
- " )\n",
- " (input_layernorm): LlamaRMSNorm()\n",
- " (post_attention_layernorm): LlamaRMSNorm()\n",
- " )\n",
- " )\n",
- " (norm): LlamaRMSNorm()\n",
- " )\n",
- " (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n",
- " )\n",
- " )\n",
- " ))"
- ]
- },
- "execution_count": 3,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"# Here are a few different model options you can play around with:\n",
"model_name = \"deberta\"\n",
@@ -229,10 +112,17 @@
" model.parallelize() # T5 is big enough that we may need to run it on multiple GPUs\n",
"elif (\"llama\" in model_name) or (\"alpaca\" in model_name):\n",
" # https://github.com/deep-diver/LLM-As-Chatbot/blob/216abb559d00a0555f41a1426ac9db6c1abc24f3/models/alpaca.py\n",
+ " \n",
+ " # working\n",
" model_repo = \"Neko-Institute-of-Science/LLaMA-7B-HF\"\n",
+ " lora_repo = \"chansung/gpt4-alpaca-lora-7b\"\n",
+ " \n",
+ " model_repo = \"Neko-Institute-of-Science/LLaMA-13B-HF\"\n",
+ " lora_repo = \"chansung/gpt4-alpaca-lora-13b\"\n",
+ " \n",
" # model_repo = \"decapoda-research/llama-7b-hf\"\n",
" # lora_repo = \"tloen/alpaca-lora-7b\"\n",
- " lora_repo = \"chansung/gpt4-alpaca-lora-7b\"\n",
+ " \n",
" \n",
" # model_repo = \"Neko-Institute-of-Science/LLaMA-13B-HF\"\n",
" # lora_repo = \"LLMs/Alpaca-LoRA-13B-elina\"\n",
@@ -271,28 +161,17 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T01:56:26.469934Z",
"start_time": "2023-05-20T01:56:26.444768Z"
}
},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "(29900, 29896)"
- ]
- },
- "execution_count": 4,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"# get the tokens for 0 and 1, we will use these later...\n",
- "id_0, id_1 = tokenizer('0')['input_ids'][-1], tokenizer('1')['input_ids'][-1]\n",
+ "id_0, id_1 = tokenizer('n')['input_ids'][-1], tokenizer('y')['input_ids'][-1]\n",
"id_0, id_1"
]
},
@@ -321,7 +200,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T01:56:27.020627Z",
@@ -329,29 +208,7 @@
},
"scrolled": false
},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Found cached dataset amazon_polarity (/home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n",
- "100%|██████████| 2/2 [00:02<00:00, 1.10s/it]\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "Dataset({\n",
- " features: ['label', 'title', 'content'],\n",
- " num_rows: 400000\n",
- "})"
- ]
- },
- "execution_count": 5,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"# debug\n",
"datasets.logging.set_verbosity_info()\n",
@@ -364,29 +221,14 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T02:25:05.461369Z",
"start_time": "2023-05-20T02:25:05.458241Z"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n",
- "Question: Is this review positive? \n",
- "Answer: 1\n",
- "---\n",
- "Review: \"The movie was the worst.... not!\"\n",
- "Question: Is this review negative?\n",
- "Answer: \n",
- "\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"def format_imdb(text, label):\n",
" return f\"\"\"Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n",
@@ -406,20 +248,43 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
- "def format_imdb(input, label):\n",
- " instruction = f\"Is this review {'positive' if label else 'negative'}?\"\n",
- " alpaca_prompt = f\"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\n{instruction}\\n\\n### Input:\\n{input}\\n\\n### Response:\\n\"\n",
+ "def format_imdb(input, label, prefix=1):\n",
+ " s_prefix = \"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n\" if prefix else \"\"\n",
+ " q = f\"Is this review {'positive' if label else 'negative'} (y/n)?\"\n",
+ " if label==1:\n",
+ " l = \"y\"\n",
+ " elif label==0:\n",
+ " l=\"n\"\n",
+ " else:\n",
+ " l = \"?\"\n",
+ " alpaca_prompt = f'{s_prefix}### Instruction:\\n{q}\"\\n\\n### Input:\\n\\n\"{input}\"\\n\\n### Response: {l}'\n",
" return alpaca_prompt"
]
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def format_imbd_multishot(input, label):\n",
+ " a = format_imdb(\"Although Hypnotic isn't without glimmers of inspiration, the ultimate effect of this often clunky crime caper will be to leave you feeling rather sleepy.\", 0, prefix=1)\n",
+ " b = format_imdb(\"Although Hypnotic isn't without it's downsides, the ultimate effect of this often funny crime caper will be to leave you feeling rather fascinated.\", 1, prefix=0)\n",
+ " c = format_imdb(input, \"\", prefix=0)\n",
+ " return \"\\n\\n\".join([a, b, c])\n",
+ "\n",
+ "text=\"The movie was the worst.... not!\"\n",
+ "print(format_imbd_multishot(text, 0))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -447,25 +312,14 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T02:25:05.902898Z",
"start_time": "2023-05-20T02:25:05.898745Z"
}
},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "66"
- ]
- },
- "execution_count": 9,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"# tokens\n",
"len(tokenizer(format_imdb(\"The movie was the worst.... not!\", 0))['input_ids'])"
@@ -480,7 +334,7 @@
},
{
"cell_type": "code",
- "execution_count": 75,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T02:25:45.805378Z",
@@ -526,7 +380,7 @@
" # remove bos token? https://github.com/oobabooga/text-generation-webui/blob/1b52bddfcc70d2db88257d36f1c6d182573588c4/modules/text_generation.py#L36\n",
" if not add_bos_token and input_ids[0][0] == tokenizer.bos_token_id:\n",
" input_ids = input_ids[:, 1:]\n",
- " print('removed')\n",
+ " # print('removed')\n",
"\n",
"\n",
" # Llama adds this extra token when the first character is '\\n', and this\n",
@@ -554,20 +408,21 @@
" return_dict_in_generate=True,\n",
" output_scores=True,\n",
" )\n",
- " s = [generation_output.sequences[0]]\n",
+ " s = generation_output.sequences[0]\n",
" # print(s)\n",
" \n",
- " # text_q = tokenizer.batch_decode(input_ids, skip_special_tokens=False)\n",
- " text_ans = tokenizer.batch_decode(s, skip_special_tokens=False)#, skip_prompt=True, skip_special_tokens=True)\n",
- " # print(text_q[0])\n",
+ " text_q = tokenizer.batch_decode(input_ids, skip_special_tokens=False)\n",
+ " text_ans = tokenizer.decode(s, skip_special_tokens=False)#, skip_prompt=True, skip_special_tokens=True)\n",
+ " print(text_q[0])\n",
" # print('-'*40+'answ'+'-'*40)\n",
- " print(text_ans[0])\n",
+ " print(text_ans)\n",
+ " return text_ans\n",
" \n"
]
},
{
"cell_type": "code",
- "execution_count": 76,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T01:54:09.323908Z",
@@ -575,13 +430,11 @@
}
},
"outputs": [],
- "source": [
- "# model.generate??"
- ]
+ "source": []
},
{
"cell_type": "code",
- "execution_count": 77,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T02:25:46.260946Z",
@@ -596,7 +449,7 @@
},
{
"cell_type": "code",
- "execution_count": 78,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T02:25:46.401304Z",
@@ -611,7 +464,7 @@
},
{
"cell_type": "code",
- "execution_count": 79,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T02:25:53.587709Z",
@@ -619,43 +472,35 @@
},
"scrolled": true
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "removed\n",
- "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n",
- "\n",
- "### Instruction:\n",
- "Is this review negative?\n",
- "\n",
- "### Input:\n",
- "Despite the fact that I have only played a small portion of the game, the music I heard (plus the connection to Chrono Trigger which was great as well) led me to purchase the soundtrack, and it remains one of my favorite albums. There is an incredible mix of fun, epic, and emotional songs. Those sad and beautiful tracks I especially like, as there's not too many of those kinds of songs in my other video game soundtracks. I must admit that one of the songs (Life-A Distant Promise) has brought tears to my eyes on many occasions.My one complaint about this soundtrack is that they use guitar fretting effects in many of the songs, which I find distracting. But even if those weren't included I would still consider the collection worth it.\n",
- "\n",
- "### Response:\n",
- "Yes, based on your description, it sounds quite negative. It seems you were initially intrigued by the inclusion of classic JRPG hit \"Chrono Trigger\" into its soundtrack, but upon listening more closely found yourself turned off by several aspects such as guitar frets used extensively throughout the album or specific songs being overly sentimental or emotionally heavy. As for whether the overall feel of the review itself might be considered positive or neutral, that will vary depending on how others may interpret the tone/vocabulary chosen.\"l,p, regardless, it appears clear from reading this review that while you are ultimately satisfied with what this particular soundtrack offers musically, it fell short of meeting some important expectations on your part.<\n",
- "`quertyguess (10 points).\n",
- "\n",
- "Answer 3 questions from Science & Nature: [Genome Editing]\n",
- "[https://www.khanacademy.org/computer-programming/algorithms/genetic-algorithm/a/genomic-editing](http://%20https://www.khanacademy.org/computer-programming/algorithms/genetic-algorithm/%E2%80%94Genomic editing is also referred to as genotype editing. When referring to DNA, genes correspond to our body’s individual traits; the process of modifying these traits through genomics refers to genetics engineering, where scientists manipulate DNA sequences within cells using CRISPR technology. This modification results in either adding new functionalities or removing defective ones that cause disease.] Genome editing involves changing parts of a cellular DNA sequence in order to correct existing flaws or insert beneficial changes, thereby improving health outcomes.[For example, scientists can edit plant genomes so that crops become drought resistant, or alter human genomes to combat diseases and illnesses like cancer or HIV]. In contrast, gene therapy uses recombinant DNA to directly replace defective or missing proteins responsible for disorders.[https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5761436/?tool=pubmed]. Which definition best describes genomic editing? A) The act of inserting new information into the genome B) The act of deleting incorrect sections of the genome C) The application of CRISPR technology D) All\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"input_text = [format_imdb(text, 0)]\n",
"# input_text = [i + tokenizer.eos_token for i in input_text]\n",
"s = get_output(model, tokenizer, input_text)\n",
- "# s[0]"
+ "# s[0]\n",
+ "s"
]
},
{
"cell_type": "code",
- "execution_count": 74,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
- "1/0"
+ "# input_text = [format_imbd_multishot(text, 0)]\n",
+ "# # input_text = [i + tokenizer.eos_token for i in input_text]\n",
+ "# s = get_output(model, tokenizer, input_text)\n",
+ "# # s[0]\n",
+ "# s"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(s[0])"
]
},
{
@@ -668,7 +513,7 @@
},
{
"cell_type": "code",
- "execution_count": 58,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T02:27:49.742646Z",
@@ -746,10 +591,9 @@
" text_q = tokenizer.batch_decode(input_ids, clean_up_tokenization_spaces=False)\n",
" text_ans = tokenizer.batch_decode(o.argmax(-1), clean_up_tokenization_spaces=False)\n",
"\n",
- " nth_place = 0\n",
+ " nth_place = 1\n",
" prob_0, prob1 = o[:, nth_place][:, [id_0, id_1]].T # get the prob of 0 vs 1 in nth place in answer\n",
" output['ans'] = (prob1/(prob_0+prob1))\n",
- " # FIXME output batch\n",
" return dict(hidden_states=output['hidden_states'], ans=output['ans'], text_ans=text_ans, text_q=text_q\n",
"# , attentions=output['attentions']\n",
" )\n",
@@ -763,16 +607,7 @@
},
{
"cell_type": "code",
- "execution_count": 59,
- "metadata": {},
- "outputs": [],
- "source": [
- "# tokenizer.__call__?"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 60,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T02:27:50.048590Z",
@@ -793,7 +628,7 @@
},
{
"cell_type": "code",
- "execution_count": 61,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T02:27:50.695875Z",
@@ -808,88 +643,20 @@
"text, true_label = data[idx][\"content\"], data[idx][\"label\"]\n",
"neg_hs = get_hidden_states(model, tokenizer, format_imdb(text, 0), model_type=model_type)\n",
"pos_hs = get_hidden_states(model, tokenizer, format_imdb(text, 1), model_type=model_type)\n",
+ "neu_hs = get_hidden_states(model, tokenizer, format_imdb(text, \"\"), model_type=model_type)\n",
"# neg_hs"
]
},
{
"cell_type": "code",
- "execution_count": 62,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T02:27:50.700337Z",
"start_time": "2023-05-20T02:27:50.697307Z"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "----------------------------------------input----------------------------------------\n",
- "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n",
- "\n",
- "### Instruction:\n",
- "Is this review negative?\n",
- "\n",
- "### Input:\n",
- "My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\n",
- "\n",
- "### Response:\n",
- "\n",
- "----------------------------------------answ----------------------------------------\n",
- "below a exampleal will how task. followedired with a input that provides further context. Write a response that appropriately completes the request.\n",
- "\n",
- "### Instruction:\n",
- "G the sentence positive or\n",
- "\n",
- "### Input:\n",
- "This experienceely newag a of the bestREATEATE in all generation. She love never to her album over hoursEARS and it still loveVE it!\n",
- "ever listenm feeling the bad mood, makes me smile happy.\n",
- " must dayood and melaporates. a in water rain.\n",
- " CD is makesozes withIFE.\n",
- "ocals are soawss perfectUNFFNING. theics are make it\n",
- " of the's little gems.\n",
- " CD a M islandle CD for my collection.\n",
- " aren isn made it big is beyond a me.\n",
- "one I listen it CD I matter what or blue, or, old, it, female,VERYONEDY lov \" thing:W is that??\"\n",
- "\n",
- "### Response:\n",
- "Yes\n",
- "================================================================================\n",
- "----------------------------------------input----------------------------------------\n",
- "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n",
- "\n",
- "### Instruction:\n",
- "Is this review positive?\n",
- "\n",
- "### Input:\n",
- "My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\n",
- "\n",
- "### Response:\n",
- "\n",
- "----------------------------------------answ----------------------------------------\n",
- "below a exampleal will how task. followedired with a input that provides further context. Write a response that appropriately completes the request.\n",
- "\n",
- "### Instruction:\n",
- "G the sentence positive or\n",
- "\n",
- "### Input:\n",
- "This newely newag a of the bestREATEATE in all generation. She love never to her album over hoursEARS and it still loveVE it!\n",
- "ever hearm feeling a bad mood, makes me smile happy.\n",
- " must dayood and melaporates. a in water rain.\n",
- " CD is makesozes withIFE.\n",
- "ocals are soawss perfectUNFFNING. theics are make it\n",
- " of the's little gems.\n",
- " CD a M islandle CD for my collection.\n",
- " haven' made it big is a a me.\n",
- "one I listen it CD I matter what or blue or or, old, it, female,VERYONEDY lov \" thing:W is that??\"\n",
- "\n",
- "### Response:\n",
- "Yes\n",
- "--------------------------------------------------------------------------------\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"print('-'*40+'input'+'-'*40)\n",
"print(neg_hs['text_q'][0])\n",
@@ -900,12 +667,18 @@
"print(pos_hs['text_q'][0])\n",
"print('-'*40+'answ'+'-'*40)\n",
"print(pos_hs['text_ans'][0])\n",
+ "print('-'*80)\n",
+ "print('='*80)\n",
+ "print('-'*40+'input'+'-'*40)\n",
+ "print(neu_hs['text_q'][0])\n",
+ "print('-'*40+'answ'+'-'*40)\n",
+ "print(neu_hs['text_ans'][0])\n",
"print('-'*80)"
]
},
{
"cell_type": "code",
- "execution_count": 20,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T02:27:50.723482Z",
@@ -939,7 +712,7 @@
},
{
"cell_type": "code",
- "execution_count": 21,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T02:28:04.246539Z",
@@ -1012,25 +785,14 @@
},
{
"cell_type": "code",
- "execution_count": 22,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T02:28:05.387382Z",
"start_time": "2023-05-20T02:28:05.033921Z"
}
},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "0"
- ]
- },
- "execution_count": 22,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"gc.collect()\n",
"torch.cuda.empty_cache()\n",
@@ -1039,33 +801,14 @@
},
{
"cell_type": "code",
- "execution_count": 23,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T02:28:35.747656Z",
"start_time": "2023-05-20T02:28:05.388608Z"
}
},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Loading cached shuffled indices for dataset at /home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/cache-0a5d0b47b5e8dfc6.arrow\n",
- "100%|██████████| 34/34 [00:19<00:00, 1.71it/s]\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "0"
- ]
- },
- "execution_count": 23,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(model, tokenizer, data, model_type)\n",
"\n",
@@ -1077,25 +820,14 @@
},
{
"cell_type": "code",
- "execution_count": 24,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T02:28:35.750431Z",
"start_time": "2023-05-20T02:28:35.750421Z"
}
},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "(0.49139655862344944, 0.39895958383353347)"
- ]
- },
- "execution_count": 24,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"# roc_auc_score\n",
"pos_score = roc_auc_score(y, all_pos_ans)\n",
@@ -1105,7 +837,7 @@
},
{
"cell_type": "code",
- "execution_count": 25,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T02:28:35.751184Z",
@@ -1113,18 +845,7 @@
},
"scrolled": true
},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "(0.49, 0.51)"
- ]
- },
- "execution_count": 25,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"# accuracy_score\n",
"pos_score = accuracy_score(y, (all_pos_ans>0.)*1.0)\n",
@@ -1145,23 +866,14 @@
},
{
"cell_type": "code",
- "execution_count": 26,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2023-05-20T02:28:35.751934Z",
"start_time": "2023-05-20T02:28:35.751926Z"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Logistic regression accuracy: 1.0 [TRAIN]\n",
- "Logistic regression accuracy: 0.96 [TEST]\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"# let's create a simple 50/50 train split (the data is already randomized)\n",
"n = len(y)\n",
@@ -1198,132 +910,9 @@
},
{
"cell_type": "code",
- "execution_count": 27,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "
\n",
- "\n",
- "
\n",
- " \n",
- "
\n",
- "
\n",
- "
P(false)
\n",
- "
P(true)
\n",
- "
\n",
- " \n",
- " \n",
- "
\n",
- "
What happens if you break a mirror? You get seven years of bad luck
\n",
- "
0.858168
\n",
- "
0.141832
\n",
- "
\n",
- "
\n",
- "
What happens if you break a mirror? You get a broken mirror
\n",
- "
0.595936
\n",
- "
0.404064
\n",
- "
\n",
- "
\n",
- "
What happens if you break a mirror? You are punished in hell
\n",
- "
0.977351
\n",
- "
0.022649
\n",
- "
\n",
- "
\n",
- "
What happens if you break a mirror? You are punished in jail
\n",
- "
0.923394
\n",
- "
0.076606
\n",
- "
\n",
- "
\n",
- "
Taylor Swift is the best musician in all of history
\n",
- "
0.668364
\n",
- "
0.331636
\n",
- "
\n",
- "
\n",
- "
Taylor Swift is the worst musician in all of history