From 356058ed93672e37a4f2959e9b146bd7db58c7ae Mon Sep 17 00:00:00 2001 From: Taylor Date: Mon, 30 Jan 2023 11:23:12 +0000 Subject: [PATCH] Add Hippocorpus dataset script (#750) * Add Hippocorpus dataset script --- .../data-augmentation/hippocorpus/README.md | 7 + .../hippocorpus/hippocorpus.ipynb | 197 ++++++++++++++++++ 2 files changed, 204 insertions(+) create mode 100644 notebooks/data-augmentation/hippocorpus/README.md create mode 100644 notebooks/data-augmentation/hippocorpus/hippocorpus.ipynb diff --git a/notebooks/data-augmentation/hippocorpus/README.md b/notebooks/data-augmentation/hippocorpus/README.md new file mode 100644 index 00000000..8eff1de0 --- /dev/null +++ b/notebooks/data-augmentation/hippocorpus/README.md @@ -0,0 +1,7 @@ +# Hippocorpus Converter + +This notebook takes an existing copy of the +[Hippocorpus](https://huggingface.co/datasets/hippocorpus) and augments it for +training OpenAssistant. **This notebook is currently unique among its peers in +the same folder as it requires an existing copy of the Hippocorpus. See the +above HuggingFace link for a download link.** diff --git a/notebooks/data-augmentation/hippocorpus/hippocorpus.ipynb b/notebooks/data-augmentation/hippocorpus/hippocorpus.ipynb new file mode 100644 index 00000000..d4cf6eba --- /dev/null +++ b/notebooks/data-augmentation/hippocorpus/hippocorpus.ipynb @@ -0,0 +1,197 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f80b8618-abf6-4763-89d9-20b831c4ea98", + "metadata": { + "tags": [] + }, + "source": [ + "# Hippocorpus converter" + ] + }, + { + "cell_type": "markdown", + "id": "b545caa0-0a32-4007-8f17-8fbdc2f1dd37", + "metadata": { + "tags": [] + }, + "source": [ + "## Import" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aefa8aac-4ab9-4b5a-b3e0-c65baa8da873", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "hippocorpus = pd.read_csv(\"hippocorpus/hcV3-stories.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ebf19352-7c90-4bdf-bc90-5e328e64161d", + "metadata": {}, + "outputs": [], + "source": [ + "# There is a surprising number of people who seem to have left capslock on while participating in the data collection process.\n", + "# These entries tend to be of lower than average quality and would be impossible to fully restore without more complex methods, so they are excluded\n", + "hippocorpus = hippocorpus[~hippocorpus[\"mainEvent\"].str.isupper()]" + ] + }, + { + "cell_type": "markdown", + "id": "968bbf1e-78b3-4436-8a3f-9335b4d2801a", + "metadata": {}, + "source": [ + "## Convert" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec73fd44-b209-4bea-a1c9-711251747647", + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "from random import choice, random, randrange\n", + "import nltk\n", + "from nltk.tokenize import sent_tokenize\n", + "\n", + "nltk.download(\"punkt\")\n", + "\n", + "\n", + "def replace_my(string):\n", + " match = re.search(r\"my (\\w+)\", string)\n", + " if match:\n", + " word = match.group(1)\n", + " if word[0] in \"aeiou\":\n", + " string = re.sub(r\"my\", \"an\", string, 1)\n", + " else:\n", + " string = re.sub(r\"my\", \"a\", string, 1)\n", + " return string\n", + "\n", + "\n", + "def sure():\n", + " ack = choice([\"Sure\", \"Of course\", \"Alright\", \"Certainly\"])\n", + " punctuation = choice([\",\", \"!\", \".\"])\n", + " return ack + punctuation\n", + "\n", + "\n", + "def convert_row(row):\n", + " interaction = \"\"\n", + " main_event = row[\"mainEvent\"].rstrip(\"!.?;:\")\n", + " main_event = main_event[0].lower() + main_event[1:]\n", + " main_event = replace_my(main_event)\n", + " an_original = choice([\"a\", \"an original\"])\n", + " write = choice([\"Write\", \"Write me\", \"Please write\"])\n", + " instruction = f\"{write} {an_original} story about {main_event}.\"\n", + " interaction += f\"User: {instruction}\"\n", + "\n", + " story = row[\"story\"]\n", + " do_sentence_instruction = random() > 0.5\n", + " if do_sentence_instruction:\n", + " sentences = sent_tokenize(story)\n", + " sentence_index = randrange(len(sentences))\n", + " if sentence_index == 0:\n", + " interaction += \" Make the first sentence \"\n", + " sentence_response_section = f\" where the first sentence is \"\n", + " elif sentence_index == len(sentences) - 1:\n", + " interaction += \" Make the last sentence \"\n", + " sentence_response_section = f\" where the last sentence is \"\n", + " else:\n", + " interaction += \" Include the sentence \"\n", + " sentence_response_section = f\" which includes the sentence \"\n", + " interaction += f'\"{sentences[sentence_index]}\"'\n", + " sentence_response_section += f'\"{sentences[sentence_index]}\"'\n", + " else:\n", + " sentence_response_section = \"\"\n", + " interaction += \"\\n\\n\"\n", + "\n", + " interaction += f\"Rosey: {sure()} Here's a story about {main_event}{sentence_response_section}.\\n\\n{story}\"\n", + " interaction += \"\\n\\n\"\n", + "\n", + " def most_surprising(interaction):\n", + " most_surprising = row[\"mostSurprising\"]\n", + " most_surprising = most_surprising[0].lower() + most_surprising[1:]\n", + " was = choice([\"was\", \"do you think was\", \"would you say was\", \"do you think someone would say was\"])\n", + " surprising = choice(\n", + " [\"the most surprising thing\", \"one of the most surprising things\", \"a surprising development\"]\n", + " )\n", + " interaction += f\"User: What {was} {surprising} in that story?\\n\\n\"\n", + " id_say = choice([\"I'd say the\", \"I would have to say the\", \"The\", \"This story's\"])\n", + " interaction += f\"Rosey: {id_say} most surprising development was {most_surprising}.\"\n", + " return interaction\n", + "\n", + " def summarize(interaction):\n", + " preamble = choice(\n", + " [\"The story is a little long. \", \"This is longer than I was expecting. \", \"It needs to be shorter. \", \"\"]\n", + " )\n", + " verb = choice(\n", + " [\"shorten it to a sentence or two\", \"summarize it\", \"shrink it way down\", \"make it way more terse\"]\n", + " )\n", + " request = choice([\"Can you \", \"I need you to \", \"Please \"])\n", + " interaction += f\"User: {preamble}{request}{verb}.\\n\\n\"\n", + " interaction += f\"Rosey: {sure()} Here's a summary of the story:\\n\\n{row['summary']}\"\n", + " return interaction\n", + "\n", + " (first, second) = (most_surprising, summarize)\n", + " if random() > 0.5:\n", + " (first, second) = (second, first)\n", + " interaction = first(interaction)\n", + " interaction += \"\\n\\n\"\n", + " interaction = second(interaction)\n", + "\n", + " return interaction\n", + "\n", + "\n", + "hippocorpus = hippocorpus.apply(convert_row, axis=1)" + ] + }, + { + "cell_type": "markdown", + "id": "d6fadffd-f0b0-44f9-abf5-41fad3c26738", + "metadata": {}, + "source": [ + "## Export" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7b0d047d-8cb4-4621-80bc-630545c1c309", + "metadata": {}, + "outputs": [], + "source": [ + "hippocorpus.to_csv(\"hippocorpus.csv\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Open-Assistant", + "language": "python", + "name": "open-assistant" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}