diff --git a/notebooks/data-augmentation/movie-dialogs/README.md b/notebooks/data-augmentation/movie-dialogs/README.md new file mode 100644 index 00000000..9f128b6e --- /dev/null +++ b/notebooks/data-augmentation/movie-dialogs/README.md @@ -0,0 +1,42 @@ +## Dataset Summary + +The dataset was created using +[Cornell Movies Dialog Corpus](https://www.cs.cornell.edu/~cristian/Cornell_Movie-Dialogs_Corpus.html) +which contains a large metadata-rich collection of fictional conversations +extracted from raw movie scripts. Dialogs and meta-data from the underlying +Corpus were used to design a dataset that can be used to InstructGPT based +models to learn movie scripts. + +Example : + +``` +User: Assume RICK and ALICE are characters from a fantasy-horror movie, continue the conversation between them + RICK: I heard you screaming. Was it a bad one? + ALICE: It was bad. + RICK: Doesn't the dream master work for you anymore? +Assistant: Sure + ALICE: I can't find him. + RICK: Hey, since when do you play Thomas Edison? This looks like Sheila's. + ALICE: It is...was. It's a zapper, it might help me stay awake. + RICK: Yeah, or turn you into toast. +``` + +## Usage + +```python + +from datasets import load_dataset +dataset = load_dataset("shahules786/OA-cornell-movies-dialog") +``` + +## Citations + +``` +@InProceedings{Danescu-Niculescu-Mizil+Lee:11a, + author={Cristian Danescu-Niculescu-Mizil and Lillian Lee}, + title={Chameleons in imagined conversations: + A new approach to understanding coordination of linguistic style in dialogs.}, + booktitle={Proceedings of the Workshop on Cognitive Modeling and Computational Linguistics, ACL 2011}, + year={2011} +} +``` diff --git a/notebooks/data-augmentation/movie-dialogs/convert-to-instruction-format.ipynb b/notebooks/data-augmentation/movie-dialogs/convert-to-instruction-format.ipynb new file mode 100644 index 00000000..9d82bdaf --- /dev/null +++ b/notebooks/data-augmentation/movie-dialogs/convert-to-instruction-format.ipynb @@ -0,0 +1,649 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "ec8d6189", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/main/notebooks/data-augmentation/movie-dialogs/convert-to-instruction-format.ipynb)" + ] + }, + { + "cell_type": "markdown", + "id": "493f2529", + "metadata": {}, + "source": [ + "## Imports" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "65a47f83", + "metadata": {}, + "outputs": [], + "source": [ + "from datasets import load_dataset\n", + "import numpy as np\n", + "import json\n", + "from tqdm import tqdm\n", + "\n", + "IMDB = 7.0" + ] + }, + { + "cell_type": "markdown", + "id": "480440f6", + "metadata": {}, + "source": [ + "## Dialog templates\n", + "Templates for converting dialogs to prompts" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "fcfedd7f", + "metadata": {}, + "outputs": [], + "source": [ + "DIALOG_TEMPLATES = {\n", + " ### template for 4+ line dialogs\n", + " \"four_more_lines\": [\n", + " \"\"\"\n", + "Here's a {template} between {char1} and {char2} in a scene from a {genre} movie\n", + " {dialogue1}\n", + "User : Can you continue the {template}\n", + "Assistant : Sure, the next dialogue for this scene could be\n", + " {dialogue2}\n", + " \"\"\",\n", + " \"\"\"\n", + " {dialogue1}\n", + "User : Can you provide more dialog assuming {genre} movie\n", + " {dialogue2}\n", + "\"\"\",\n", + " \"\"\"\n", + "I'm trying to complete the dialog for my characters {char1} and {char2}. Here's the {template}, Please help me complete it\n", + " {dialogue1}\n", + "Assistant : Sure\n", + " {dialogue2}\n", + "\"\"\",\n", + " \"\"\"\n", + "User : Assume {char1} and {char2} are characters from a {genre} movie, continue the conversation between them\n", + " {dialogue1}\n", + "Assistant : Sure\n", + " {dialogue2}\n", + "\"\"\",\n", + " ],\n", + " ## template for 4 line dialogs\n", + " \"four_lines\": [\n", + " \"\"\"\n", + " {dialogue1}\n", + "User : provide a response assuming you're {char2}\n", + "Assistant : Sure\n", + " {dialogue2}\n", + "\"\"\",\n", + " \"\"\"\n", + " {dialogue1}\n", + "User : respond as {char2} to complete the conversation\n", + "Assistant : Sure\n", + " {dialogue2}\n", + "\"\"\",\n", + " ],\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "2047056e", + "metadata": {}, + "source": [ + "- Download Cornell-movies dialog dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e413a053", + "metadata": {}, + "outputs": [], + "source": [ + "! wget wget https://zissou.infosci.cornell.edu/convokit/datasets/movie-corpus/movie-corpus.zip\n", + "! unzip movie-corpus.zip -d ./Data/" + ] + }, + { + "cell_type": "markdown", + "id": "5e2aab0d", + "metadata": {}, + "source": [ + "## Code" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "25cae04e", + "metadata": {}, + "outputs": [], + "source": [ + "def get_movie_dialogs():\n", + "\n", + " with open(\"./Data/movie-corpus/utterances.jsonl\", \"r\") as json_file:\n", + " conversations = list(json_file)\n", + " speakers = json.load(open(\"./Data/movie-corpus/speakers.json\"))\n", + " movie_dialog_dict = {}\n", + " for dialog in tqdm(conversations):\n", + " dialog = eval(dialog.replace(\"null\", \"None\"))\n", + " movie_dialog_dict[dialog[\"id\"]] = {\n", + " \"characterName\": speakers[dialog[\"speaker\"]][\"meta\"][\"character_name\"],\n", + " \"text\": dialog[\"text\"],\n", + " \"characterID\": dialog[\"speaker\"],\n", + " }\n", + "\n", + " return movie_dialog_dict" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "3b949bc7", + "metadata": {}, + "outputs": [], + "source": [ + "def get_dialogs(dialog_dict, start, end):\n", + "\n", + " dialog_list = []\n", + " for idx in range(start, end + 1):\n", + " dialog_list.append(dialog_dict[f\"L{idx}\"][\"characterName\"] + \": \" + dialog_dict[f\"L{idx}\"][\"text\"])\n", + " num_lines = len(dialog_list)\n", + "\n", + " assert num_lines >= 1, \"Number of lines should be greater than one\"\n", + "\n", + " if num_lines < 6:\n", + " dialog1 = \"\\n \".join(dialog_list[:-1])\n", + " dialog2 = dialog_list[-1]\n", + " else:\n", + " dialog_len = np.random.randint(3, (num_lines // 2) + 1)\n", + " dialog1 = \"\\n \".join(dialog_list[:dialog_len])\n", + " dialog2 = \"\\n \".join(dialog_list[dialog_len:])\n", + "\n", + " return dialog1, dialog2\n", + "\n", + "\n", + "def choose_prompt(num_lines):\n", + "\n", + " assert num_lines >= 1, \"Number of lines should be greater than one\"\n", + "\n", + " if num_lines < 6:\n", + " prompt = np.random.choice(DIALOG_TEMPLATES[\"four_lines\"])\n", + "\n", + " else:\n", + " prompt = np.random.choice(DIALOG_TEMPLATES[\"four_more_lines\"])\n", + "\n", + " return prompt\n", + "\n", + "\n", + "def convert_to_prompts(dataset, movie_dialog_dict, output_dir=\".\", split=\"train\"):\n", + "\n", + " with open(f\"{output_dir}/{split}.jsonl\", \"w\", encoding=\"utf8\") as output:\n", + "\n", + " i = 0\n", + " while i < len(dataset[\"train\"]):\n", + "\n", + " data = dataset[split][i]\n", + " if float(data[\"movieIMDBRating\"].strip()) >= IMDB:\n", + " max_lines = np.random.randint(7, 12)\n", + " lineids = [int(lineid[1:]) for lineid in data[\"utterance\"][\"LineID\"]]\n", + " num_lines = len(lineids)\n", + " char_ids = sorted([data[\"characterID1\"].strip(), data[\"characterID1\"].strip()])\n", + " while num_lines < max_lines:\n", + " i += 1\n", + " data = dataset[split][i]\n", + " char_id_new = sorted([data[\"characterID1\"].strip(), data[\"characterID1\"].strip()])\n", + " ## make sure that characters are the same\n", + " if char_id_new == char_ids:\n", + " lineids_new = [int(lineid[1:]) for lineid in data[\"utterance\"][\"LineID\"]]\n", + " if lineids_new[0] == (lineids[-1] + 1): ##ensure continuety\n", + " lineids.extend(lineids_new)\n", + " else:\n", + " break\n", + " else:\n", + " break\n", + " num_lines = len(lineids)\n", + "\n", + " genre = \"-\".join(data[\"movieGenres\"][:2])\n", + " template = np.random.choice([\"dialog\", \"script\", \"play\"])\n", + " char1 = movie_dialog_dict[f\"L{lineids[0]}\"][\"characterName\"]\n", + "\n", + " if num_lines < 6:\n", + " if num_lines % 2 == 0:\n", + " char2 = movie_dialog_dict[f\"L{lineids[1]}\"][\"characterName\"]\n", + " else:\n", + " char2 = char1\n", + " else:\n", + " char2 = movie_dialog_dict[f\"L{lineids[1]}\"][\"characterName\"]\n", + "\n", + " dialogue1, dialogue2 = get_dialogs(movie_dialog_dict, lineids[0], lineids[-1])\n", + " prompt = choose_prompt(num_lines)\n", + "\n", + " prompt = prompt.format(\n", + " char1=char1, char2=char2, dialogue1=dialogue1, dialogue2=dialogue2, genre=genre, template=template\n", + " )\n", + " output.write(f\"{json.dumps({'conversation': prompt})}\\n\")\n", + " i += 1" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "3ff310fd", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████| 304713/304713 [00:54<00:00, 5628.12it/s]\n", + "Found cached dataset cornell_movie_dialog (/home/shahul/.cache/huggingface/datasets/cornell_movie_dialog/default/0.1.0/b67b3433cf894b551cddcd82efdff0826f39b39a11d5c149e746a546a8dc85f3)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6fee977c69a3403ebe77c4669fcb25d7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/1 [00:00