Mass refactoring

notebooks/dataset_analysis/AnalyzeDataset.ipynb

@@ -0,0 +1,352 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "TTS_PATH = \"/home/erogol/projects/\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "sys.path.append(TTS_PATH) # set this if TTS is not installed globally\n",
+    "import glob\n",
+    "import librosa\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from scipy.stats import norm\n",
+    "from tqdm import tqdm_notebook as tqdm\n",
+    "from multiprocessing import Pool\n",
+    "from matplotlib import pylab as plt\n",
+    "from collections import Counter\n",
+    "from TTS.tts.datasets.preprocess import *\n",
+    "%matplotlib inline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "DATA_PATH = \"/home/erogol/Data/m-ai-labs/de_DE/by_book/male/karlsson/\"\n",
+    "META_DATA = [\"kleinzaches/metadata.csv\",\n",
+    "            \"spiegel_kaetzchen/metadata.csv\",\n",
+    "            \"herrnarnesschatz/metadata.csv\",\n",
+    "            \"maedchen_von_moorhof/metadata.csv\",\n",
+    "            \"koenigsgaukler/metadata.csv\",\n",
+    "            \"altehous/metadata.csv\",\n",
+    "            \"odysseus/metadata.csv\",\n",
+    "            \"undine/metadata.csv\",\n",
+    "            \"reise_tilsit/metadata.csv\",\n",
+    "            \"schmied_seines_glueckes/metadata.csv\",\n",
+    "            \"kammmacher/metadata.csv\",\n",
+    "            \"unterm_birnbaum/metadata.csv\",\n",
+    "            \"liebesbriefe/metadata.csv\",\n",
+    "            \"sandmann/metadata.csv\"]\n",
+    "NUM_PROC = 8"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# use your own preprocessor at this stage - TTS/datasets/proprocess.py\n",
+    "items = mailabs(DATA_PATH, META_DATA)\n",
+    "print(\" > Number of audio files: {}\".format(len(items)))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# check wavs if exist\n",
+    "wav_files = []\n",
+    "for item in items:\n",
+    "    wav_file = item[1].strip()\n",
+    "    wav_files.append(wav_file)\n",
+    "    if not os.path.exists(wav_file):\n",
+    "        print(waf_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# show duplicate items\n",
+    "c = Counter(wav_files)\n",
+    "print([item for item, count in c.items() if count > 1])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def load_item(item):\n",
+    "    file_name = item[1].strip()\n",
+    "    text = item[0].strip()\n",
+    "    audio = librosa.load(file_name, sr=None)\n",
+    "    sr = audio[1]\n",
+    "    audio = audio[0]\n",
+    "    audio_len = len(audio) / sr\n",
+    "    text_len = len(text)\n",
+    "    return file_name, text, text_len, audio, audio_len\n",
+    "\n",
+    "# This will take a while depending on size of dataset\n",
+    "if NUM_PROC == 1:\n",
+    "    data = []\n",
+    "    for m in tqdm(items):\n",
+    "        data += [load_item(m)]\n",
+    "else:\n",
+    "    with Pool(8) as p:\n",
+    "        data = list(tqdm(p.imap(load_item, items), total=len(items)))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# count words in the dataset\n",
+    "w_count = Counter()\n",
+    "for item in tqdm(data):\n",
+    "    text = item[1].lower().strip()\n",
+    "    for word in text.split():\n",
+    "        w_count[word] += 1\n",
+    "print(\" > Number of words: {}\".format(len(w_count)))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text_vs_durs = {}  # text length vs audio duration\n",
+    "text_len_counter = Counter()  # number of sentences with the keyed length\n",
+    "for item in tqdm(data):\n",
+    "    text = item[1].lower().strip()\n",
+    "    text_len = len(text)\n",
+    "    text_len_counter[text_len] += 1\n",
+    "    audio_len = item[-1]\n",
+    "    try:\n",
+    "        text_vs_durs[text_len] += [audio_len]\n",
+    "    except:\n",
+    "        text_vs_durs[text_len] = [audio_len]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# text_len vs avg_audio_len, median_audio_len, std_audio_len\n",
+    "text_vs_avg = {}\n",
+    "text_vs_median = {}\n",
+    "text_vs_std = {}\n",
+    "for key, durs in text_vs_durs.items():\n",
+    "    text_vs_avg[key] = np.mean(durs)\n",
+    "    text_vs_median[key] = np.median(durs)\n",
+    "    text_vs_std[key] = np.std(durs)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Avg audio length per char"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for item in data:\n",
+    "    if item[-1] < 2:\n",
+    "        print(item)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sec_per_chars = []\n",
+    "for item in data:\n",
+    "    text = item[1]\n",
+    "    dur = item[-1]\n",
+    "    sec_per_char = dur / len(text)\n",
+    "    sec_per_chars.append(sec_per_char)\n",
+    "# sec_per_char /= len(data)\n",
+    "# print(sec_per_char)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mean = np.mean(sec_per_chars)\n",
+    "std = np.std(sec_per_chars)\n",
+    "print(mean)\n",
+    "print(std)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dist = norm(mean, std)\n",
+    "\n",
+    "# find irregular instances long or short voice durations\n",
+    "for item in data:\n",
+    "    text = item[1]\n",
+    "    dur = item[-1]\n",
+    "    sec_per_char = dur / len(text)\n",
+    "    pdf =norm.pdf(sec_per_char)\n",
+    "    if pdf < 0.39:\n",
+    "        print(item)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Plot Dataset Statistics"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "plt.title(\"text length vs mean audio duration\")\n",
+    "plt.scatter(list(text_vs_avg.keys()), list(text_vs_avg.values()))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "plt.title(\"text length vs median audio duration\")\n",
+    "plt.scatter(list(text_vs_median.keys()), list(text_vs_median.values()))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "plt.title(\"text length vs STD\")\n",
+    "plt.scatter(list(text_vs_std.keys()), list(text_vs_std.values()))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "plt.title(\"text length vs # instances\")\n",
+    "plt.scatter(list(text_len_counter.keys()), list(text_len_counter.values()))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Check words frequencies"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "w_count_df = pd.DataFrame.from_dict(w_count, orient='index')\n",
+    "w_count_df.sort_values(0, ascending=False, inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "w_count_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# check a certain word\n",
+    "w_count_df.at['minute', 0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# fequency bar plot - it takes time!!\n",
+    "w_count_df.plot.bar()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/dataset_analysis/CheckDatasetSNR.ipynb

@@ -0,0 +1,239 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This notebook computes the average SNR a given Voice Dataset. If the SNR is too low, that might reduce the performance or prevent model to learn.\n",
+    "\n",
+    "To use this notebook, you need:\n",
+    "- WADA SNR estimation: http://www.cs.cmu.edu/~robust/archive/algorithms/WADA_SNR_IS_2008/\n",
+    "    1. extract in the same folder as this notebook\n",
+    "    2. under MacOS you'll have to rebuild the executable. In the build folder: 1) remove existing .o files and 2) run make\n",
+    "\n",
+    "\n",
+    "- FFMPEG: ```sudo apt-get install ffmpeg ```     \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true,
+    "jupyter": {
+     "outputs_hidden": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import os, sys\n",
+    "import glob\n",
+    "import subprocess\n",
+    "import tempfile\n",
+    "import IPython\n",
+    "import soundfile as sf\n",
+    "import numpy as np\n",
+    "from tqdm import tqdm\n",
+    "from multiprocessing import Pool\n",
+    "from matplotlib import pylab as plt\n",
+    "%matplotlib inline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true,
+    "jupyter": {
+     "outputs_hidden": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Set the meta parameters\n",
+    "DATA_PATH = \"/home/erogol/Data/m-ai-labs/de_DE/by_book/female/eva_k/\"\n",
+    "NUM_PROC = 1\n",
+    "CURRENT_PATH = os.getcwd()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true,
+    "jupyter": {
+     "outputs_hidden": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def compute_file_snr(file_path):\n",
+    "    \"\"\" Convert given file to required format with FFMPEG and process with WADA.\"\"\"\n",
+    "    _, sr = sf.read(file_path)\n",
+    "    new_file = file_path.replace(\".wav\", \"_tmp.wav\")\n",
+    "    if sr != 16000:\n",
+    "        command = f'ffmpeg -i \"{file_path}\" -ac 1 -acodec pcm_s16le -y -ar 16000 \"{new_file}\"'\n",
+    "    else:\n",
+    "        command = f'cp \"{file_path}\" \"{new_file}\"'\n",
+    "    os.system(command)\n",
+    "    command = [f'\"{CURRENT_PATH}/WadaSNR/Exe/WADASNR\"', f'-i \"{new_file}\"', f'-t \"{CURRENT_PATH}/WadaSNR/Exe/Alpha0.400000.txt\"', '-ifmt mswav']\n",
+    "    output = subprocess.check_output(\" \".join(command), shell=True)\n",
+    "    try:\n",
+    "        output = float(output.split()[-3].decode(\"utf-8\"))\n",
+    "    except:\n",
+    "        raise RuntimeError(\" \".join(command))\n",
+    "    os.system(f'rm \"{new_file}\"')\n",
+    "    return output, file_path\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true,
+    "jupyter": {
+     "outputs_hidden": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "wav_file = \"/home/erogol/Data/LJSpeech-1.1/wavs/LJ001-0001.wav\"\n",
+    "output = compute_file_snr(wav_file)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wav_files = glob.glob(f\"{DATA_PATH}/**/*.wav\", recursive=True)\n",
+    "print(f\" > Number of wav files {len(wav_files)}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if NUM_PROC == 1:\n",
+    "    file_snrs = [None] * len(wav_files) \n",
+    "    for idx, wav_file in tqdm(enumerate(wav_files)):\n",
+    "        tup = compute_file_snr(wav_file)\n",
+    "        file_snrs[idx] = tup\n",
+    "else:\n",
+    "    with Pool(NUM_PROC) as pool:\n",
+    "        file_snrs = list(tqdm(pool.imap(compute_file_snr, wav_files), total=len(wav_files)))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "snrs = [tup[0] for tup in file_snrs]\n",
+    "\n",
+    "error_idxs = np.where(np.isnan(snrs) == True)[0]\n",
+    "error_files = [file_names[idx] for idx in error_idxs]\n",
+    "\n",
+    "file_snrs = [i for j, i in enumerate(file_snrs) if j not in error_idxs]\n",
+    "file_names = [tup[1] for tup in file_snrs]\n",
+    "snrs = [tup[0] for tup in file_snrs]\n",
+    "file_idxs = np.argsort(snrs)\n",
+    "\n",
+    "\n",
+    "print(f\" > Average SNR of the dataset:{np.mean(snrs)}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true,
+    "jupyter": {
+     "outputs_hidden": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def output_snr_with_audio(idx):\n",
+    "    file_idx = file_idxs[idx]\n",
+    "    file_name = file_names[file_idx]\n",
+    "    wav, sr = sf.read(file_name)\n",
+    "    # multi channel to single channel\n",
+    "    if len(wav.shape) == 2:\n",
+    "        wav = wav[:, 0]\n",
+    "    print(f\" > {file_name} - snr:{snrs[file_idx]}\")\n",
+    "    IPython.display.display(IPython.display.Audio(wav, rate=sr))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# find worse SNR files\n",
+    "N = 10  # number of files to fetch\n",
+    "for i in range(N):\n",
+    "    output_snr_with_audio(i)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# find best recordings\n",
+    "N = 10  # number of files to fetch\n",
+    "for i in range(N):\n",
+    "    output_snr_with_audio(-i-1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "plt.hist(snrs, bins=100)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true,
+    "jupyter": {
+     "outputs_hidden": true
+    }
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

notebooks/dataset_analysis/README.md

@@ -0,0 +1,7 @@
+## Simple Notebook to Analyze a Dataset
+
+By the use of this notebook, you can easily analyze a brand new dataset, find exceptional cases and define your training set.
+
+What we are looking in here is reasonable distribution of instances in terms of sequence-length, audio-length and word-coverage. 
+
+This notebook is inspired from https://github.com/MycroftAI/mimic2

notebooks/dataset_analysis/analyze.py

@@ -0,0 +1,215 @@
+# visualisation tools for mimic2
+import matplotlib.pyplot as plt
+from statistics import stdev, mode, mean, median
+from statistics import StatisticsError
+import argparse
+import os
+import csv
+import seaborn as sns
+import random
+from text.cmudict import CMUDict
+
+def get_audio_seconds(frames):
+    return (frames*12.5)/1000
+
+
+def append_data_statistics(meta_data):
+    # get data statistics
+    for char_cnt in meta_data:
+        data = meta_data[char_cnt]["data"]
+        audio_len_list = [d["audio_len"] for d in data]
+        mean_audio_len = mean(audio_len_list)
+        try:
+            mode_audio_list = [round(d["audio_len"], 2) for d in data]
+            mode_audio_len = mode(mode_audio_list)
+        except StatisticsError:
+            mode_audio_len = audio_len_list[0]
+        median_audio_len = median(audio_len_list)
+
+        try:
+            std = stdev(
+                d["audio_len"] for d in data
+            )
+        except StatisticsError:
+            std = 0
+
+        meta_data[char_cnt]["mean"] = mean_audio_len
+        meta_data[char_cnt]["median"] = median_audio_len
+        meta_data[char_cnt]["mode"] = mode_audio_len
+        meta_data[char_cnt]["std"] = std
+    return meta_data
+
+
+def process_meta_data(path):
+    meta_data = {}
+
+    # load meta data
+    with open(path, 'r') as f:
+        data = csv.reader(f, delimiter='|')
+        for row in data:
+            frames = int(row[2])
+            utt = row[3]
+            audio_len = get_audio_seconds(frames)
+            char_count = len(utt)
+            if not meta_data.get(char_count):
+                meta_data[char_count] = {
+                    "data": []
+                }
+
+            meta_data[char_count]["data"].append(
+                {
+                    "utt": utt,
+                    "frames": frames,
+                    "audio_len": audio_len,
+                    "row": "{}|{}|{}|{}".format(row[0], row[1], row[2], row[3])
+                }
+            )
+
+    meta_data = append_data_statistics(meta_data)
+
+    return meta_data
+
+
+def get_data_points(meta_data):
+    x = [char_cnt for char_cnt in meta_data]
+    y_avg = [meta_data[d]['mean'] for d in meta_data]
+    y_mode = [meta_data[d]['mode'] for d in meta_data]
+    y_median = [meta_data[d]['median'] for d in meta_data]
+    y_std = [meta_data[d]['std'] for d in meta_data]
+    y_num_samples = [len(meta_data[d]['data']) for d in meta_data]
+    return {
+        "x": x,
+        "y_avg": y_avg,
+        "y_mode": y_mode,
+        "y_median": y_median,
+        "y_std": y_std,
+        "y_num_samples": y_num_samples
+    }
+
+
+def save_training(file_path, meta_data):
+    rows = []
+    for char_cnt in meta_data:
+        data = meta_data[char_cnt]['data']
+        for d in data:
+            rows.append(d['row'] + "\n")
+
+    random.shuffle(rows)
+    with open(file_path, 'w+') as f:
+        for row in rows:
+            f.write(row)
+
+
+def plot(meta_data, save_path=None):
+    save = False
+    if save_path:
+        save = True
+
+    graph_data = get_data_points(meta_data)
+    x = graph_data['x']
+    y_avg = graph_data['y_avg']
+    y_std = graph_data['y_std']
+    y_mode = graph_data['y_mode']
+    y_median = graph_data['y_median']
+    y_num_samples = graph_data['y_num_samples']
+
+    plt.figure()
+    plt.plot(x, y_avg, 'ro')
+    plt.xlabel("character lengths", fontsize=30)
+    plt.ylabel("avg seconds", fontsize=30)
+    if save:
+        name = "char_len_vs_avg_secs"
+        plt.savefig(os.path.join(save_path, name))
+
+    plt.figure()
+    plt.plot(x, y_mode, 'ro')
+    plt.xlabel("character lengths", fontsize=30)
+    plt.ylabel("mode seconds", fontsize=30)
+    if save:
+        name = "char_len_vs_mode_secs"
+        plt.savefig(os.path.join(save_path, name))
+
+    plt.figure()
+    plt.plot(x, y_median, 'ro')
+    plt.xlabel("character lengths", fontsize=30)
+    plt.ylabel("median seconds", fontsize=30)
+    if save:
+        name = "char_len_vs_med_secs"
+        plt.savefig(os.path.join(save_path, name))
+
+    plt.figure()
+    plt.plot(x, y_std, 'ro')
+    plt.xlabel("character lengths", fontsize=30)
+    plt.ylabel("standard deviation", fontsize=30)
+    if save:
+        name = "char_len_vs_std"
+        plt.savefig(os.path.join(save_path, name))
+
+    plt.figure()
+    plt.plot(x, y_num_samples, 'ro')
+    plt.xlabel("character lengths", fontsize=30)
+    plt.ylabel("number of samples", fontsize=30)
+    if save:
+        name = "char_len_vs_num_samples"
+        plt.savefig(os.path.join(save_path, name))
+
+
+def plot_phonemes(train_path, cmu_dict_path, save_path):
+    cmudict = CMUDict(cmu_dict_path)
+
+    phonemes = {}
+
+    with open(train_path, 'r') as f:
+        data = csv.reader(f, delimiter='|')
+        phonemes["None"] = 0
+        for row in data:
+            words = row[3].split()
+            for word in words:
+                pho = cmudict.lookup(word)
+                if pho:
+                    indie = pho[0].split()
+                    for nemes in indie:
+                        if phonemes.get(nemes):
+                            phonemes[nemes] += 1
+                        else:
+                            phonemes[nemes] = 1
+                else:
+                    phonemes["None"] += 1
+
+    x, y = [], []
+    for key in phonemes:
+        x.append(key)
+        y.append(phonemes[key])
+
+    plt.figure()
+    plt.rcParams["figure.figsize"] = (50, 20)
+    barplot = sns.barplot(x, y)
+    if save_path:
+        fig = barplot.get_figure()
+        fig.savefig(os.path.join(save_path, "phoneme_dist"))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--train_file_path', required=True,
+        help='this is the path to the train.txt file that the preprocess.py script creates'
+    )
+    parser.add_argument(
+        '--save_to', help='path to save charts of data to'
+    )
+    parser.add_argument(
+        '--cmu_dict_path', help='give cmudict-0.7b to see phoneme distribution'
+    )
+    args = parser.parse_args()
+    meta_data = process_meta_data(args.train_file_path)
+    plt.rcParams["figure.figsize"] = (10, 5)
+    plot(meta_data, save_path=args.save_to)
+    if args.cmu_dict_path:
+        plt.rcParams["figure.figsize"] = (30, 10)
+        plot_phonemes(args.train_file_path, args.cmu_dict_path, args.save_to)
+
+    plt.show()
+
+if __name__ == '__main__':
+    main()