From 42b016cae52c70f7e338f02a637963bf9885b7a2 Mon Sep 17 00:00:00 2001
From: Fraser <tomatoesneverlie@gmail.com>
Date: Fri, 31 Mar 2023 21:02:53 -0400
Subject: [PATCH] move dataset load to build-time

---
 api/dataset_dl.py | 21 +++++++++++++++++++++
 api/main.py       | 23 +++--------------------
 api/nixpacks.toml |  5 +++++
 3 files changed, 29 insertions(+), 20 deletions(-)
 create mode 100644 api/dataset_dl.py
 create mode 100644 api/nixpacks.toml

diff --git a/api/dataset_dl.py b/api/dataset_dl.py
new file mode 100644
index 0000000..a1ccd9c
--- /dev/null
+++ b/api/dataset_dl.py
@@ -0,0 +1,21 @@
+import pickle
+import requests
+import os
+
+print('Downloading dataset...')
+
+url = os.environ.get('DATASET_URL')
+if url is None:
+    print('No dataset url provided.')
+    exit()
+
+dataset_dict_bytes = requests.get(url).content
+
+print('Unpacking dataset...')
+dataset_dict = pickle.loads(dataset_dict_bytes)
+
+print('Writing dataset to disk...')
+with open('dataset.pkl', 'wb') as f:
+    pickle.dump(dataset_dict, f)
+
+print('Done!')
diff --git a/api/main.py b/api/main.py
index 54221f4..bdac5fe 100644
--- a/api/main.py
+++ b/api/main.py
@@ -6,7 +6,6 @@ import dataclasses
 import os
 import openai
 import pickle
-import requests
 
 # ---------------------------------- env setup ---------------------------------
 
@@ -20,25 +19,9 @@ openai.api_key = OPENAI_API_KEY
 # -------------------------------- load dataset --------------------------------
 
 
-if os.path.exists('dataset.pkl'):
-    print('Found dataset.pkl')
-    print('Loading dataset...')
-    with open('dataset.pkl', 'rb') as f:
-        dataset_dict = pickle.load(f)
-
-else:
-    print('No dataset.pkl found on disk.')
-    print('Downloading dataset...')
-
-    url = os.environ.get('DATASET_URL')
-    if url is None:
-        print('No dataset url provided.')
-        exit()
-
-    dataset_dict_bytes = requests.get(url).content
-    print('Unpacking dataset...')
-    dataset_dict = pickle.loads(dataset_dict_bytes)
-
+print('Loading dataset...')
+with open('dataset.pkl', 'rb') as f:
+    dataset_dict = pickle.load(f)
 print('Done!')
 
 
diff --git a/api/nixpacks.toml b/api/nixpacks.toml
new file mode 100644
index 0000000..f03f2c2
--- /dev/null
+++ b/api/nixpacks.toml
@@ -0,0 +1,5 @@
+[phases.build]
+cmds = ['python3 dataset_dl.py']
+
+[start]
+cmd = 'gunicorn --timeout 300 "main:app"'