BUG: fixed issue with incremental ingestion

catalyst/marketplace/utils/bundle_utils.py

@@ -2,6 +2,7 @@ import os
 import shutil
 
 import bcolz
+import pandas as pd
 
 
 def merge_bundles(zsource, ztarget):
@@ -18,14 +19,13 @@ def merge_bundles(zsource, ztarget):
     """
     # TODO: find a way to do this iteratively instead of in-memory
     df_source = zsource.todataframe()
-    df_source.set_index('date', drop=False, inplace=True)
     df_target = ztarget.todataframe()
-    df_target.set_index('date', drop=False, inplace=True)
 
-    df = df_target.merge(
-        right=df_source,
-        how='right',
+    df = pd.concat(
+        [df_source, df_target], ignore_index=True
     )  # type: pd.DataFrame
+    df.drop_duplicates(inplace=True)
+    df.set_index(['date', 'symbol'], drop=False, inplace=True)
 
     dirname = os.path.basename(ztarget.rootdir)
     bak_dir = ztarget.rootdir.replace(dirname, '.{}'.format(dirname))

tests/marketplace/test_marketplace.py

@@ -21,7 +21,7 @@ class TestMarketplace(WithLogger, ZiplineTestCase):
 
     def test_ingest(self):
         marketplace = Marketplace()
-        ds_def = marketplace.ingest('marketcap1234')
+        ds_def = marketplace.ingest('github')
         pass
 
     def test_publish(self):