diff --git a/zipline/data/loader_utils.py b/zipline/data/loader_utils.py deleted file mode 100644 index b21c144c..00000000 --- a/zipline/data/loader_utils.py +++ /dev/null @@ -1,136 +0,0 @@ -# -# Copyright 2012 Quantopian, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -""" -Various utilites used by different date loaders. - -Could stand to be broken up more into components. -e.g. the mapping utilities. - -""" - -import datetime - -import pytz - -from collections import namedtuple - -from functools import partial - -from six import iteritems - - -def get_utc_from_exchange_time(naive): - local = pytz.timezone('US/Eastern') - local_dt = naive.replace(tzinfo=local) - utc_dt = local_dt.astimezone(pytz.utc) - return utc_dt - - -def guarded_conversion(conversion, str_val): - """ - Returns the result of applying the @conversion to @str_val - """ - if str_val in (None, ""): - return None - return conversion(str_val) - - -def date_conversion(date_str, date_pattern='%m/%d/%Y', to_utc=True): - """ - Convert date strings from TickData (or other source) into epoch values. - - Specify to_utc=False if the input date is already UTC (or is naive). - """ - dt = datetime.datetime.strptime(date_str, date_pattern) - if to_utc: - dt = get_utc_from_exchange_time(dt) - else: - dt = dt.replace(tzinfo=pytz.utc) - return dt - - -# Mapping is a structure for how want to convert the source data into -# the form we insert into the database. -# - conversion, a function used to convert source input to our target value -# - source, the key(s) in the original source to pass to the conversion -# method -# If a single string, then it's a direct lookup into the -# source row by that key -# If an iterator, pass the source to as a list of keys, -# in order, to the conversion function. -# If empty, then the conversion method provides a 'default' value. -Mapping = namedtuple('Mapping', ['conversion', 'source']) - - -def apply_mapping(mapping, row): - """ - Returns the value of a @mapping for a given @row. - - i.e. the @mapping.source values are extracted from @row and fed - into the @mapping.conversion method. - """ - if isinstance(mapping.source, str): - # Do a 'direct' conversion of one key from the source row. - return guarded_conversion(mapping.conversion, row[mapping.source]) - if mapping.source is None: - # For hardcoded values. - # conversion method will return a constant value - return mapping.conversion() - else: - # Assume we are using multiple source values. - # Feed the source values in order prescribed by mapping.source - # to mapping.conversion. - return mapping.conversion(*[row[source] for source in mapping.source]) - - -def _row_cb(mapping, row): - """ - Returns the dict created from our @mapping of the source @row. - - Not intended to be used directly, but rather to be the base of another - function that supplies the mapping value. - """ - return { - target: apply_mapping(mapping, row) - for target, mapping - in iteritems(mapping) - } - - -def make_row_cb(mapping): - """ - Returns a func that can be applied to a dict that returns the - application of the @mapping, which results in a dict. - """ - return partial(_row_cb, mapping) - - -def source_to_records(mappings, - source, - source_wrapper=None, - records_wrapper=None): - if source_wrapper: - source = source_wrapper(source) - - callback = make_row_cb(mappings) - - records = (callback(row) for row in source) - - if records_wrapper: - records = records_wrapper(records) - - return records