mirror of
https://github.com/wassname/catalyst.git
synced 2026-07-23 12:50:22 +08:00
Eddie Hebert
a203f69635
Adding a copy of the Event's dt field as datetime via the `alias_dt` generator, so that the API was forgiving and allowed both datetime and dt on a SIDData object, was creating noticeable overhead, even on an noop algorithms. Instead of incurring the cost of copying the datetime value and assigning it to the Event object on every event that is passed through the system, add a property to SIDData which acts as an alias `datetime` to `dt`. Eventually support for `data['foo'].datetime` may be removed, and could be considered deprecated.
50 lines
1.5 KiB
Python
50 lines
1.5 KiB
Python
#
|
|
# Copyright 2013 Quantopian, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import heapq
|
|
|
|
from six.moves import reduce
|
|
|
|
|
|
def _decorate_source(source):
|
|
for message in source:
|
|
yield ((message.dt, message.source_id), message)
|
|
|
|
|
|
def date_sorted_sources(*sources):
|
|
"""
|
|
Takes an iterable of sources, generating namestrings and
|
|
piping their output into date_sort.
|
|
"""
|
|
sorted_stream = heapq.merge(*(_decorate_source(s) for s in sources))
|
|
|
|
# Strip out key decoration
|
|
for _, message in sorted_stream:
|
|
yield message
|
|
|
|
|
|
def sequential_transforms(stream_in, *transforms):
|
|
"""
|
|
Apply each transform in transforms sequentially to each event in stream_in.
|
|
Each transform application will add a new entry indexed to the transform's
|
|
hash string.
|
|
"""
|
|
# Recursively apply all transforms to the stream.
|
|
stream_out = reduce(lambda stream, tnfm: tnfm.transform(stream),
|
|
transforms,
|
|
stream_in)
|
|
|
|
return stream_out
|