From d454f2b4ebc89b11d801692aa5a3bcfc68554da1 Mon Sep 17 00:00:00 2001 From: Robert Smallshire Date: Fri, 30 Jan 2015 20:57:26 +0100 Subject: [PATCH] Use the array module rather than the struct module for reading the sample arrays. Gives a x3 speed-up. Also fix automatic encoding detection. --- segpy/datatypes.py | 59 +++++++++++++++------------------------------- segpy/encoding.py | 13 ++++++++-- segpy/reader.py | 6 ++++- segpy/toolkit.py | 18 ++++++-------- segpy/util.py | 5 +++- 5 files changed, 46 insertions(+), 55 deletions(-) diff --git a/segpy/datatypes.py b/segpy/datatypes.py index 11c1f2e..0ffbc1f 100644 --- a/segpy/datatypes.py +++ b/segpy/datatypes.py @@ -1,53 +1,32 @@ DATA_SAMPLE_FORMAT = {1: 'ibm', - 2: 'l', - 3: 'h', - 5: 'f', - 8: 'b'} + 2: 'int32', + 3: 'int16', + 5: 'float32', + 8: 'int8'} -# A mapping from SEG Y data types to format characters used by the struct module, -# known a 'ctypes' -CTYPES = {'l': 'l', - 'long': 'l', - 'int32': 'l', - - 'L': 'L', - 'ulong': 'L', - 'uint32': 'L', - - 'h': 'h', - 'short': 'h', +# A mapping from SEG Y data types to format characters used by the struct module +CTYPES = {'int32': 'i', + 'uint32': 'I', 'int16': 'h', - - 'H': 'H', - 'ushort': 'H', 'uint16': 'H', - - 'c': 'b', - 'char': 'b', - 'b': 'b', - - 'B': 'B', - 'uchar': 'B', - - 'f': 'f', - 'float': 'f', - + 'int8': 'b', + 'uint8': 'B', + 'float32': 'f', 'ibm': 'ibm'} -# TODO This is redundant with data in the SH_def below CTYPE_DESCRIPTION = {'ibm': 'IBM float', - 'l': '32 bit signed integer', - 'L': '32 bit unsigned integer', - 'h': '16 bit signed integer', - 'H': '16 bit unsigned integer', - 'f': 'IEEE float32', - 'b': '8 bit signed char', - 'B': '8 bit unsigned char'} + 'int32': '32 bit signed integer', + 'uint32': '32 bit unsigned integer', + 'int16': '16 bit signed integer', + 'uint16': '16 bit unsigned integer', + 'float32': 'IEEE float32', + 'int8': '8 bit signed integer (byte)', + 'uint8': '8 bit unsigned integer (byte)'} -SIZES = dict(l=4, - L=4, +SIZES = dict(i=4, + I=4, h=2, H=2, b=1, diff --git a/segpy/encoding.py b/segpy/encoding.py index decfb4b..149230e 100644 --- a/segpy/encoding.py +++ b/segpy/encoding.py @@ -42,6 +42,7 @@ def guess_encoding(bs, threshold=0.5): ebcdic_count = 0 ascii_count = 0 + null_count = 0 count = 0 for b in bs: @@ -49,6 +50,8 @@ def guess_encoding(bs, threshold=0.5): ebcdic_count +=1 if b in COMMON_ASCII_CHARS: ascii_count +=1 + if b == 0: + null_count += 1 count += 1 if count == 0: @@ -56,9 +59,10 @@ def guess_encoding(bs, threshold=0.5): ebcdic_freq = ebcdic_count / count ascii_freq = ascii_count / count + null_freq = null_count / count - if ebcdic_freq < threshold and ascii_freq < threshold: - return None + if null_freq == 1.0: + return ASCII # Doesn't matter if ebcdic_freq < threshold and ascii_freq >= threshold: return ASCII @@ -66,4 +70,9 @@ def guess_encoding(bs, threshold=0.5): if ebcdic_freq >= threshold and ascii_freq < threshold: return EBCDIC + if ebcdic_freq < threshold and ascii_freq < threshold: + return None + + + return None diff --git a/segpy/reader.py b/segpy/reader.py index 7302ada..c371d9b 100644 --- a/segpy/reader.py +++ b/segpy/reader.py @@ -1,4 +1,5 @@ from __future__ import print_function +from segpy.encoding import ASCII from segpy.portability import seekable from segpy.util import file_length, filename_from_handle @@ -92,6 +93,9 @@ def create_reader(fh, encoding=None, endian='>', progress=None): if encoding is None: encoding = guess_textual_header_encoding(fh) + if encoding is None: + encoding = ASCII + textual_reel_header = read_textual_reel_header(fh, encoding) binary_reel_header = read_binary_reel_header(fh, endian) extended_textual_header = read_extended_textual_headers(fh, binary_reel_header, encoding) @@ -336,7 +340,7 @@ class SegYReader(object): def data_sample_format_description(self): """A descriptive human-readable description of the data sample format """ - return CTYPE_DESCRIPTION[CTYPES[self.data_sample_format]] + return CTYPE_DESCRIPTION[self.data_sample_format] @property def encoding(self): diff --git a/segpy/toolkit.py b/segpy/toolkit.py index 253c630..c237028 100644 --- a/segpy/toolkit.py +++ b/segpy/toolkit.py @@ -16,7 +16,7 @@ from segpy.binary_reel_header_definition import HEADER_DEF from segpy.ibm_float import ibm2ieee, ieee2ibm from segpy.revisions import canonicalize_revision from segpy.trace_header_definition import TRACE_HEADER_DEF -from segpy.util import file_length, batched, pad, complementary_slices +from segpy.util import file_length, batched, pad, complementary_slices, NATIVE_ENDIANNESS from segpy.portability import EMPTY_BYTE_STRING HEADER_NEWLINE = '\r\n' @@ -410,7 +410,7 @@ def read_trace_header(fh, trace_header_format, pos=None): return trace_header -def read_binary_values(fh, pos=None, ctype='l', count=1, endian='>'): +def read_binary_values(fh, pos=None, ctype='int32', count=1, endian='>'): """Read a series of values from a binary file. Args: @@ -475,14 +475,10 @@ def unpack_values(buf, count, fmt, endian='>'): Returns: A sequence of objects with type corresponding to the format code. """ - c_format = '{}{}{}'.format(endian, count, fmt) - return struct.unpack(c_format, buf) - # We could use array.fromfile() here. On the one hand it's likely - # to be faster and more compact, - # - # On the other, it only works on "real" files, not arbitrary - # file-like-objects and it would require us to handle endian byte - # swapping ourselves. + a = array(fmt, buf) + if endian != NATIVE_ENDIANNESS: + a.byteswap() + return a def format_standard_textual_header(revision, **kwargs): @@ -776,7 +772,7 @@ def write_trace_samples(fh, samples, ctype='l', pos=None, endian='>'): write_binary_values(fh, samples, ctype, pos, endian) -def write_binary_values(fh, values, ctype='l', pos=None, endian='>'): +def write_binary_values(fh, values, ctype, pos=None, endian='>'): """Write a series of values to a file. Args: diff --git a/segpy/util.py b/segpy/util.py index a12ddc9..a531089 100644 --- a/segpy/util.py +++ b/segpy/util.py @@ -1,9 +1,12 @@ import itertools import time import os +import sys from segpy.portability import izip +NATIVE_ENDIANNESS = '<' if sys.byteorder == 'little' else '>' + UNSET = object() def pairwise(iterable): @@ -218,4 +221,4 @@ def now_millis(): def round_up(integer, multiple): """Round up to the nearest multiple""" - return integer if integer % multiple == 0 else integer + multiple - integer % multiple \ No newline at end of file + return integer if integer % multiple == 0 else integer + multiple - integer % multiple