wassname/segpy
1
0
Fork 0
mirror of https://github.com/wassname/segpy.git synced 2026-08-04 13:14:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

Use the array module rather than the struct module for reading the sample arrays. Gives a x3 speed-up. Also fix automatic encoding detection.

Browse Source
This commit is contained in:
Robert Smallshire
2015-01-30 20:57:26 +01:00
parent 5322c2a5d2
commit d454f2b4eb
5 changed files with 46 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
segpy/datatypes.py
+19 -40
View File
@@ -1,53 +1,32 @@
DATA_SAMPLE_FORMAT = {1: 'ibm',
2: 'l',
3: 'h',
5: 'f',
8: 'b'}
2: 'int32',
3: 'int16',
5: 'float32',
8: 'int8'}
# A mapping from SEG Y data types to format characters used by the struct module,
# known a 'ctypes'
CTYPES = {'l': 'l',
'long': 'l',
'int32': 'l',
'L': 'L',
'ulong': 'L',
'uint32': 'L',
'h': 'h',
'short': 'h',
# A mapping from SEG Y data types to format characters used by the struct module
CTYPES = {'int32': 'i',
'uint32': 'I',
'int16': 'h',
'H': 'H',
'ushort': 'H',
'uint16': 'H',
'c': 'b',
'char': 'b',
'b': 'b',
'B': 'B',
'uchar': 'B',
'f': 'f',
'float': 'f',
'int8': 'b',
'uint8': 'B',
'float32': 'f',
'ibm': 'ibm'}
# TODO This is redundant with data in the SH_def below
CTYPE_DESCRIPTION = {'ibm': 'IBM float',
'l': '32 bit signed integer',
'L': '32 bit unsigned integer',
'h': '16 bit signed integer',
'H': '16 bit unsigned integer',
'f': 'IEEE float32',
'b': '8 bit signed char',
'B': '8 bit unsigned char'}
'int32': '32 bit signed integer',
'uint32': '32 bit unsigned integer',
'int16': '16 bit signed integer',
'uint16': '16 bit unsigned integer',
'float32': 'IEEE float32',
'int8': '8 bit signed integer (byte)',
'uint8': '8 bit unsigned integer (byte)'}
SIZES = dict(l=4,
L=4,
SIZES = dict(i=4,
I=4,
h=2,
H=2,
b=1,
segpy/encoding.py
+11 -2
View File
@@ -42,6 +42,7 @@ def guess_encoding(bs, threshold=0.5):
ebcdic_count = 0
ascii_count = 0
null_count = 0
count = 0
for b in bs:
@@ -49,6 +50,8 @@ def guess_encoding(bs, threshold=0.5):
ebcdic_count +=1
if b in COMMON_ASCII_CHARS:
ascii_count +=1
if b == 0:
null_count += 1
count += 1
if count == 0:
@@ -56,9 +59,10 @@ def guess_encoding(bs, threshold=0.5):
ebcdic_freq = ebcdic_count / count
ascii_freq = ascii_count / count
null_freq = null_count / count
if ebcdic_freq < threshold and ascii_freq < threshold:
return None
if null_freq == 1.0:
return ASCII # Doesn't matter
if ebcdic_freq < threshold and ascii_freq >= threshold:
return ASCII
@@ -66,4 +70,9 @@ def guess_encoding(bs, threshold=0.5):
if ebcdic_freq >= threshold and ascii_freq < threshold:
return EBCDIC
if ebcdic_freq < threshold and ascii_freq < threshold:
return None
return None
segpy/reader.py
+5 -1
View File
@@ -1,4 +1,5 @@
from __future__ import print_function
from segpy.encoding import ASCII
from segpy.portability import seekable
from segpy.util import file_length, filename_from_handle
@@ -92,6 +93,9 @@ def create_reader(fh, encoding=None, endian='>', progress=None):
if encoding is None:
encoding = guess_textual_header_encoding(fh)
if encoding is None:
encoding = ASCII
textual_reel_header = read_textual_reel_header(fh, encoding)
binary_reel_header = read_binary_reel_header(fh, endian)
extended_textual_header = read_extended_textual_headers(fh, binary_reel_header, encoding)
@@ -336,7 +340,7 @@ class SegYReader(object):
def data_sample_format_description(self):
"""A descriptive human-readable description of the data sample format
"""
return CTYPE_DESCRIPTION[CTYPES[self.data_sample_format]]
return CTYPE_DESCRIPTION[self.data_sample_format]
@property
def encoding(self):
segpy/toolkit.py
+7 -11
View File
@@ -16,7 +16,7 @@ from segpy.binary_reel_header_definition import HEADER_DEF
from segpy.ibm_float import ibm2ieee, ieee2ibm
from segpy.revisions import canonicalize_revision
from segpy.trace_header_definition import TRACE_HEADER_DEF
from segpy.util import file_length, batched, pad, complementary_slices
from segpy.util import file_length, batched, pad, complementary_slices, NATIVE_ENDIANNESS
from segpy.portability import EMPTY_BYTE_STRING
HEADER_NEWLINE = '\r\n'
@@ -410,7 +410,7 @@ def read_trace_header(fh, trace_header_format, pos=None):
return trace_header
def read_binary_values(fh, pos=None, ctype='l', count=1, endian='>'):
def read_binary_values(fh, pos=None, ctype='int32', count=1, endian='>'):
"""Read a series of values from a binary file.
Args:
@@ -475,14 +475,10 @@ def unpack_values(buf, count, fmt, endian='>'):
Returns:
A sequence of objects with type corresponding to the format code.
"""
c_format = '{}{}{}'.format(endian, count, fmt)
return struct.unpack(c_format, buf)
# We could use array.fromfile() here. On the one hand it's likely
# to be faster and more compact,
#
# On the other, it only works on "real" files, not arbitrary
# file-like-objects and it would require us to handle endian byte
# swapping ourselves.
a = array(fmt, buf)
if endian != NATIVE_ENDIANNESS:
a.byteswap()
return a
def format_standard_textual_header(revision, **kwargs):
@@ -776,7 +772,7 @@ def write_trace_samples(fh, samples, ctype='l', pos=None, endian='>'):
write_binary_values(fh, samples, ctype, pos, endian)
def write_binary_values(fh, values, ctype='l', pos=None, endian='>'):
def write_binary_values(fh, values, ctype, pos=None, endian='>'):
"""Write a series of values to a file.
Args:
segpy/util.py
+4 -1
View File
@@ -1,9 +1,12 @@
import itertools
import time
import os
import sys
from segpy.portability import izip
NATIVE_ENDIANNESS = '<' if sys.byteorder == 'little' else '>'
UNSET = object()
def pairwise(iterable):
@@ -218,4 +221,4 @@ def now_millis():
def round_up(integer, multiple):
"""Round up to the nearest multiple"""
return integer if integer % multiple == 0 else integer + multiple - integer % multiple
return integer if integer % multiple == 0 else integer + multiple - integer % multiple