mirror of
https://github.com/wassname/greater_tables_project.git
synced 2026-09-12 12:22:43 +08:00
Improved random dates in quick_fab
This commit is contained in:
Stephen Mildenhall
@@ -11,6 +11,10 @@ Versions and Change Log
|
||||
.. TODO
|
||||
* self.padl and r / 12 in make html width adj s/b elsewhere
|
||||
|
||||
5.2.1
|
||||
-----
|
||||
* Added ``SmartTitle`` class in utilites - eventually add capitalize option for index?
|
||||
* Added ``tikz`` option: if true compute tikz output else when known not to be needed.
|
||||
|
||||
5.2.0
|
||||
------
|
||||
|
||||
@@ -3,5 +3,5 @@ __author__ = 'Stephen J Mildenhall'
|
||||
__version__ = '5.2.1'
|
||||
|
||||
from . core import GT
|
||||
from . fabrications import Fabricator
|
||||
from . fabrications import *
|
||||
from . etcher import Etcher
|
||||
|
||||
@@ -54,7 +54,6 @@ class Configurator(BaseModel):
|
||||
default_formatter: Optional[Union[str, Callable[[Any, str], str]]] = Field(
|
||||
None, description="Optional fallback formatter f-string"
|
||||
)
|
||||
|
||||
table_float_format: Optional[Union[str, Callable[[Any, str], str]]] = Field(
|
||||
None, description="Float format function or format string for the entire table; overrides column-specific formats"
|
||||
)
|
||||
@@ -157,6 +156,9 @@ class Configurator(BaseModel):
|
||||
)
|
||||
|
||||
# tikz specific options
|
||||
tikz: bool = Field(
|
||||
True, description="Compute tikz output (default), else skipped for speed."
|
||||
)
|
||||
tikz_scale: float = Field(
|
||||
1.0, description="Scaling factor applied to LaTeX TikZ tables"
|
||||
)
|
||||
|
||||
+112
-105
@@ -45,6 +45,7 @@ warnings.simplefilter(action='ignore', category=FutureWarning)
|
||||
# GPT recommended approach
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GT(object):
|
||||
"""
|
||||
Create a greater_tables formatting object.
|
||||
@@ -100,101 +101,6 @@ class GT(object):
|
||||
* Sparsify if requested and if multiindex
|
||||
* Result is a dataframe with all object column types and values that
|
||||
reflect the formatting choices.
|
||||
|
||||
|
||||
Parameters
|
||||
-----------
|
||||
|
||||
:param df: target DataFrame or list of lists or markdown table string
|
||||
:param caption: table caption, optional (GT will look for gt_caption
|
||||
attribute of df and use that)
|
||||
:param label: TeX label (used in \\label{} command). For markdown
|
||||
tables with #tbl:... in the caption it is extracted automatically.
|
||||
:param aligners: None or dict (type or colname) -> left | center |
|
||||
right
|
||||
:param formatters: None or dict (type or colname) -> format function
|
||||
for the column; formatters trump ratio_cols
|
||||
:param unbreakable: None or list of columns to be considered unbreakable
|
||||
:param ratio_cols: None, or "all" or list of column names treated as
|
||||
ratios. Set defaults in derived class suitable to application.
|
||||
:param year_cols: None, or "all" or list of column names treated as
|
||||
years (no commas, no decimals). Set defaults in derived class suitable
|
||||
to application.
|
||||
:param date_cols: None, or "all" or list of column names treated as
|
||||
dates. Set defaults in derived class suitable to application.
|
||||
:param raw_cols: None, or "all" or list of column names that are NOT
|
||||
cast to floats. Set defaults in derived class suitable to application.
|
||||
:param show_index: if True, show the index columns, default True
|
||||
:param config.default_integer_str: format f-string for integers, default
|
||||
value '{x:,d}'
|
||||
:param config.default_float_str: format f-string for floats, default
|
||||
value '{x:,.3f}'
|
||||
:param config.default_date_str: format f-string for dates, default '%Y-%m-%d'.
|
||||
NOTE: no braces or x!
|
||||
:param config.default_ratio_str: format f-string for ratios, default '{x:.1%}'
|
||||
:param config.table_float_format: None or format string for floats in the
|
||||
table format function, applied to entire table, default None
|
||||
:param config.table_hrule_width: width of the table top, botton and header
|
||||
hrule, default 1
|
||||
:param config.table_vrule_width: width of the table vrule, separating the
|
||||
index from the body, default 1
|
||||
:param config.hrule_widths: None or tuple of three ints for hrule widths
|
||||
(for use with multiindexes)
|
||||
:param config.vrule_widths: None or tuple of three ints for vrule widths
|
||||
(for use when columns have multiindexes)
|
||||
:param config.sparsify: if True, config.sparsify the index columns, you almost always
|
||||
want this to be true!
|
||||
:param config.sparsify_columns: if True, config.sparsify the columns, default True,
|
||||
generally a better look, headings centered in colspans
|
||||
:param config.spacing: 'tight', 'medium', 'wide' to quickly set cell padding.
|
||||
Medium is default (2, 10, 2, 10).
|
||||
:param config.padding_trbl: None or tuple of four ints for padding, in order
|
||||
top, right, bottom, left.
|
||||
:param config.tikz_scale: scale factor applied to tikz LaTeX tables.
|
||||
:param config.font_body: font size for body text, default 0.9. Units in em.
|
||||
:param config.font_head: font size for header text, default 1.0. Units in em.
|
||||
:param config.font_caption: font size for caption text, default 1.1.
|
||||
Units in em.
|
||||
:param config.font_bold_index: if True, make the index columns bold,
|
||||
default False.
|
||||
:param config.pef_precision: precision (digits after period) for pandas
|
||||
engineering format, default 3.
|
||||
:param config.pef_lower: apply engineering format to floats with absolute
|
||||
value < 10**config.pef_lower; default -3.
|
||||
:param config.pef_upper: apply engineering format to floats with absolute
|
||||
value > 10**config.pef_upper; default 6.
|
||||
:param config.cast_to_floats: if True, try to cast all non-integer, non-date
|
||||
columns to floats
|
||||
:param config.header_row: True: use first row as headers; False no headings.
|
||||
Default True
|
||||
:param config.tabs: None or list of column widths in characters or a common
|
||||
int or float width. (It is converted into em; one character is about
|
||||
0.5em on average; digits are exactly 0.5em.) If None, will be calculated.
|
||||
Default None.
|
||||
:param config.equal: if True, set all column widths config.equal. Default False. Maybe
|
||||
ignored, depending on computed ideal column widths.
|
||||
:param config.caption_align: for the caption
|
||||
:param config.large_ok: signal that you are intentionally applying to a large
|
||||
dataframe. Sub-classes may restrict or apply .head() to df.
|
||||
:param config.max_str_length: maximum displayed length of object types, that
|
||||
are cast to strings. Eg if you have nested DataFrames!
|
||||
:param str_table_fmt: table border format used for string output
|
||||
(markdown), default mixed_grid DEPRECATED??
|
||||
:param config.table_width_mode:
|
||||
'explicit': set using config.max_table_width_em
|
||||
'natural': each cell on one line (can be very wide with long strings)
|
||||
'breakable': wrap breakable cells (text strings) at word boundaries
|
||||
to fit longest word
|
||||
'minimum': wrap breakable and ok-to-break (dates) cells
|
||||
:param config.table_width_header_adjust: additional proportion of table width
|
||||
used to balance header columns.
|
||||
:param config.table_width_header_relax: extra spaces allowed per column heading
|
||||
to facilitate better column header wrapping.
|
||||
:param config.max_table_width_em: max table width used for markdown string output,
|
||||
default 200; width is never less than minimum width. Padding (3 chars
|
||||
per row plus 1) consumed out of config.max_table_width_em in string output mode.
|
||||
:param config.debug: if True, add id to caption and use colored lines in table,
|
||||
default False.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -217,6 +123,100 @@ class GT(object):
|
||||
config_path: Path | None = None,
|
||||
**overrides,
|
||||
):
|
||||
"""
|
||||
Available keyword ``**overrides`:
|
||||
|
||||
:param df: target DataFrame or list of lists or markdown table string
|
||||
:param caption: table caption, optional (GT will look for gt_caption
|
||||
attribute of df and use that)
|
||||
:param label: TeX label (used in \\label{} command). For markdown
|
||||
tables with #tbl:... in the caption it is extracted automatically.
|
||||
:param aligners: None or dict (type or colname) -> left | center |
|
||||
right
|
||||
:param formatters: None or dict (type or colname) -> format function
|
||||
for the column; formatters trump ratio_cols
|
||||
:param unbreakable: None or list of columns to be considered unbreakable
|
||||
:param ratio_cols: None, or "all" or list of column names treated as
|
||||
ratios. Set defaults in derived class suitable to application.
|
||||
:param year_cols: None, or "all" or list of column names treated as
|
||||
years (no commas, no decimals). Set defaults in derived class suitable
|
||||
to application.
|
||||
:param date_cols: None, or "all" or list of column names treated as
|
||||
dates. Set defaults in derived class suitable to application.
|
||||
:param raw_cols: None, or "all" or list of column names that are NOT
|
||||
cast to floats. Set defaults in derived class suitable to application.
|
||||
:param show_index: if True, show the index columns, default True
|
||||
:param config.default_integer_str: format f-string for integers, default
|
||||
value '{x:,d}'
|
||||
:param config.default_float_str: format f-string for floats, default
|
||||
value '{x:,.3f}'
|
||||
:param config.default_date_str: format f-string for dates, default '%Y-%m-%d'.
|
||||
NOTE: no braces or x!
|
||||
:param config.default_ratio_str: format f-string for ratios, default '{x:.1%}'
|
||||
:param config.table_float_format: None or format string for floats in the
|
||||
table format function, applied to entire table, default None
|
||||
:param config.table_hrule_width: width of the table top, botton and header
|
||||
hrule, default 1
|
||||
:param config.table_vrule_width: width of the table vrule, separating the
|
||||
index from the body, default 1
|
||||
:param config.hrule_widths: None or tuple of three ints for hrule widths
|
||||
(for use with multiindexes)
|
||||
:param config.vrule_widths: None or tuple of three ints for vrule widths
|
||||
(for use when columns have multiindexes)
|
||||
:param config.sparsify: if True, config.sparsify the index columns, you almost always
|
||||
want this to be true!
|
||||
:param config.sparsify_columns: if True, config.sparsify the columns, default True,
|
||||
generally a better look, headings centered in colspans
|
||||
:param config.spacing: 'tight', 'medium', 'wide' to quickly set cell padding.
|
||||
Medium is default (2, 10, 2, 10).
|
||||
:param config.padding_trbl: None or tuple of four ints for padding, in order
|
||||
top, right, bottom, left.
|
||||
:param config.tikz_scale: scale factor applied to tikz LaTeX tables.
|
||||
:param config.font_body: font size for body text, default 0.9. Units in em.
|
||||
:param config.font_head: font size for header text, default 1.0. Units in em.
|
||||
:param config.font_caption: font size for caption text, default 1.1.
|
||||
Units in em.
|
||||
:param config.font_bold_index: if True, make the index columns bold,
|
||||
default False.
|
||||
:param config.pef_precision: precision (digits after period) for pandas
|
||||
engineering format, default 3.
|
||||
:param config.pef_lower: apply engineering format to floats with absolute
|
||||
value < 10**config.pef_lower; default -3.
|
||||
:param config.pef_upper: apply engineering format to floats with absolute
|
||||
value > 10**config.pef_upper; default 6.
|
||||
:param config.cast_to_floats: if True, try to cast all non-integer, non-date
|
||||
columns to floats
|
||||
:param config.header_row: True: use first row as headers; False no headings.
|
||||
Default True
|
||||
:param config.tabs: None or list of column widths in characters or a common
|
||||
int or float width. (It is converted into em; one character is about
|
||||
0.5em on average; digits are exactly 0.5em.) If None, will be calculated.
|
||||
Default None.
|
||||
:param config.equal: if True, set all column widths config.equal. Default False. Maybe
|
||||
ignored, depending on computed ideal column widths.
|
||||
:param config.caption_align: for the caption
|
||||
:param config.large_ok: signal that you are intentionally applying to a large
|
||||
dataframe. Sub-classes may restrict or apply .head() to df.
|
||||
:param config.max_str_length: maximum displayed length of object types, that
|
||||
are cast to strings. Eg if you have nested DataFrames!
|
||||
:param str_table_fmt: table border format used for string output
|
||||
(markdown), default mixed_grid DEPRECATED??
|
||||
:param config.table_width_mode:
|
||||
'explicit': set using config.max_table_width_em
|
||||
'natural': each cell on one line (can be very wide with long strings)
|
||||
'breakable': wrap breakable cells (text strings) at word boundaries
|
||||
to fit longest word
|
||||
'minimum': wrap breakable and ok-to-break (dates) cells
|
||||
:param config.table_width_header_adjust: additional proportion of table width
|
||||
used to balance header columns.
|
||||
:param config.table_width_header_relax: extra spaces allowed per column heading
|
||||
to facilitate better column header wrapping.
|
||||
:param config.max_table_width_em: max table width used for markdown string output,
|
||||
default 200; width is never less than minimum width. Padding (3 chars
|
||||
per row plus 1) consumed out of config.max_table_width_em in string output mode.
|
||||
:param config.debug: if True, add id to caption and use colored lines in table,
|
||||
default False.
|
||||
"""
|
||||
if config and config_path:
|
||||
raise ValueError(
|
||||
"Pass either 'config' or 'config_path', not both.")
|
||||
@@ -400,14 +400,12 @@ class GT(object):
|
||||
if not np.any((is_integer_dtype(self.df.iloc[:, i]),
|
||||
is_datetime64_any_dtype(self.df.iloc[:, i]))):
|
||||
try:
|
||||
self.df.iloc[:, i] = self.df.iloc[:,
|
||||
i].astype(float)
|
||||
self.df.iloc[:, i] = self.df.iloc[: ,i].astype(float)
|
||||
logger.debug(
|
||||
f'coerce {i}={c} from {old_type} to float')
|
||||
f'coerced column {i}={c} from {old_type} to float success')
|
||||
except (ValueError, TypeError):
|
||||
logger.debug(
|
||||
f'coercing {i}={c} from {old_type} to float FAILED')
|
||||
|
||||
# massage unbreakable
|
||||
if unbreakable is None:
|
||||
unbreakable = []
|
||||
@@ -663,7 +661,7 @@ class GT(object):
|
||||
"""Year formatter."""
|
||||
try:
|
||||
return f'{int(x):d}'
|
||||
except ValueError:
|
||||
except (TypeError, ValueError):
|
||||
return str(x)
|
||||
|
||||
def default_raw_formatter(self, x):
|
||||
@@ -862,8 +860,9 @@ class GT(object):
|
||||
def apply_formatters_work(df, formatters):
|
||||
"""Apply formatters to a DataFrame."""
|
||||
try:
|
||||
# very surprising bug: if df is empty new_df will have type float!!
|
||||
new_df = pd.DataFrame({i: map(f, df.iloc[:, i])
|
||||
for i, f in enumerate(formatters)})
|
||||
for i, f in enumerate(formatters)}, dtype=object)
|
||||
except TypeError:
|
||||
print('NASTY TYPE ERROR')
|
||||
raise
|
||||
@@ -926,7 +925,11 @@ class GT(object):
|
||||
# and all(self.df_tex == self.df_html)):
|
||||
# self._tex_knowledge_df = self.html_knowledge_df
|
||||
# else:
|
||||
self._tex_knowledge_df = self.estimate_column_widths_by_mode('tex')
|
||||
if not self.config.tikz:
|
||||
# just repeat html so you have something
|
||||
self._tex_knowledge_df = self.html_knowledge_df
|
||||
else:
|
||||
self._tex_knowledge_df = self.estimate_column_widths_by_mode('tex')
|
||||
return self._tex_knowledge_df
|
||||
|
||||
@property
|
||||
@@ -1040,6 +1043,7 @@ class GT(object):
|
||||
minimum_width = {}
|
||||
header_natural = {}
|
||||
header_minimum = {}
|
||||
|
||||
for col_name in df.columns:
|
||||
minimum_width[col_name] = (
|
||||
df[col_name].str
|
||||
@@ -1128,8 +1132,8 @@ class GT(object):
|
||||
ans['recommended'] = ans['minimum_width']
|
||||
space = target_width - minimum
|
||||
logger.warning(
|
||||
'Desired width too small for pleasant formatting, table will be too wide by spare space %s < 0.',
|
||||
space)
|
||||
'Mode %s, desired width too small for pleasant formatting, table will be too wide by spare space %s em < 0.',
|
||||
mode, space)
|
||||
logger.info(f'{mode=} {target_width=}, {natural=}, {acceptable=}, {minimum=}, {max_extra=}, {space=}')
|
||||
|
||||
# this section tweaks the widths for column headers -> text output only.
|
||||
@@ -1609,6 +1613,8 @@ class GT(object):
|
||||
:param label:
|
||||
:return:
|
||||
"""
|
||||
if not self.config.tikz:
|
||||
return ''
|
||||
# pull out arguments (convert to local vars - these used to be arguments)
|
||||
column_sep = self.config.tikz_column_sep
|
||||
row_sep = self.config.tikz_row_sep
|
||||
@@ -1792,7 +1798,8 @@ class GT(object):
|
||||
# c = wfloat_format(c)
|
||||
s = f'{nl} {{cell:{ad2[al]}{colw[c]}s}} '
|
||||
nl = '\\&'
|
||||
sio.write(s.format(cell=c + '\\I'))
|
||||
# cols may not be strings...
|
||||
sio.write(s.format(cell=str(c) + '\\I'))
|
||||
sio.write('\\& \\\\\n')
|
||||
|
||||
# write table entries
|
||||
|
||||
+113
-28
@@ -9,7 +9,7 @@ from itertools import cycle, chain, count, zip_longest, product, islice
|
||||
import logging
|
||||
from math import prod
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
from typing import Optional, Union, Literal
|
||||
import hashlib
|
||||
import random
|
||||
import re
|
||||
@@ -18,17 +18,80 @@ import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
__all__ = ['Fabricator', 'make_df', 'quick_df', 'quick_fab', 'rand_df']
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# helpers: random date times
|
||||
def random_datetime_series(
|
||||
n: int,
|
||||
*,
|
||||
rng,
|
||||
now: pd.Timestamp | None = None,
|
||||
) -> pd.Series:
|
||||
"""
|
||||
Generate a pandas Series of random datetimes with millisecond precision.
|
||||
|
||||
Algorithm:
|
||||
1) Randomly choose units ∈ {days, months, years}.
|
||||
2) Draw K ~ Uniform{0, 1, ..., 20} (integer).
|
||||
3) Set start = now - K·units.
|
||||
4) Sample n datetimes uniformly between [start, now].
|
||||
|
||||
Parameters
|
||||
----------
|
||||
n
|
||||
Number of random datetimes to generate.
|
||||
seed
|
||||
RNG seed for reproducibility.
|
||||
now
|
||||
Reference end timestamp; defaults to current UTC time.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pandas.Series
|
||||
Random datetimes (datetime64[ns]) with ms precision.
|
||||
"""
|
||||
rng = rng or np.random.default_rng()
|
||||
now_ts = pd.Timestamp.utcnow() if now is None else pd.Timestamp(now)
|
||||
|
||||
# 1) Units
|
||||
unit: Literal["days", "months", "years"] = rng.choice(
|
||||
["days", "months", "years"])
|
||||
|
||||
# 2) Integer K for length of period
|
||||
default_spans = {"days": 730, "months": 120, "years": 16}
|
||||
k = int(rng.integers(0, default_spans[unit]))
|
||||
|
||||
# 3) Start
|
||||
if unit == "days":
|
||||
start = now_ts - pd.DateOffset(days=k)
|
||||
elif unit == "months":
|
||||
start = now_ts - pd.DateOffset(months=k)
|
||||
else:
|
||||
start = now_ts - pd.DateOffset(years=k)
|
||||
|
||||
# 4) Sample uniformly in nanoseconds
|
||||
start_ns = start.value
|
||||
now_ns = now_ts.value
|
||||
u = rng.random(n)
|
||||
ns = start_ns + (now_ns - start_ns) * u
|
||||
stamps = pd.to_datetime(ns.astype("int64"))
|
||||
|
||||
return pd.Series(stamps, name="datetime")
|
||||
|
||||
|
||||
# main class
|
||||
class Fabricator:
|
||||
"""
|
||||
Fabricate dataframes.
|
||||
"""
|
||||
|
||||
metric_roots = ['absorption', 'acceleration', 'account', 'activation', 'adjustment', 'allocation', 'amplitude', 'approval', 'asset', 'atom', 'attrition', 'balance', 'band', 'binding', 'cancellation', 'capacitance', 'capital', 'cashflow', 'category', 'cell', 'charge', 'claim', 'commission', 'compound', 'concentration', 'conductivity', 'constraint', 'consumption', 'conversion', 'correlation', 'cost', 'count', 'coverage', 'credit', 'current', 'debt', 'decay', 'decibel', 'deductible', 'deficit', 'deflator', 'demand', 'density', 'development', 'diffusion', 'discount', 'distribution', 'dividend', 'dose', 'duration', 'earnings', 'efficiency', 'elasticity', 'employment', 'energy', 'entropy', 'enzyme', 'estimate', 'excess', 'exhaustion', 'expense', 'exposure', 'failure', 'field', 'flux', 'force', 'frequency', 'funding', 'gdp', 'gene', 'gradient', 'growth', 'half_life', 'incidence', 'income', 'index', 'indicator', 'inequality', 'inflation', 'inhibition', 'input', 'intensity', 'investment', 'kurtosis', 'lapse', 'layer', 'leverage', 'liability', 'limit', 'loss', 'luminosity', 'margin', 'mass', 'molecule', 'momentum', 'mortality', 'neutron', 'noise', 'operating', 'output', 'penalty', 'photon', 'policy', 'portfolio', 'potential', 'power', 'preference', 'premium', 'pressure', 'price', 'productivity', 'profit', 'protein', 'proton', 'provision', 'radiation', 'rate', 'ratio', 'reaction', 'recovery', 'reflection', 'refraction', 'renewal', 'reserve', 'residual', 'resistance', 'return', 'revenue', 'risk', 'sample', 'savings', 'scenario', 'score', 'sector', 'settlement', 'severity', 'shock', 'shortfall', 'signal', 'skewness', 'spread', 'strain', 'stress', 'subsidy', 'supply', 'tail', 'tariff', 'tax', 'temperature', 'tension', 'term', 'threshold', 'trade', 'trend', 'turbulence', 'unemployment', 'uptake', 'utility', 'utilization', 'valuation', 'variance', 'velocity', 'viscosity', 'volatility', 'voltage', 'volume', 'wage', 'wavelength', 'wealth', 'weight', 'yield']
|
||||
metric_roots = ['absorption', 'acceleration', 'account', 'activation', 'adjustment', 'allocation', 'amplitude', 'approval', 'asset', 'atom', 'attrition', 'balance', 'band', 'binding', 'cancellation', 'capacitance', 'capital', 'cashflow', 'category', 'cell', 'charge', 'claim', 'commission', 'compound', 'concentration', 'conductivity', 'constraint', 'consumption', 'conversion', 'correlation', 'cost', 'count', 'coverage', 'credit', 'current', 'debt', 'decay', 'decibel', 'deductible', 'deficit', 'deflator', 'demand', 'density', 'development', 'diffusion', 'discount', 'distribution', 'dividend', 'dose', 'duration', 'earnings', 'efficiency', 'elasticity', 'employment', 'energy', 'entropy', 'enzyme', 'estimate', 'excess', 'exhaustion', 'expense', 'exposure', 'failure', 'field', 'flux', 'force', 'frequency', 'funding', 'gdp', 'gene', 'gradient', 'growth', 'half_life', 'incidence', 'income', 'index', 'indicator', 'inequality', 'inflation', 'inhibition', 'input', 'intensity', 'investment',
|
||||
'kurtosis', 'lapse', 'layer', 'leverage', 'liability', 'limit', 'loss', 'luminosity', 'margin', 'mass', 'molecule', 'momentum', 'mortality', 'neutron', 'noise', 'operating', 'output', 'penalty', 'photon', 'policy', 'portfolio', 'potential', 'power', 'preference', 'premium', 'pressure', 'price', 'productivity', 'profit', 'protein', 'proton', 'provision', 'radiation', 'rate', 'ratio', 'reaction', 'recovery', 'reflection', 'refraction', 'renewal', 'reserve', 'residual', 'resistance', 'return', 'revenue', 'risk', 'sample', 'savings', 'scenario', 'score', 'sector', 'settlement', 'severity', 'shock', 'shortfall', 'signal', 'skewness', 'spread', 'strain', 'stress', 'subsidy', 'supply', 'tail', 'tariff', 'tax', 'temperature', 'tension', 'term', 'threshold', 'trade', 'trend', 'turbulence', 'unemployment', 'uptake', 'utility', 'utilization', 'valuation', 'variance', 'velocity', 'viscosity', 'volatility', 'voltage', 'volume', 'wage', 'wavelength', 'wealth', 'weight', 'yield']
|
||||
|
||||
metric_suffix = ["", "rate", "score", "amount", "index", "ratio", "factor", "value"]
|
||||
metric_suffix = ["", "rate", "score", "amount",
|
||||
"index", "ratio", "factor", "value"]
|
||||
|
||||
def __init__(self, decorate=False, seed: Optional[int] = None):
|
||||
"""
|
||||
@@ -66,7 +129,7 @@ class Fabricator:
|
||||
# trim down slightly
|
||||
# dont' want | in tex...messes up tables!
|
||||
pat = re.compile(r'(?<!\\)\b[a-z]{4,}\b|\|')
|
||||
tex_list = [i for i in tex_list if not pat.search(i) and len(i)<=50]
|
||||
tex_list = [i for i in tex_list if not pat.search(i) and len(i) <= 50]
|
||||
self.rng.shuffle(tex_list)
|
||||
self._tex_gen = cycle(tex_list)
|
||||
|
||||
@@ -104,14 +167,15 @@ class Fabricator:
|
||||
if isinstance(df.index, pd.MultiIndex):
|
||||
# mustn't drop all the levels!
|
||||
drop_levels = [i for i, lvl in enumerate(df.index.levels)
|
||||
if len(lvl) == 1]
|
||||
if len(lvl) == 1]
|
||||
if len(drop_levels) == df.index.nlevels:
|
||||
drop_levels.pop()
|
||||
if len(drop_levels):
|
||||
logger.info('dropping empty index levels %s', drop_levels)
|
||||
df = df.droplevel(drop_levels)
|
||||
if isinstance(df.columns, pd.MultiIndex):
|
||||
drop_levels = [i for i, lvl in enumerate(df.columns.levels) if len(lvl) == 1]
|
||||
drop_levels = [i for i, lvl in enumerate(
|
||||
df.columns.levels) if len(lvl) == 1]
|
||||
if len(drop_levels) == df.columns.nlevels:
|
||||
drop_levels.pop()
|
||||
if len(drop_levels):
|
||||
@@ -126,6 +190,8 @@ class Fabricator:
|
||||
"""
|
||||
Fabricate a dataframe with the given specification.
|
||||
|
||||
metric_name_spec = '' or tuple.list of names, or a spec
|
||||
|
||||
Data types
|
||||
|
||||
d date
|
||||
@@ -158,38 +224,48 @@ class Fabricator:
|
||||
"""
|
||||
# validate args
|
||||
assert column_groups == 0 or column_levels <= column_groups, 'Column levels must be <= groups'
|
||||
assert index_names is None or len(index_names) == index_levels, 'Index names must have length index_levels'
|
||||
assert column_names is None or len(column_names) == column_levels, 'Column names must have length column_levels'
|
||||
assert index_names is None or len(
|
||||
index_names) == index_levels, 'Index names must have length index_levels'
|
||||
assert column_names is None or len(
|
||||
column_names) == column_levels, 'Column names must have length column_levels'
|
||||
|
||||
self._last_args = dict(rows=rows, data_spec=data_spec, index_levels=index_levels,
|
||||
index_names=index_names, column_groups=column_groups, column_levels=column_levels,
|
||||
column_names=column_names, decorate=decorate, simplify=simplify, oversample=oversample)
|
||||
index_names=index_names, column_groups=column_groups, column_levels=column_levels,
|
||||
column_names=column_names, decorate=decorate, simplify=simplify, oversample=oversample)
|
||||
|
||||
# figure data_spec and hence (important) number of metrics
|
||||
data_spec = self._parse_colspec(data_spec)
|
||||
metrics = len(data_spec)
|
||||
if oversample > 1:
|
||||
df = self.uber(oversample * rows, metrics, data_spec, index_levels=index_levels,
|
||||
|
||||
index_names=index_names, column_groups=column_groups, column_levels=column_levels,
|
||||
column_names=column_names, decorate=decorate, oversample=1)
|
||||
df = self.make(oversample * rows, metrics, data_spec, index_levels=index_levels,
|
||||
index_names=index_names, column_groups=column_groups, column_levels=column_levels,
|
||||
column_names=column_names, decorate=decorate, oversample=1)
|
||||
df = df.iloc[:rows, :]
|
||||
return df
|
||||
|
||||
inames = index_names or [f'i_{i}' for i in range(index_levels)]
|
||||
index = pd.MultiIndex.from_tuples(islice(product(*(self._generate_column('s', v) for v in self.primes_for_product(rows, index_levels))), rows), names=inames)
|
||||
index = pd.MultiIndex.from_tuples(islice(product(*(self._generate_column(
|
||||
's', v) for v in self.primes_for_product(rows, index_levels))), rows), names=inames)
|
||||
|
||||
# create with col groups and drop later if needed
|
||||
if metric_name_spec == '':
|
||||
metric_names = [self.metric_name(t) for t in data_spec]
|
||||
elif isinstance(metric_name_spec, (list, tuple)):
|
||||
metric_names = [i.strip() for i in metric_name_spec]
|
||||
assert len(metric_names) == metrics, f"metric_name_spec must have name for each of {
|
||||
metrics} metrics"
|
||||
else:
|
||||
metric_name_spec = self._parse_colspec(metric_name_spec)
|
||||
assert len(metric_name_spec) == len(data_spec), "metric name spec not consistent with data spec"
|
||||
metric_names = [self._generate_column(dt, 1).iloc[0] for dt in metric_name_spec]
|
||||
assert len(metric_name_spec) == len(
|
||||
data_spec), "metric name spec not consistent with data spec"
|
||||
metric_names = [self._generate_column(
|
||||
dt, 1).iloc[0] for dt in metric_name_spec]
|
||||
if column_groups > 0:
|
||||
cnames = (column_names or [f'c_{i}' for i in range(column_levels)]) + ['metric']
|
||||
cnames = (column_names or [
|
||||
f'c_{i}' for i in range(column_levels)]) + ['metric']
|
||||
columns_pfp = self.primes_for_product(column_groups, column_levels)
|
||||
cgroup_product = product(*(self._generate_column('s', v) for v in columns_pfp))
|
||||
cgroup_product = product(
|
||||
*(self._generate_column('s', v) for v in columns_pfp))
|
||||
# take first column_groups entries - islice works without creating the full iterable
|
||||
cgroup_product = islice(cgroup_product, column_groups)
|
||||
# add metrics
|
||||
@@ -214,7 +290,7 @@ class Fabricator:
|
||||
|
||||
# fill in the data, data_spec x column_groups groups
|
||||
for c, dt in zip(df.columns, data_spec * column_groups):
|
||||
df[c] =self._generate_column(dt, rows).values
|
||||
df[c] = self._generate_column(dt, rows).values
|
||||
|
||||
if simplify:
|
||||
df = self.drop_singleton_levels(df)
|
||||
@@ -254,14 +330,16 @@ class Fabricator:
|
||||
metrics = columns
|
||||
index_levels = self.rng.integers(1, 3 + 1)
|
||||
if rows == 0:
|
||||
rows = self.rng.integers(5 * (column_groups + index_levels), 10 * (column_groups + index_levels) + 1)
|
||||
rows = self.rng.integers(
|
||||
5 * (column_groups + index_levels), 10 * (column_groups + index_levels) + 1)
|
||||
|
||||
valid_types = [i for i in ['f', 's2', 's' 'i', 'l', 'f', 'f', 'd', 'f', 'i', 's3', 'l', 'h', 't', 'p', 'x', 'r', 'y'] if i not in omit]
|
||||
valid_types = [i for i in ['f', 's2', 's' 'i', 'l', 'f', 'f', 'd',
|
||||
'f', 'i', 's3', 'l', 'h', 't', 'p', 'x', 'r', 'y'] if i not in omit]
|
||||
data_spec = ''.join(self.rng.choice(valid_types, size=metrics))
|
||||
missing = round(float(self.rng.uniform(0, 0.15)), 2)
|
||||
return self.make(rows=rows, data_spec=data_spec, index_levels=index_levels,
|
||||
column_groups=column_groups, column_levels=column_levels,
|
||||
decorate=False, simplify=True, oversample=1)
|
||||
column_groups=column_groups, column_levels=column_levels,
|
||||
decorate=False, simplify=True, oversample=1)
|
||||
|
||||
def _parse_colspec(self, spec: str) -> list[str]:
|
||||
return re.findall(r's\d+|[a-z]', spec)
|
||||
@@ -286,9 +364,7 @@ class Fabricator:
|
||||
if dtype == 'i':
|
||||
return pd.Series(self.rng.integers(-1e4, 1e6, size=n), dtype='int64')
|
||||
if dtype == 'd':
|
||||
start_date = Fabricator.random_date_within_last_n_years(
|
||||
10)
|
||||
return pd.Series(pd.date_range(start=start_date, periods=n, freq='D'))
|
||||
return random_datetime_series(n, rng=self.rng)
|
||||
if dtype == 'y':
|
||||
return pd.Series(random.sample(range(1990, 2031), n))
|
||||
if dtype == 't':
|
||||
@@ -312,7 +388,7 @@ class Fabricator:
|
||||
|
||||
def metric_name(self, type_hint):
|
||||
"""Return a one-word metric name."""
|
||||
nm = next(self._metric_namer)
|
||||
nm = next(self._metric_namer)
|
||||
if self.decorate:
|
||||
if type_hint == 'y':
|
||||
nm += ' year'
|
||||
@@ -395,3 +471,12 @@ class Fabricator:
|
||||
if shuffle:
|
||||
self.rng.shuffle(primes)
|
||||
return primes
|
||||
|
||||
|
||||
def quick_fab(rows: int = 10, data_spec: str = 's3sfid', **kwargs):
|
||||
"""One-stop quick fabrication of a random dataframe."""
|
||||
fab = Fabricator()
|
||||
return fab.make(rows, data_spec, **kwargs)
|
||||
|
||||
|
||||
rand_df = make_df = quick_df = quick_fab
|
||||
|
||||
@@ -310,6 +310,10 @@ class TextLength:
|
||||
@staticmethod
|
||||
def text_display_len(s: str) -> float:
|
||||
"""Estimate display width in ems, ignoring HTML tags, interpreting TeX, and HTML entities."""
|
||||
# can be called on an index that may not be a string??
|
||||
if not isinstance(s, str):
|
||||
# print(f'{s} is {type(s)}!!')
|
||||
s = str(s)
|
||||
def strip_html_tags(text):
|
||||
return re.sub(r'<[^>]*>', '', text)
|
||||
|
||||
@@ -863,3 +867,25 @@ class RichOutput:
|
||||
table.add_row(*row.tolist())
|
||||
|
||||
return table
|
||||
|
||||
|
||||
class SmartTitle():
|
||||
"""Support reasonable Title case for text."""
|
||||
# TODO: Implement smart titling!
|
||||
@staticmethod
|
||||
def smart_title(text):
|
||||
"""Slightly smart title capitalization (GPT4o)."""
|
||||
small_words = {"a", "an", "and", "as", "at", "but", "by", "for",
|
||||
"in", "is", "of", "on", "or", "the", "to", "up", "via", "vs"}
|
||||
words = text.split()
|
||||
result = []
|
||||
|
||||
for i, word in enumerate(words):
|
||||
if len(word) <= 3 and word.isupper():
|
||||
result.append(word) # already acronym-like
|
||||
elif word.lower() in small_words and i != 0:
|
||||
result.append(word.lower())
|
||||
else:
|
||||
result.append(word.capitalize())
|
||||
|
||||
return " ".join(result)
|
||||
|
||||
Reference in New Issue
Block a user