diff --git a/greater_tables/greater_tables.py b/greater_tables/greater_tables.py index 3071d1c..526ad15 100644 --- a/greater_tables/greater_tables.py +++ b/greater_tables/greater_tables.py @@ -2,14 +2,18 @@ from bs4 import BeautifulSoup from decimal import InvalidOperation from io import StringIO +from itertools import groupby import logging import numpy as np import pandas as pd from pandas.api.types import is_datetime64_any_dtype, is_integer_dtype,\ is_float_dtype # , is_numeric_dtype +from pathlib import Path import sys import warnings -from .hasher import df_short_hash + + +from .hasher import df_short_hash # turn this fuck-fest off pd.set_option('future.no_silent_downcasting', True) @@ -25,7 +29,7 @@ if logger.hasHandlers(): # Clear existing handlers logger.handlers.clear() # SET DEGBUUGER LEVEL -LEVEL = logging.ERROR # DEBUG or INFO, WARNING, ERROR, CRITICAL +LEVEL = logging.WARNING # DEBUG or INFO, WARNING, ERROR, CRITICAL logger.setLevel(LEVEL) handler = logging.StreamHandler(sys.stderr) handler.setLevel(LEVEL) @@ -38,39 +42,80 @@ logger.info('Logger Setup; module recompiled.') class GT(object): """Create greater_tables.""" - def __init__(self, df, caption='', - aligners=None, default_integer_str='{x:,d}', + def __init__(self, + df, + caption='', + aligners=None, + ratio_cols=None, + year_cols=None, + default_integer_str='{x:,d}', default_float_str='{x:,.3f}', - default_date_str='{x:%Y-%m-%d}', - ratio_cols=None, ratio_default_str='{x:.1%}', - cast_to_floats=True, hrule_widths=None, vrule_widths=None, - column_names=False , - sparsify=True, - pef_precision=3, pef_lower=-3, pef_upper=6, - font_size=0.9, debug=False): + default_date_str='%Y-%m-%d', + default_ratio_str='{x:.1%}', + cast_to_floats=True, + table_hrule_width=1, + table_vrule_width=1, + hrule_widths=None, + vrule_widths=None, + sparsify=True, # index sparsification - almost certainly want this! + sparsify_columns=True, # column sparsification with colspans + spacing='medium', # tight, medium, wide + padding_trbl=None, # tuple of four ints for padding + font_body=0.9, + font_head=1.0, + font_caption=1.1, + font_bold_index=False, + pef_precision=3, + pef_lower=-3, + pef_upper=6, + debug=False): """ - Create a greater_tables formatting object, setting defaults. + Create a greater_tables formatting object. + Provides html and latex output in quarto/Jupyter accessible manner. Wraps AND COPIES the dataframe df. WILL NOT REFLECT CHANGES TO DF. - Provides html, latex, and markdown output in quarto/Jupyter accessible manner. - - pef determines where engineering format is used; pef_precision - - other date format '{%H:%M:%S}' + Recommended usage is to derive from GT and set defaults suitable to your particular application. + In that way you can maintain a "house-style" + :param df: target DataFrame + :param caption: table caption, optional :param aligners: None or dict (type or colname) -> left | center | right - :param formatters: None or dict type -> format function to override defaults. + :param ratio_cols: None, or "all" or list of column names treated as ratios. Set defaults in derived class suitable to application. + :param year_cols: None, or "all" or list of column names treated as years (no commas, no decimals). Set defaults in derived class suitable to application. + :param default_integer_str: format f-string for integers, default '{x:,d}' + :param default_float_str: format f-string for floats, default '{x:,.3f}' + :param default_date_str: format f-string for dates, default '%Y-%m-%d'. NOTE: no braces or x! + :param default_ratio_str: format f-string for ratios, default '{x:.1%}' + :param cast_to_floats: if True, try to cast all non-integer, non-date columns to floats + :param table_hrule_width: width of the table top, botton and header hrule, default 1 + :param table_vrule_width: width of the table vrule, separating the index from the body, default 1 + :param hrule_widths: None or tuple of three ints for hrule widths (for use with multiindexes) + :param vrule_widths: None or tuple of three ints for vrule widths (for use when columns have multiindexes) + :param sparsify: if True, sparsify the index columns, you almost always want this to be true! + :param sparsify_columns: if True, sparsify the columns, default True, generally a better look, headings centered in colspans + :param spacing: 'tight', 'medium', 'wide' to quickly set cell padding. Medium is default (2, 10, 2, 10). + :param padding_trbl: None or tuple of four ints for padding, in order top, right, bottom, left. + :param font_body: font size for body text, default 0.9. Units in em. + :param font_head: font size for header text, default 1.0. Units in em. + :param font_caption: font size for caption text, default 1.1. Units in em. + :param font_bold_index: if True, make the index columns bold, default False. + :param pef_precision: precision (digits after period) for pandas engineering format, default 3. + :param pef_lower: apply engineering format to floats with absolute value < 10**pef_lower; default -3. + :param pef_upper: apply engineering format to floats with absolute value > 10**pef_upper; default 6. + :param debug: if True, add id to caption and use colored lines in table, default False. """ if not df.columns.is_unique: logger.warning('column names are not unqiue: will cause problems') self.df = df.copy(deep=True) # the object being formatted self.raw_df = df.copy(deep=True) - if not column_names: - self.df.columns.names = [None] * self.df.columns.nlevels + # if not column_names: + # get rid of column names + # self.df.columns.names = [None] * self.df.columns.nlevels self.df_id = df_short_hash(self.df) - self.caption = caption + (' (id: ' + self.df_id + ')' if debug else '') + self.debug = debug + self.caption = caption + (' (id: ' + self.df_id + ')' if self.debug else '') # before messing self.nindex = self.df.index.nlevels @@ -81,7 +126,8 @@ class GT(object): with warnings.catch_warnings(): warnings.simplefilter("ignore", category=pd.errors.PerformanceWarning) self.df = self.df.reset_index(drop=False, col_level=self.df.columns.nlevels - 1) - + # want the new index to be ints - that is not default if old was multiindex + self.df.index = np.arange(self.df.shape[0], dtype=int) self.index_change_level = GT.changed_column(self.df.iloc[:, :self.nindex]) if self.ncolumns > 1: # will be empty rows above the index headers @@ -89,12 +135,6 @@ class GT(object): self.column_change_level = GT.changed_level(self.raw_df.columns) - # sparsify - if sparsify and self.nindex > 1: - for c in self.df.columns[:self.nindex]: - # spartify returns some other stuff... - self.df[c], _ = GT._sparsify(self.df[c]) - # determine ratio columns if ratio_cols is not None and np.any(self.df.columns.duplicated()): logger.warning('Ratio cols specified with non-unique column names: ignoring request.') @@ -104,13 +144,23 @@ class GT(object): self.ratio_cols = [] elif ratio_cols == 'all': self.ratio_cols = [i for i in self.df.columns] - elif ratio_cols == 'base' or ratio_cols == 'default': - self.ratio_cols = ['max_LR', 'gross_LR', 'net_LR', 'ceded_LR', 'LR', 'COC', 'CoC', 'ROE'] elif ratio_cols is not None and not isinstance(ratio_cols, (tuple, list)): self.ratio_cols = [ratio_cols] else: self.ratio_cols = ratio_cols + # determine year columns + if year_cols is not None and np.any(self.df.columns.duplicated()): + logger.warning('Year cols specified with non-unique column names: ignoring request.') + self.year_cols = [] + else: + if year_cols is None: + self.year_cols = [] + elif year_cols is not None and not isinstance(year_cols, (tuple, list)): + self.year_cols = [year_cols] + else: + self.year_cols = year_cols + with warnings.catch_warnings(): warnings.simplefilter("ignore", category=pd.errors.PerformanceWarning) if cast_to_floats: @@ -155,7 +205,7 @@ class GT(object): aligners = [] self.df_aligners = [] - lrc = {'l': 'gt-left', 'r': 'gt-right', 'c': 'gt-center'} + lrc = {'l': 'grt-left', 'r': 'grt-right', 'c': 'grt-center'} # FIX INDEX ALIGNERS HERE for i, c in enumerate(self.df.columns): if i < self.nindex: @@ -166,6 +216,8 @@ class GT(object): elif c in self.ratio_cols or i in self.float_col_indices or i in self.integer_col_indices: # number -> right self.df_aligners.append('grt-right') + elif c in self.year_cols: + self.df_aligners.append('grt-center') elif i in self.date_col_indices: # center dates, why not! self.df_aligners.append('grt-center') @@ -175,66 +227,87 @@ class GT(object): self.df_idx_aligners = self.df_aligners[:self.nindex] - # do same for index - but klunky... - # not actually sure you ever want right or centered? - # self.df_idx_aligners = [] - # for ser in [self.df.index] if self.df.index.nlevels == 1 else ( - # self.df.index.get_level_values(i) for i in - # range(self.df.index.nlevels)): - # if is_datetime64_any_dtype(ser): - # self.df_idx_aligners.append('gt_left') - # elif is_integer_dtype(ser): - # self.df_idx_aligners.append('gt_left') - # elif is_float_dtype(ser): - # self.df_idx_aligners.append('gt_left') - # else: - # self.df_idx_aligners.append('gt_left') - # store defaults self.default_integer_str = default_integer_str self.default_float_str = default_float_str # VERY rarely used; for floats in cols that are not floats - self.default_date_str = default_date_str - self.ratio_default_str = ratio_default_str + self.default_date_str = default_date_str.replace('{x:', '').replace('}', '') + self.default_ratio_str = default_ratio_str self.pef_precision = pef_precision self.pef_lower = pef_lower self.pef_upper = pef_upper self._pef = None - self.font_size = font_size - self.hrule_widths = hrule_widths - self.vrule_widths = vrule_widths + self.hrule_widths = hrule_widths or (0, 0, 0) + self.vrule_widths = vrule_widths or (0, 0, 0) + self.table_hrule_width = table_hrule_width + self.table_vrule_width = table_vrule_width + self.font_body = font_body + self.font_head = font_head + self.font_caption = font_caption + self.font_bold_index = font_bold_index + self.sparsify_columns = sparsify_columns + + if padding_trbl is None: + if spacing == 'tight': + padding_trbl = (0, 5, 0, 5) + elif spacing == 'medium': + padding_trbl = (2, 10, 2, 10) + elif spacing == 'wide': + padding_trbl = (4, 15, 4, 15) + else: + raise ValueError('spacing must be tight, medium, or wide or tuple of four ints.') + try: + self.padt, self.padr, self.padb, self.padl = padding_trbl + except ValueError: + logger.error(f'padding_trbl {padding_trbl=}, must be four ints, defaultign to medium') + self.padt, self.padr, self.padb, self.padl = 2, 10, 2, 10 + # because of the problem of non-unique indexes use a list and # not a dict to pass the formatters to to_html self._df_formatters = None self.df_style = None self.df_html = None - # finally apply formaters, this radically alters the df + # finally sparsify and then apply formaters + # this radically alters the df, so keep a copy for now... self.df_pre_applying_formatters = self.df.copy() self.df = self.apply_formatters(self.df) + # sparsify + if sparsify and self.nindex > 1: + for c in self.df.columns[:self.nindex]: + # spartify returns some other stuff... + self.df[c], _ = GT.sparsify(self.df[c]) # define the default and easy formatters =================================================== - def default_ratio(self, x): + def default_ratio_formatter(self, x): """Ratio formatter.""" try: - return self.ratio_default_str.format(x=x) + return self.default_ratio_str.format(x=x) except ValueError: return str(x) - def default_date(self, x): + def default_date_formatter(self, x): """Date formatter.""" try: - return self.default_date_str.format(x=x) + return x.strftime(self.default_date_str) if pd.notna(x) else "" + # return self.default_date_str.format(x=x) # return f'{x:%Y-%m-%d}' # f"{dt:%H:%M:%S}" except ValueError: # logger.error(f'date error with {x=}') return str(x) - def default_integer(self, x): + def default_integer_formatter(self, x): """Integer formatter.""" try: return self.default_integer_str.format(x=x) except ValueError: return str(x) + def default_year_formatter(self, x): + """Year formatter.""" + try: + return f'{int(x):d}' + except ValueError: + return str(x) + def default_formatter(self, x): """Universal formatter for other types.""" try: @@ -320,12 +393,14 @@ class GT(object): # non-unique index so work with position i if c in self.ratio_cols: # print(f'{i} ratio') - self._df_formatters.append(self.default_ratio) + self._df_formatters.append(self.default_ratio_formatter) + elif c in self.year_cols: + self._df_formatters.append(self.default_year_formatter) elif i in self.date_col_indices: - self._df_formatters.append(self.default_date) + self._df_formatters.append(self.default_date_formatter) elif i in self.integer_col_indices: # print(f'{i} int') - self._df_formatters.append(self.default_integer) + self._df_formatters.append(self.default_integer_formatter) elif i in self.float_col_indices: # trickier approach... self._df_formatters.append(self.make_float_formatter(self.df.iloc[:, i])) @@ -347,201 +422,231 @@ class GT(object): ratio cols like in constructor """ - # nindex = self.df.index.nlevels - # ncolumns = self.df.columns.nlevels - # ncols = self.df.shape[1] - # dt = self.df.dtypes + self.df_style = self.make_style() + self.df_html = self.make_html() + logger.info('CREATED HTML STYLE') + return self.df_style + self.df_html - nindex = self.nindex - ncolumns = self.ncolumns - ncols = self.ncols - df = self.dt + def make_style(self): + if self.debug: + head_tb = '#0ff' + body_b = '#f0f' + h0 = '#f00' + h1 = '#b00' + h2 = '#900' + bh0 = '#f00' + bh1 = '#b00' + v0 = '#0f0' + v1 = '#0a0' + v2 = '#090' + else: + head_tb = '#000' + body_b = '#000' + h0 = '#000' + h1 = '#000' + h2 = '#000' + bh0 = '#000' + bh1 = '#000' + v0 = '#000' + v1 = '#000' + v2 = '#000' + table_hrule = self.table_hrule_width + table_vrule = self.table_vrule_width + # for local use + padt, padr, padb, padl = self.padt, self.padr, self.padb, self.padl - # call pandas built-in html converter - # no escape so tex works - html = self.df.to_html(table_id=self.df_id, formatters=self.df_formatters, - index_names=False, index=False, escape=True, - border=0) - - hrule_widths = self.hrule_widths - if hrule_widths is None: - if nindex > 1: - hrule_widths = (1.5, 1.0, 0) - else: - hrule_widths = (0, 0, 0) - vrule_widths = self.vrule_widths - if vrule_widths is None: - if nindex > 1: - vrule_widths = (1.5, 1.0, 0) - else: - vrule_widths = (0, 0, 0) - - # start to build style - style = [] - style.append('\n') + +''' + return style - if len(style) > 2: - style = '\n'.join(style) + def make_html(self): + """Convert a pandas DataFrame to an HTML table with sparsification.""" + index_name_to_level = dict(zip(self.raw_df.index.names, range(self.nindex))) + index_change_level = self.index_change_level.map(index_name_to_level) + # this is easier and computed in the init + column_change_level = self.column_change_level + + # Start table + html = [f''] + if self.caption != '': + html.append(f'') + + # Process header: , allow_duplicates=True) means can create cols with the same name + bit = self.df.T.reset_index(drop=False, allow_duplicates=True) + idx_header = bit.iloc[:self.nindex, :self.ncolumns] + columns = bit.iloc[self.nindex:, :self.ncolumns] + + # TODO Add header aligners + if self.sparsify_columns: + # this is TRANSPOSED!! + html.append("") + for i in range(self.ncolumns): + # one per row of columns m index, usually only 1 + html.append("") + for j, r in enumerate(idx_header.iloc[:, i]): + # columns one per level of index + html.append(f'') + cum_col = 0 # keep track of where we are up to + for j, (nm, g) in enumerate(groupby(columns.iloc[:, i])): + hrule = f'grt-bhrule-{i}' if i < self.ncolumns - 1 else '' + colspan = sum(1 for _ in g) + if 0 < j: + vrule = f'grt-vrule-{column_change_level[cum_col]}' + elif j == 0: + # start with the first column come what may + vrule = f'grt-vrule-index' + html.append(f'') + cum_col += colspan + html.append("") + html.append("") else: - style = '' - self.df_style = style - self.df_rawhtml = html + # this is TRANSPOSED!! + html.append("") + for i in range(self.ncolumns): + # one per row of columns m index, usually only 1 + html.append("") + for j, r in enumerate(idx_header.iloc[:, i]): + # columns one per level of index + html.append(f'') + for j, r in enumerate(columns.iloc[:, i]): + # one per column of dataframe + # figure how high up mindex the vrules go + # all headings get hrules, it's the vrules that are tricky + hrule = f'grt-bhrule-{i}' if i < self.ncolumns - 1 else '' + if 0 < j < self.ncols and i >= column_change_level[j]: + vrule = f'grt-vrule-{column_change_level[j]}' + elif j == 0: + # start with the first column come what may + vrule = f'grt-vrule-index' + else: + vrule = '' + html.append(f'') + html.append("") + html.append("") - # now alter html using bs4 - soup = BeautifulSoup(html, 'html.parser') - - # center all heading rows - for r in soup.select('thead tr'): - for i, td in enumerate(r.find_all('th')): - if i < self.nindex: - td['class'] = 'gt_left gt_head' - else: - td['class'] = 'gt_center gt_head' - - # figure which index level changes for h rules - chg_level = GT.changed_level(self.df.index) - for i, r in zip(range(len(self.df)), soup.select('tbody tr')): - # row rules: if not first row and new level 0 - if i and (i in chg_level): - lv = chg_level.loc[i] - r['class'] = f'gt_ruled_row_{lv}' - # to handle sparse index you have to work from the right! - for a, td in zip(self.df_idx_aligners[::-1], r.find_all('th')[::-1]): - td['class'] = f'{a} gt_head' - # td['style'] = 'text-align: left;' - for a, td in zip(self.df_aligners, r.find_all('td')): - td['class'] = a - # find the body column - td = r.find_all('td')[-ncols] - if td is not None: - # print(f'Adding body col to {td.text}') - existing_classes = td.get('class', '') # Get existing classes as a list, or an empty list if none - td['class'] = existing_classes + ' gt_body_column' # Append new classes - - if self.caption != "": - table = soup.find('table') - if table: - c = soup.new_tag('caption') - c.string = self.caption - c['class'] = 'gt_caption' - table.insert(0, c) - - # take out dataframe - table = soup.find("table", {"class": "dataframe"}) - if table: - del table["class"] - html = soup.prettify() - self.df_html = html # after alteration - # return - logger.info('CREATED HTML STYLE') - return style + html + bold_idx = 'grt-bold' if self.font_bold_index else '' + html.append("") + for i, (n, r) in enumerate(self.df.iterrows()): + # one per row of dataframe + html.append("") + hrule = '' + for j, c in enumerate(r.iloc[:self.nindex]): + # dx = data in index + # if this is the level that changes for this row + # will use a top rule hence omit i = 0 which already has an hrule + if i > 0 and hrule == '' and j == index_change_level[i]: + hrule = f'grt-hrule-{j}' + # html.append(f'') + html.append(f'') + for j, c in enumerate(r.iloc[self.nindex:]): + # first col left handled by index/body divider + if 0 < j < self.ncols: + vrule = f'grt-vrule-{column_change_level[j]}' + elif j == 0: + # start with the first column come what may + vrule = f'grt-vrule-index' + # html.append(f'') + html.append(f'') + html.append("") + html.append("") + return '\n'.join(html) @property def html(self): - return ('' if self.df_style is None else self.df_style) + ( + code = ('' if self.df_style is None else self.df_style) + ( '' if self.df_html is None else self.df_html) + soup = BeautifulSoup(code, 'html.parser') + return soup.prettify() def _repr_latex_(self): """Generate a LaTeX tabular representation.""" - logger.info('CREATED LATEX STYLE') + # return '' # latex = self.df.to_latex(caption=self.caption, formatters=self._df_formatters) - latex = self.df_to_tikz() + latex = self.make_tikz() + logger.info('CREATED LATEX STYLE') return latex @staticmethod @@ -570,30 +675,75 @@ class GT(object): level_changes = tf.idxmax(axis=1) return level_changes - def apply_formatters(self, df): - """Replace df (the raw df) with formatted df, including the index.""" - # because of non-unique indexes, index by position not name - data_formatters = self.df_formatters[self.nindex:] - index_formatters = self.df_formatters[:self.nindex] + @staticmethod + def apply_formatters_work(df, formatters): + """Apply formatters to a DataFrame.""" + new_df = pd.DataFrame({i: map(f, df[c]) + for i, f, c in zip(range(len(df.columns)), + formatters, + df.columns)}) + new_df.columns = df.columns + return new_df - for i, f in enumerate(data_formatters): - # apply cell by cell, f not necessarily vectorized - df.iloc[:, i + self.nindex] = list(map(f, df.iloc[:, i + self.nindex])) - # df.iloc[:, i] = df.iloc[:, i].astype(object) - # adjust the index - if df.index.nlevels == 1: - df.index = map(index_formatters[0], df.index) + def apply_formatters(self, df, mode='adjusted'): + """ + Replace df (the raw df) with formatted df, including the index. + + If mode is 'adjusted' operates on columns only, does not touch the + index. Otherwise, called from tikz and operating on raw_df + """ + if mode == 'adjusted': + # apply to df where the index has been reset + # number of columns = len(self.df_formatters) + return GT.apply_formatters_work(df, self.df_formatters) + elif mode == 'raw': + # work on raw_df where the index has not been reset + # because of non-unique indexes, index by position not name + # create the df and the index separately + data_formatters = self.df_formatters[self.nindex:] + new_body = GT.apply_formatters_work(df, data_formatters) + # now create the index + index_formatters = self.df_formatters[:self.nindex] + df_index = df.reset_index(drop=False, col_level=self.df.columns.nlevels - 1).iloc[:, :self.nindex] + new_index = GT.apply_formatters_work(df_index, index_formatters) + # put them back together + new_df = pd.concat([new_index, new_body], axis=1) + new_df = new_df.set_index(list(df_index.columns)) + return new_df else: - new_levels = [list(map(func, df.index.get_level_values(i))) for i, func in - enumerate(index_formatters)] - df.index = df.index.set_levels(new_levels, verify_integrity=False) - return df + raise ValueError(f'unknown mode {mode}') - def df_to_tikz(self, float_format=None, tabs=None, - show_index=False, scale=0.635, column_sep=3 / 8, row_sep=1 / 8, - figure='figure', extra_defs='', hrule=None, equal=False, - vrule=None, post_process='', label='', caption='', latex=None, - sparsify=1, clean_index=False): + # df.index = df.index.set_levels(new_levels, verify_integrity=False) + + ## PROBLEMS + # if mode == 'adjusted': + # for i, f in enumerate(self.df_formatters): + # # apply cell by cell, f not necessarily vectorized + # df.iloc[:, i] = list(map(f, df.iloc[:, i])) + # elif mode == 'raw': + # # work on raw_df where the index has not been reset + # # because of non-unique indexes, index by position not name + # index_formatters = self.df_formatters[:self.nindex] + # data_formatters = self.df_formatters[self.nindex:] + # for i, f in enumerate(data_formatters): + # # apply cell by cell, f not necessarily vectorized + # df.iloc[:, i] = list(map(f, df.iloc[:, i])) + # # adjust the index + # if df.index.nlevels == 1: + # df.index = map(index_formatters[0], df.index) + # else: + # new_levels = [list(map(func, df.index.get_level_values(i))) for i, func in + # enumerate(index_formatters)] + # df.index = df.index.set_levels(new_levels, verify_integrity=False) + # else: + # raise ValueError(f'unknown mode {mode}') + # return df + + def make_tikz(self, float_format=None, tabs=None, + show_index=False, scale=0.635, column_sep=3 / 8, row_sep=1 / 8, + figure='figure', extra_defs='', hrule=None, equal=False, + vrule=None, post_process='', label='', caption='', latex=None, + sparsify=1, clean_index=False): """ Write DataFrame to custom tikz matrix to allow greater control of formatting and insertion of horizontal divider lines @@ -660,7 +810,7 @@ class GT(object): :return: """ # local variable - with all formatters already applied - df = self.apply_formatters(self.raw_df) + df = self.apply_formatters(self.raw_df.copy(), mode='raw') if not df.columns.is_unique: raise ValueError('tikz routine requires unique column names') # \\begin{{{figure}}}{latex} @@ -722,7 +872,7 @@ class GT(object): if hrule is None: hrule = set() for i in range(sparsify): - df.iloc[:, i], rules = GT._sparsify(df.iloc[:, i]) + df.iloc[:, i], rules = GT.sparsify(df.iloc[:, i]) # print(rules, len(rules), len(df)) # don't want lines everywhere if len(rules) < len(df) - 1: @@ -822,8 +972,8 @@ class GT(object): if isinstance(df.columns, pd.MultiIndex): for lvl in range(len(df.columns.levels)): nl = '' - sparse_columns[lvl], mi_vrules[lvl] = GT._sparsify_mi(df.columns.get_level_values(lvl), - lvl == len(df.columns.levels) - 1) + sparse_columns[lvl], mi_vrules[lvl] = GT.sparsify_mi(df.columns.get_level_values(lvl), + lvl == len(df.columns.levels) - 1) for cn, c, al in zip(df.columns, sparse_columns[lvl], align): # c = wfloat_format(c) s = f'{nl} {{cell:{ad2[al]}{colw[cn]}s}} ' @@ -944,7 +1094,7 @@ class GT(object): label = f'\\label{{tab:{label}}}' if caption == '': if label != '': - logger.warning(f'You have a label but no caption; the label {label} will be ignored.') + logger.info(f'You have a label but no caption; the label {label} will be ignored.') caption = '% caption placeholder' else: caption = f'\\caption{{{caption} {label}}}' @@ -1003,7 +1153,6 @@ class GT(object): mxmn[c] = (_.max(), _.min()) except: e = sys.exc_info()[0] - print(c, 'ERROR', e) logger.error(f'{c} error {e} DO SOMETHING ABOUT THIS...if it never occurs dont need the if') colw[c] = df[c].str.len().max() mxmn[c] = (df[c].str.len().max(), df[c].str.len().min()) @@ -1055,7 +1204,7 @@ class GT(object): return colw, mxmn, tabs @staticmethod - def _sparsify(col): + def sparsify(col): """ sparsify col values, col a pd.Series or dict, with items and accessor column results from a reset_index so has index 0,1,2... this is relied upon. @@ -1073,7 +1222,7 @@ class GT(object): return new_col, rules @staticmethod - def _sparsify_mi(mi, bottom_level=False): + def sparsify_mi(mi, bottom_level=False): """ as above for a multi index level, without the benefit of the index... really all should use this function @@ -1196,437 +1345,95 @@ class GT(object): logger.debug(f'AttributeError {e}') return str(x) -# class GT_ORIGINAL(object): + def save_html(self, fn): + """Save HTML to file.""" + p = Path(fn) + p.parent.mkdir(parents=True, exist_ok=True) + p = p.with_suffix('.html') + soup = BeautifulSoup(self.html, 'html.parser') + p.write_text(soup.prettify(), encodign='utf-8') + logger.info(f'Saved to {p}') -# def __init__(self, aligners=None, formatters=None, ratio_cols=None, -# precision=3, pef_lower=-3, pef_upper=6, font_size=0.9): -# """ -# Create a greater_tables formatting object, setting defaults. -# :param aligners: None or dict type -> left | center | right -# :param formatters: None or dict type -> format function -# """ -# # set defaults -# if aligners is None: -# aligners = {} -# _base_aligners = {'number': 'right', 'date': 'center', 'str': 'left', -# 'object': 'left'} -# self.aligner = _base_aligners.update(aligners) +class sGT(GT): + """Standard GT with Steve House-Style defaults.""" + def __init__(self, df, caption="", guess_years=True, ratio_cols_regex='lr|roe|coc', **kwargs): + """Create Steve House-Style Formatter.""" + nindex = df.index.nlevels + ncolumns = df.columns.nlevels + if ratio_cols_regex != '' and ncolumns == 1: + ratio_cols = list(df.filter(regex=ratio_cols_regex).columns) + else: + ratio_cols = None -# if formatters is None: -# formatters = {} -# _base_formatters = {int: GT._integer, float: GT._float, 'else': GT._default} -# self.formatter_cache = _base_formatters.update(formatters) + if guess_years: + year_cols = sGT.guess_years(df) + else: + year_cols = kwargs.get('year_cols', None) -# _base_ratios = ['max_LR', 'gross_LR', 'net_LR', 'ceded_LR', 'LR', -# 'COC', 'CoC', 'ROE'] -# self.ratio_cols = _base_ratios if ratio_cols is None else ratio_cols + # rule sizes + hrule_widths = (1.5, 1, 0) if nindex > 1 else None + vrule_widths = (1.5, 1, 0.5) if ncolumns > 1 else None -# self.precision = precision -# self.pef_lower = pef_lower -# self.pef_upper = pef_upper -# self._pef = None -# self.font_size = font_size + table_hrule_width = 1 if nindex == 1 else 2 + table_vrule_width = 1 if ncolumns == 1 else 2 -# # archive what is used -# self.formatter_cache = LRUCache(maxsize=10) -# self.aligner_cache = LRUCache(maxsize=10) -# self.last_id = '' -# self.last_style = None -# self.last_html = None + # padding + nr, nc = df.shape + pad_tb = 4 if nr < 10 else (2 if nr < 20 else 1) + pad_lr = 10 if nc < 4 else (5 if nc < 8 else 2) + pad = (pad_tb, pad_lr, pad_tb, pad_lr) -# @property -# def html(self): -# return self.last_style + self.last_html + font_body = 0.9 if nr < 20 else (0.8 if nr < 40 else 0.7) + font_caption = 1.1 * font_body + font_head = 1.1 * font_body -# # default formatters -# @staticmethod -# def _ratio(x): -# try: -# return f'{x:.1%}' -# except: -# return x + pef_lower = -3 + pef_upper = 6 + pef_precision = 3 -# @staticmethod -# def _date(x): -# try: -# return f'{x:%Y-%m-%d}' # f"{dt:%H:%M:%S}" -# except: -# return x + defaults = { + 'ratio_cols': ratio_cols, + 'year_cols': year_cols, + 'default_integer_str': '{x:,.0f}', + 'default_float_str': '{x:,.3f}', + 'default_date_str': '%Y-%m-%d', + 'default_ratio_str': '{x:.1%}', + 'cast_to_floats': True, + 'table_hrule_width': table_hrule_width, + 'table_vrule_width': table_vrule_width, + 'hrule_widths': hrule_widths, + 'vrule_widths': vrule_widths, + 'sparsify': True, + 'sparsify_columns': True, + 'padding_trbl': pad, + 'font_body': font_body, + 'font_head': font_head, + 'font_caption': font_caption, + 'pef_precision': pef_precision, + 'pef_lower': pef_lower, + 'pef_upper': pef_upper, + 'debug': False + } + defaults.update(kwargs) + super().__init__(df, caption=caption, **defaults) -# @staticmethod -# def _float(x): -# try: -# return f'{x:,.3f}' -# except: -# return x + @staticmethod + def guess_years(df): + """Try to guess which columns (body or index) are years. -# @staticmethod -# def _integer(x): -# try: -# return f'{x:,d}' -# except: -# return x + A column is considered a year if: + - It is numeric (integer or convertible to integer) + - All values are within a reasonable range (e.g., 1800–2100) + """ + year_columns = [] + df = df.reset_index(drop=False) + for col in df.columns: + try: + series = pd.to_numeric(df[col], errors='coerce').dropna() + if series.dtype.kind in 'iu' and series.between(1800, 2100).all(): + year_columns.append(col) + except Exception: + continue -# @staticmethod -# def _default(x): -# try: -# i = int(x) -# f = float(x) -# if i == f: -# return f'{i:,d}' -# else: -# return f'{i:,.3f}' -# except ValueError: -# return str(x) - -# def pef(self, x): -# """Pandas engineering format.""" -# if self._pef is None: -# self._pef = pd.io.formats.format.EngFormatter(accuracy=self.precision, use_eng_prefix=True) -# return self._pef(x) - -# def make_float_formatter(self, ser): -# """ -# Make a float formatter suitable for the Series ser. - -# Obeys these rules: -# * All elements in the column are formatted consistently -# * ... - -# """ -# amean = ser.abs().mean() -# # mean = ser.mean() -# # mn = ser.min() -# # mx = ser.max() -# # smallest = ser.abs().min() -# # sd = ser.sd() -# # p10, p50, p90 = np.quantile(ser, [0.1, .5, 0.9], method='inverted_cdf') -# # pl = 10. ** self.pef_lower -# # pu = 10. ** self.pef_upper -# if amean < 1: -# precision = 5 -# elif amean < 10: -# precision = 3 -# elif amean < 20000: -# precision = 2 -# else: -# precision = 0 -# fmt = f'{{x:,.{precision}f}}' -# logger.info(f'{ser.name=}, {amean=}, {fmt=}') - -# def ff(x): -# try: -# if x == int(x): -# return f'{x:,.0f}.' -# else: -# return fmt.format(x=x) -# except: -# return str(x) -# return ff - -# @property -# def last_formatter(self): # noqa -# return self.formatter_cache.get(self.last_id, 'None defined.') - -# @property -# def last_aligner(self): # noqa -# return self.aligner_cache.get(self.last_id, 'None defined.') - -# def to_html(self, df, table_id, caption='', ratio_cols=None, **kwargs): -# """ -# Convert to html with suitable number formatting. - -# Step 1 of apply format, separated so it can be called stand-alone. -# """ -# if ratio_cols is not None and np.any(df.columns.duplicated()): -# logger.warning('Ratio cols specified with non-unique column names: ignoring request.') -# else: -# if ratio_cols == 'all': -# ratio_cols = [i for i in df.columns] -# elif ratio_cols == 'default': -# ratio_cols = self.ratio_cols -# elif ratio_cols is not None and type(ratio_cols) != list: -# ratio_cols = [ratio_cols] -# # check index valid -# elif ratio_cols is None: -# ratio_cols = [] - -# # because of non-unique indexes, index by position not -# # name -# float_cols = df.select_dtypes(include=["float64", "float32", "float"]).columns -# float_cols = [j for j, i in enumerate(df.columns) if i in float_cols] -# integer_cols = df.select_dtypes(include=["int64", "int32", "int"]).columns -# integer_cols = [j for j, i in enumerate(df.columns) if i in integer_cols] -# date_cols = df.select_dtypes(include=["datetime64"]).columns -# date_cols = [j for j, i in enumerate(df.columns) if i in date_cols] - -# # because of the problem of non-unique indexes use a list and -# # not a dict to pass the formatters to to_html -# formatter_list = [] - -# for i, c in enumerate(df.columns): -# # set a default, note here can have -# # non-unique index so work with position i -# if c in ratio_cols: -# # print(f'{i} ratio') -# formatter_list.append(self._ratio) -# elif i in date_cols: -# formatter_list.append(self._date) -# elif i in integer_cols: -# # print(f'{i} int') -# formatter_list.append(self._integer) -# elif i in float_cols: -# # trickier approach... -# formatter_list.append(self.make_float_formatter(df.iloc[:, i])) -# else: -# # print(f'{i} default') -# formatter_list.append(self._default) -# # logger.info(str(formatter_list)) -# # formatter_list is now a list of length equal to cols in df -# assert len(formatter_list) == df.shape[1], f'Something wrong: {len(formatter_list)=} != {df.shape=}' -# self.formatter_cache[table_id] = formatter_list -# html = df.to_html(table_id=table_id, formatters=formatter_list, **kwargs) -# return html - -# def apply_format(self, df, caption='', hrule_widths=None, ratio_cols=None, **kwargs): -# """ -# Apply format to df. - -# ratio cols like in constructor -# """ -# self.last_id = table_id = f'T{id(df):x}'[::2].upper() - -# nindex = df.index.nlevels -# ncolumns = df.columns.nlevels -# ncols = df.shape[1] - -# dt = df.dtypes - -# html = self.to_html(df, table_id, caption=caption, ratio_cols=ratio_cols, **kwargs) - -# # for now... -# # guess: index l, numeric r rest l -# idx = 'l' * df.index.nlevels -# numeric_cols = df.select_dtypes('number').columns -# rc = ''.join('r' if c in numeric_cols else 'l' for c in df.columns) -# col_align = idx + rc - -# if hrule_widths is None: -# if nindex > 1: -# hrule_widths = (1.5, 1.0, 0.5) -# else: -# hrule_widths = (0, 0, 0) -# # start to build style -# style = [] -# style.append('\n') - -# if len(style) > 2: -# style = '\n'.join(style) -# else: -# style = '' -# self.last_style = style -# self.last_rawhtml = html - -# # now alter html using bs4 -# soup = BeautifulSoup(html, 'html.parser') - -# # center all heading rows -# for r in soup.select('thead tr'): -# for td in r.find_all('th'): -# td['class'] = 'gt_center gt_head' -# # td['style'] = 'text-align: left;' - -# # in each body row: headings left -# alignment = [] -# for y in dt.values: -# # if index is not unique this can return > 1 element -# # know in same order as df.columns, so can use -# # this approach -# if y in (int, float): -# a = 'gt_right' -# elif y == object: -# # print(c, 'object') -# a = 'gt_left' -# else: -# # print(c, type(c), 'else') -# a = 'gt_center' -# alignment.append(a) - -# idx_alignment = [] -# if nindex == 1: -# idx_alignment.append(df.index.dtype) -# else: -# for c, t in df.index.dtypes.items(): -# if t in (int, float): -# a = 'gt_right' -# elif t == object: -# a = 'gt_left' -# else: -# a = 'gt_center' -# idx_alignment.append(a) - -# # figure which index level changes for h rules -# chg_level = GT.changed_level(df) -# for i, r in zip(range(len(df)), soup.select('tbody tr')): -# # row rules: if not first row and new level 0 -# if i and (i in chg_level): -# lv = chg_level.loc[i] -# r['class'] = f'gt_ruled_row_{lv}' -# # to handle sparse index you have to work from the right! -# for a, td in zip(idx_alignment[::-1], r.find_all('th')[::-1]): -# td['class'] = f'{a} gt_head' -# # td['style'] = 'text-align: left;' -# for a, td in zip(alignment, r.find_all('td')): -# td['class'] = a -# # find the body column -# td = r.find_all('td')[-ncols] -# if td is not None: -# # print(f'Adding body col to {td.text}') -# existing_classes = td.get('class', '') # Get existing classes as a list, or an empty list if none -# td['class'] = existing_classes + ' gt_body_column' # Append new classes -# if caption != "": -# table = soup.find('table') - -# # Add a caption -# if table: -# c = soup.new_tag('caption') -# c.string = caption -# c['class'] = 'gt_caption' -# table.insert(0, c) - -# # take out dataframe -# table = soup.find("table", {"class": "dataframe"}) -# if table: -# del table["class"] - -# html = soup.prettify() -# self.last_html = html # after alteration -# # return -# return style + html - -# __call__ = apply_format - -# def apply_styles(self, html): -# """ -# Apply css styles to the base table. - -# Whine. -# """ -# pass - -# @staticmethod -# def changed_level(df): -# """ -# Return the level of index that changes with each row. - -# Very ingenious GTP code with some SM enhancements. -# """ -# idx = df.index -# idx.names = [i for i in range(idx.nlevels)] -# # Determine at which level the index changes -# index_df = idx.to_frame(index=False) # Convert MultiIndex to a DataFrame -# # true / false match last row -# tf = index_df.ne(index_df.shift()) -# # changes need at least one true -# tf = tf.loc[tf.any(axis=1)] -# level_changes = tf.idxmax(axis=1) -# return level_changes + return year_columns \ No newline at end of file diff --git a/greater_tables/utilities.py b/greater_tables/utilities.py index b43a31b..6bbd558 100644 --- a/greater_tables/utilities.py +++ b/greater_tables/utilities.py @@ -7,15 +7,9 @@ from pathlib import Path import random import sys from IPython.display import HTML, display -from docs.conf import html_theme from . greater_tables import GT -# Load a list of words -p = Path('C:\\s\\Websites\\new_mynl\\word_lists\\match 12.md') -word_list = p.read_text().split('\n') - - # GPT recommended approach logger = logging.getLogger(__name__) # Disable log propagation to prevent duplicates @@ -34,142 +28,9 @@ logger.addHandler(handler) logger.info('Logger Setup; module recompiled.') -# __gt_global = GT() - - -# def qhtml(df, **kwargs): -# """Generic "quick display" function.""" -# return HTML(__gt_global(df, **kwargs)) - - -# def qd(df, **kwargs): -# """Generic "quick display" function.""" -# if isinstance(df, pd.Series): -# if df.name is None: -# df.name = 'value' -# df = df.to_frame() -# return display(HTML(__gt_global(df, **kwargs))) - - -def create_three_level_multiindex(df): - """ - Adds two random levels to a DataFrame's column MultiIndex. - - Parameters: - df (pd.DataFrame): Input DataFrame. - - Returns: - pd.DataFrame: DataFrame with a three-level MultiIndex on the columns. - """ - n_columns = len(df.columns) - level_1 = np.random.choice(["A", "B", "C"], size=n_columns) - level_2 = np.random.choice(["X", "Y", "Z"], size=n_columns) - - # Create the MultiIndex - multi_index = pd.MultiIndex.from_tuples( - [(l1, l2, col) for l1, l2, col in zip(level_1, level_2, df.columns)], - names=["Level 1", "Level 2", df.columns.name] - ) - - # Apply the new MultiIndex to the DataFrame - df.columns = multi_index - return df - - -def test_df(date=False, mi_columns=True): - """Make a test dataframe nr rows with multi index.""" - nr = 10 - words = 'Parliament organised a year-long programme of events called Parliament in the Making to celebrate the 800th anniversary of the sealing of Magna Carta on 15 June and the 750th anniversary of the first representative parliament on 20 January Events were coordinated with Parliament Week' - words = list(set(words.split(' '))) - w1 = ['Abel', 'Cain', 'Issac', 'Fred', 'George', 'Harry', 'Ivan', 'John', 'Karl', 'Lenny', 'Moe', 'Ned', 'Otto', 'Paul', 'Quinn', 'Ralph', 'Steve', 'Tom', 'Ulysses', 'Victor', 'Walter', 'Xavier', 'Yuri', 'Zach'] - w2 = ['South', 'East', 'West', 'North', "North West", "North East", "South West", "South East", "Central", "Outer", "Inner", "Mid", "Upper", "Lower", "Far", "Near", "Middle", "Farthest", "Nearest"] - w4 = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'AA', 'BB', 'CC', 'DD', 'EE', 'FF', 'GG', 'HH', 'II', 'JJ', 'KK', 'LL', 'MM', 'NN', 'OO', 'PP', 'QQ', 'RR', 'SS', 'TT', 'UU', 'VV', 'WW', 'XX', 'YY', 'ZZ'] - rints = [0, -10, 10, 100, -100, 1000, -1000, 10000, -10000, 100000, -100000, 1000000, -1000000] - df = pd.DataFrame({'idx1': np.random.choice(w1, nr), - 'idx2': np.random.choice(w2, nr), - 'idx3': np.random.poisson(2, nr), - 'floats': np.random.rand(nr) * 3000., - 'smaller': np.random.rand(nr) * 10 ** np.linspace(-3, 4, nr), - 'larger': np.random.choice([-1., 0, 1.], nr) * np.random.rand(nr) * 10 ** np.linspace(3, 12, nr), - 'ints': np.random.poisson(20, nr), - 'powers': np.pi * 10. ** np.arange(-20, 26, 5), - 'ratios': np.random.rand(nr) * 3. - 1., - 'string': [' '.join(np.random.choice(words, 4, replace=False)) for i in range(nr)], - # 'object': [np.random.poisson(2, nr) for i in range(nr)] - }) - - if date: - df['date'] = [dt.datetime.fromordinal(np.random.randint(dt.date(2020, 1, 1).toordinal(), - dt.date(2030, 1, 1).toordinal())) + dt.timedelta(seconds=np.random.randint(86400)) for _ in range(nr)] - df['date'] = pd.to_datetime(df['date']) - df.columns.name = 'Col name' - df = df.set_index(['idx1', 'idx2', 'idx3']) - if mi_columns: - df = create_three_level_multiindex(df) - - # check unique and sort - df = df.loc[df.index[~df.index.duplicated()]] - assert np.all(~df.columns.duplicated()), 'Columns not all unique' - df = df.sort_index(axis=0).sort_index(axis=1) - return df - - -def make_test_dfs(): - """Make a dict of test dataframes with different characteristics.""" - ans = {} - df = pd.DataFrame({'x': [int(i) for i in 3. ** np.arange(10)], 'y': np.arange(10, dtype=float)}) - df['w'] = df.x ** 0.35 - df['z'] = df.y ** .25 - - ans['basic'] = df.copy() - df1 = df.copy() - df1.index.name = 'idx name' - ans['basic w idx name'] = df1.copy() - - df1 = df.copy() - df1.columns.name = 'col name' - ans['basic w col name'] = df1.copy() - - df1 = df.copy() - df1.index.name = 'idx name' - df1.columns.name = 'col name' - ans['basic w both names'] = df1.copy() - - df1['date'] = [dt.datetime.fromordinal(np.random.randint(dt.date(2020, 1, 1).toordinal(), - dt.date(2030, 1, 1).toordinal())) + dt.timedelta(seconds=np.random.randint(86400)) - for _ in range(len(df1))] - df2 = df1.set_index('date') - ans['time series'] = df2.copy() - - df2 = df1.set_index('date', append=True) - ans['time series and range'] = df2.copy() - - valid = False - while not valid: - a = ans['realistic'] = test_df(date=False, mi_columns=False) - valid = a.index.is_unique and a.columns.is_unique - - valid = False - while not valid: - a = ans['realistic w date'] = test_df(date=True, mi_columns=False).droplevel(2, axis=0) - valid = a.index.is_unique and a.columns.is_unique - - valid = False - while not valid: - a = ans['realistic mi'] = test_df(date=False, mi_columns=True).droplevel(2, axis=1) - valid = a.index.is_unique and a.columns.is_unique - - valid = False - while not valid: - a = ans['realistic mi w date'] = test_df(date=True, mi_columns=True).droplevel(2, axis=0).droplevel(2, axis=1) - valid = a.index.is_unique and a.columns.is_unique - - return ans - - - -header = ''' ---- +def write_all_tables(out_path='\\s\\telos\\pmir_studynote\\quarto_scratch\\tables.qmd'): + """Write a tester for all tables to a qmd file.""" + header = '''--- title: {title} format: html: @@ -187,6 +48,7 @@ format: import proformas as pf import greater_tables as gter +import greater_tables.utilities as gtu gter.logger.setLevel(gter.logging.WARNING) from IPython.display import display @@ -194,174 +56,112 @@ from IPython.display import display ...code build completed. -# Greater_tables - +# Greater_tables Output + ```{{python}} #| echo: true #| label: greater-tables-test -ans = gter.make_test_dfs() +test_gen = gtu.TestDFGenerator() +ans = test_gen.test_suite() ``` - + ''' - - -def write_all_tables(): - """Write a tester for all tables to a qmd file.""" - global header - template = ''' + +## Test Table {k} + ```{{python}} #| echo: fold #| label: tbl-greater-tables-test-{i} #| tbl-cap: Output for test table {k} hrw = {hrw} -gter.GT(ans['{k}'], "{title}", ratio_cols='z', aligners={{'w': 'l'}}, +f = gter.GT(ans['{k}'], "{title}", ratio_cols='z', aligners={{'w': 'l'}}, hrule_widths=hrw) +h = f._repr_html_() +print(f.df.dtypes) +h ``` -SPACER +Comments go here. ''' - ans = make_test_dfs() - out = [header.format(title='All Tables Test')] + tdf = TestDFGenerator() + ans = tdf.test_suite() + out = [header.format(title='All Tables Test - New TestDFGenerator test_suite')] for i, (k, v) in enumerate(ans.items()): if v.index.nlevels > 1: - hrw = (1.5, 0.5, 0) + hrw = (1.5, 1.0, 0.5) else: hrw = (0,0,0) out.append(template.format(i=i, k=k, hrw=hrw, title=k.title())) - p = Path('\\s\\telos\\pmir_studynote\\quarto_scratch\\tables.qmd') - p.write_text('\n'.join(out), encoding='utf-8') - - -def apply_css(f, idx): - """Apply subset of css to the GT object f""" - style = f.df_style - split_style = style.split(f'#{f.df_id}') - print(f'style has {len(split_style)} separate entries') - n = 2000 - if idx == 0: - print('RAW') - display(HTML(f.df.to_html(formatters=f.df_formatters))) - print('='*80) - if not isinstance(idx, (tuple, list)): - idx = [idx] - newbit = [] - for i in idx: - code = f'T{n+i}' - h = f.df_html.replace(f.df_id, code) - bit = split_style[i].strip() - if i == 1: - newbit.append(f'#{code} {split_style[1]}\n') - else: - newbit.append(f'#{code} {bit}') - newbit = '\n' - print(newbit) - newcode = f'{newbit}{h}' - display(HTML(newcode)) - - -def incremental_qmd(nm, css_bits=22): - """Write a qmd file with incremental tables.""" - global header - - bit = ''' - -```{{python}} -#| echo: fold -#| label: tbl-greater-tables-test-{i} -gter.apply_css(f, {i}) -``` -''' - step1 = f''' - -## GT Format - -```{{python}} -#| echo: fold -#| label: setup-01 -df = ans['{nm}'] -f = gter.GT(df, "{nm}", ratio_cols='z', aligners={{'w': 'l'}}, -hrule_widths=(1.5, 1, 0.5), vrule_widths=(1.5, 1, 0.5)) -f -``` - -# Incremental Test Suite - -''' - out = [header.format(title='Incremental Build Test'), step1] - - for i in range(css_bits): - out.append(bit.format(i=i)) - p = Path('\\s\\telos\\pmir_studynote\\quarto_scratch\\seq_build.qmd') + p = Path(out_path) p.write_text('\n'.join(out), encoding='utf-8') +# ================================================== # SUPER DOOPER test df generator with help from GPT -def make_column_names(n, g, words): - """Make n column names each g words long.""" - return [' '.join(x).title() for x in zip(*[iter(words[:n*g])] * g)] +class TestDFGenerator: + """Make excellent test DataFrames.""" + # Load a list of words + _word_list_path = 'C:\\s\\Websites\\new_mynl\\word_lists\\match 12.md' + _word_list = None + def __init__(self, nan_proportion=0.05, missing_proportion=0, + title=False, sep='_', file_path=None): + """Initialise the generator.""" + self.nan_proportion = nan_proportion + self.missing_proportion = missing_proportion + self.title = title # whether to apply title to col names + self.sep = sep # separator for column names + file_path = file_path or TestDFGenerator._word_list_path + if TestDFGenerator._word_list is None: + TestDFGenerator._word_list = TestDFGenerator.load_words(file_path) + # control datatypes + self.data_types = ["int", "float", "str", "year", "date", 'datetime'] + # types: + self.index_probs = np.array([20, 1, 20, 45, 12, 5], dtype=float) + self.index_probs /= self.index_probs.sum() + # control datatypes, types as above + self.data_type_probs = np.array([1, 2, 0.5, 0.5, 0.5, 0.5], dtype=float) + self.data_type_probs /= self.data_type_probs.sum() -def generate_test_dataframe( - num_rows=10, - num_columns=5, - num_index_levels=1, - num_column_levels=1, - column_name_length=3, - dtype_label=True, - nan_proportion=0.05, - missing_proportion=0.05, - index_types=None, - words=None -): - """ - Generate a random pandas DataFrame with diverse structures for testing. + def __repr__(self): + """Return a string representation.""" + return f"TestDFGenerator({len(self.words):,d} words)" - Parameters: - - num_rows (int): Number of rows. - - num_columns (int): Number of columns. - - num_index_levels (int): Levels in the index (1+). - - num_column_levels (int): Levels in the columns (1+). - - column_name_length (int): Words per column name. - - dtype_label (bool): Whether to tag columns with their type. - - nan_proportion (float): Proportion of NaNs. - - missing_proportion (float): Proportion of None values. - - index_types (list): List of index data types for each level. - - words (list): List of words for generating column names. + @staticmethod + def load_words(file_path): + """Load a list of words from a file.""" + p = Path(file_path) + txt = p.read_text(encoding='utf-8') + wl = txt.split('\n') + logger.info(f"Loaded wordlist.") # Debug print + return wl - Returns: - - pd.DataFrame: A test DataFrame with diverse structures. - """ - global word_list # Ensure access to global words + @property + def words(self): + """Return the word list.""" + random.shuffle(self._word_list) + return self._word_list - # Use default word list if not provided - if words is None or len(words) < num_columns: - words = word_list - random.shuffle(words) + def make_column_names(self, n, g): + """Make n column names each g words long.""" + if self.title: + return [self.sep.join(x).title() for x in zip(*[iter(self.words[:n * g])] * g)] + else: + return [self.sep.join(x) for x in zip(*[iter(self.words[:n * g])] * g)] - # Generate column names - col_names = make_column_names(num_columns, max(1, column_name_length - (1 if dtype_label else 0)), words) - - # Randomly select index data types for each level - index_dtypes = ["int", "float", "str", "date", "datetime"] - index_probs = np.array([20, 1, 20, 5, 5], dtype=float) - index_probs /= index_probs.sum() - if index_types is None: - index_types = np.random.choice(index_dtypes, num_index_levels, p=index_probs, replace=True) - if len(index_types) < num_index_levels: - # well... - index_types = (index_types * 10)[:num_index_levels] - - def generate_index_data(dtype, size): + def make_index_data(self, dtype, size): """Generate index values with natural nesting.""" if dtype == "int": values = np.random.randint(0, 100000, size=size) elif dtype == "float": values = np.random.uniform(-1e6, 1e6, size=size).round(2) elif dtype == "str": - values = np.random.choice(words, size=size) + values = np.random.choice(self.words, size=size) + elif dtype == 'year': + values = np.random.choice(np.arange(1990, 2030, dtype=int), size=size, replace=False) elif dtype == "date": start_date = datetime(2020, 1, 1) values = [start_date + timedelta(days=random.randint(-5000, 5000)) for _ in range(size)] @@ -373,67 +173,46 @@ def generate_test_dataframe( seconds=random.randint(0, 59), microseconds=random.randint(0, 999999)) for _ in range(size)] - return values + return values # noqa - def generate_multi_index(dtypes, levels, num_rows): + def make_multi_index(self, dtypes, levels, size): """Generate a MultiIndex with natural nesting.""" # lowest level of index - detailed_index = generate_index_data(dtypes[-1], num_rows) - # now make the higher levels, here we want far fewer unique values to generate repeats + detailed_index = self.make_index_data(dtypes[-1], size) + # now make the higher levels, here we want far fewer unique values to make repeats higher_levels = [] for i in range(levels - 1): # at level i have i + 2 types?? no just go with 3 - sample = generate_index_data(dtypes[i], 2 if i==0 else 3) - higher_levels.append(np.random.choice(sample, size=num_rows)) - index_names = np.random.choice(words, levels, replace=False) + sample = self.make_index_data(dtypes[i], 2 if i==0 else 3) + higher_levels.append(np.random.choice(sample, size=size)) + index_names = np.random.choice(self.words, levels, replace=False) return pd.MultiIndex.from_arrays([*higher_levels, detailed_index], names=index_names) - # Generate hierarchical MultiIndex with natural grouping - if num_index_levels > 1: - index = generate_multi_index(index_types, num_index_levels, num_rows) - else: - name = np.random.choice(words, 1)[0] - index = pd.Index(generate_index_data(index_types[0], num_rows), name=name) - - # Data types - data_types = ["int", "float", "str", "date", 'datetime'] - p = np.array([1, 2, 0.5, 0.5, 0.5], dtype=float) - p /= p.sum() - dtype_choices = np.random.choice(data_types, num_columns, p=p, replace=True) - - # Generate column structure - if num_column_levels > 1: - columns = generate_multi_index(['str'] * num_column_levels, num_column_levels, num_columns) - # don't want the index names - columns.names = [''] * num_column_levels - else: - columns = pd.Index([f"{col} {dtype}" if dtype_label else col - for col, dtype in zip(col_names, dtype_choices)], - name="Column") - - def generate_column_data(dtype): + def make_column_data(self, dtype, size): """Generate column data based on type.""" if dtype == "int": picker = np.random.rand() if picker < 0.5: - return np.random.randint(-10000, 10000, size=num_rows) + return np.random.randint(-10000, 10000, size=size) else: - return np.random.randint(0, 10**9, size=num_rows) + return np.random.randint(0, 10**9, size=size) elif dtype == "float": picker = np.random.rand() if picker < 0.4: - return 10. ** np.random.uniform(-9, 1, size=num_rows) + return 10. ** np.random.uniform(-9, 1, size=size) elif picker < 0.8: - return 10. ** np.random.uniform(-1, 10, size=num_rows) + return 10. ** np.random.uniform(-1, 10, size=size) else: - signs = np.random.choice([-1, 1], size=num_rows) - return np.pi ** np.random.uniform(-75, 75, size=num_rows) * signs + signs = np.random.choice([-1, 1], size=size) + return np.pi ** np.random.uniform(-75, 75, size=size) * signs elif dtype == "str": - return np.random.choice(words, size=num_rows) + return np.random.choice(self.words, size=size) + elif dtype == 'year': + return np.random.choice(range(1990, 2030), size=size) elif dtype == "date": start_date = datetime(2020, 1, 1) - dates = [start_date + timedelta(days=random.randint(-5000, 5000)) for _ in range(num_rows)] - return np.random.choice([d.strftime("%Y-%m-%d") for d in dates], size=num_rows) + dates = [start_date + timedelta(days=random.randint(-5000, 5000)) for _ in range(size)] + return pd.to_datetime(np.random.choice([d.strftime("%Y-%m-%d") for d in dates], size=size)) elif dtype == "datetime": start_date = datetime(2020, 1, 1) dates = [start_date + timedelta(days=random.randint(-5000, 5000), @@ -441,219 +220,124 @@ def generate_test_dataframe( minutes=random.randint(0, 59), seconds=random.randint(0, 59), microseconds=random.randint(0, 999999)) - for _ in range(num_rows)] - return np.random.choice([d.strftime("%Y-%m-%d %H:%M:%S.%f") for d in dates], size=num_rows) + for _ in range(size)] + return pd.to_datetime(np.random.choice([d.strftime("%Y-%m-%d %H:%M:%S.%f") for d in dates], size=size)) - # Generate data - data = {col: generate_column_data(dtype) for col, dtype in zip(columns, dtype_choices)} - df = pd.DataFrame(data, index=index, columns=columns) + def make_test_dataframe(self, + num_rows=10, + num_columns=5, + num_index_levels=1, + num_column_levels=1, + column_name_length=3, + dtype_label=True, + index_types=None, + title=False, + sep='_' + ): + """ + Generate a random pandas DataFrame with diverse structures for testing. - # Convert date columns to datetime dtype - for col, dtype in zip(columns, dtype_choices): - if dtype == "date": - df[col] = pd.to_datetime(df[col], errors="coerce") + Parameters: + - num_rows (int): Number of rows. + - num_columns (int): Number of columns. + - num_index_levels (int): Levels in the index (1+). + - num_column_levels (int): Levels in the columns (1+). + - column_name_length (int): Words per column name. + - dtype_label (bool): Whether to tag columns with their type. + - index_types (list): List of index data types for each level. + - words (list): List of words for generating column names. - # Introduce NaNs - num_nans = int(nan_proportion * num_rows * num_columns) - for _ in range(num_nans): - df.iat[random.randint(0, num_rows - 1), random.randint(0, num_columns - 1)] = np.nan + Returns: + - pd.DataFrame: A test DataFrame with diverse structures. + """ + # update + self.title = title + self.sep = sep + # Generate column names + col_names = self.make_column_names(num_columns, + max(1, column_name_length - (1 if dtype_label else 0))) - # Introduce radical None values - num_missing = int(missing_proportion * num_rows * num_columns) - for _ in range(num_missing): - df.iat[random.randint(0, num_rows - 1), random.randint(0, num_columns - 1)] = None - df = df.sort_index().sort_index(axis=1) - return df + # Randomly select index data types for each level + if index_types is None: + index_types = np.random.choice(self.data_types, num_index_levels, p=self.index_probs, replace=True) + if not isinstance(index_types, (tuple, list)): + index_types = [index_types] + if len(index_types) < num_index_levels: + # well... + index_types = (index_types * 10)[:num_index_levels] + # Generate hierarchical MultiIndex with natural grouping + if num_index_levels > 1: + index = self.make_multi_index(index_types, num_index_levels, num_rows) + else: + name = np.random.choice(self.words, 1)[0] + index = pd.Index(self.make_index_data(index_types[0], num_rows), name=name) -###################################### -### roll my own -def make_style(self, spacing='medium', debug=False): - if debug: - head_tb = '#0ff' - body_b = '#f0f' - h0 = '#f00' - h1 = '#b00' - h2 = '#900' - bh0 = '#f00' - bh1 = '#b00' - v0 = '#0f0' - v1 = '#0a0' - v2 = '#090' - padt, padr, padb, padl = 2, 10, 2, 10 - else: - head_tb = '#000' - body_b = '#000' - h0 = '#000' - h1 = '#000' - h2 = '#000' - bh0 = '#000' - bh1 = '#000' - v0 = '#000' - v1 = '#000' - v2 = '#000' - table_hrule = 2.5 - if spacing == 'tight': - padt, padr, padb, padl = 0, 5, 0, 5 - elif spacing == 'medium': - padt, padr, padb, padl = 2, 10, 2, 10 - elif spacing == 'loose': - padt, padr, padb, padl = 4, 15, 4, 15 - else: - raise ValueError('spacing must be tight, medium or loose') + # Data types + dtype_choices = np.random.choice(self.data_types, num_columns, p=self.data_type_probs, replace=True) - style = f''' - - ''' - return style + # Generate column structure + if num_column_levels > 1: + columns = self.make_multi_index(['str'] * num_column_levels, num_column_levels, num_columns) + # don't want the index names + columns.names = [''] * num_column_levels + else: + columns = pd.Index([f"{col} {dtype}" if dtype_label else col + for col, dtype in zip(col_names, dtype_choices)], + name="Column") + # Generate data + data = {col: self.make_column_data(dtype, num_rows) for col, dtype in zip(columns, dtype_choices)} + df = pd.DataFrame(data, index=index, columns=columns) -def df_to_html(self, spacing='medium', debug=False): - """Convert a pandas DataFrame to an HTML table with sparsification.""" - index_name_to_level = dict(zip(self.raw_df.index.names, range(self.nindex))) - index_change_level = self.index_change_level.map(index_name_to_level) - # this is easier and computed in the init - column_change_level = self.column_change_level + # Convert date columns to datetime dtype + for col, dtype in zip(columns, dtype_choices): + if dtype == "date": + df[col] = pd.to_datetime(df[col], errors="coerce") - # Start table - html = [f'
{self.caption}
{r}{nm}
{r}{r}
{c}{c}{c}{c}
'] + # Introduce NaNs + num_nans = int(self.nan_proportion * num_rows * num_columns) + for _ in range(num_nans): + df.iat[random.randint(0, num_rows - 1), random.randint(0, num_columns - 1)] = np.nan - # Process header - bit = self.df.T.reset_index(drop=False) - idx_header = bit.iloc[:self.nindex, :self.ncolumns] - columns = bit.iloc[self.nindex:, :self.ncolumns] + # Introduce radical None values + if self.missing_proportion: + num_missing = int(self.missing_proportion * num_rows * num_columns) + for _ in range(num_missing): + df.iat[random.randint(0, num_rows - 1), random.randint(0, num_columns - 1)] = None - # this is TRANSPOSED!! - html.append("") - for i in range(self.ncolumns): - # one per row of columns m index, usually only 1 - # TODO Add header aligners - html.append("") - for j, r in enumerate(idx_header.iloc[:, i]): - # columns one per level of index - html.append(f'') - for j, r in enumerate(columns.iloc[:, i]): - # one per column of dataframe - # figure how high up mindex the vrules go - # all headings get hrules, it's the vrules that are tricky - hrule = f'grt-bhrule-{i}' if i < self.ncolumns - 1 else '' - if 0 < j < self.ncols and i >= column_change_level[j]: - vrule = f'grt-vrule-{column_change_level[j]}' - elif j == 0: - # start with the first column come what may - vrule = f'grt-vrule-{column_change_level[0]}' - else: - vrule = '' - html.append(f'') - html.append("") - html.append("") + df = df.sort_index().sort_index(axis=1) + return df - html.append("") - for i, (n, r) in enumerate(self.df.iterrows()): - # one per row of dataframe - html.append("") - hrule = '' - for j, c in enumerate(r.iloc[:self.nindex]): - # dx = data in index - # if this is the level that changes for this row - # will use a top rule hence omit i = 0 which already has an hrule - if i > 0 and hrule == '' and j == index_change_level[i]: - hrule = f'grt-hrule-{j}' - html.append(f'') - for j, c in enumerate(r.iloc[self.nindex:]): - # first col left handled by index/body divider - if 0 < j < self.ncols: - vrule = f'grt-vrule-{column_change_level[j]}' - elif j == 0: - # start with the first column come what may - vrule = f'grt-vrule-{column_change_level[0]}' - html.append(f'') - html.append("") - html.append("") - return '\n'.join(html) + __call__ = make_test_dataframe + def test_suite(self): + """Make a dict of test dataframes with different characteristics.""" + ans = {} -def to_html(self, spacing='medium', debug=False): - """Full monty, raw string.""" - html = df_to_html(self, spacing=spacing, debug=debug) - style = make_style(self, spacing=spacing, debug=debug) - return style + html + ans['basic'] = self.make_test_dataframe(num_rows=10, num_columns=8, + num_index_levels=1, num_column_levels=1, + column_name_length=1, + index_types=['int']) + ans['timeseries'] = self.make_test_dataframe(num_rows=20, num_columns=3, + num_index_levels=1, num_column_levels=1, + column_name_length=4, title=True, sep=' ', + index_types=['datetime']) -def process(self, spacing='medium', debug=False): - """Full monty.""" - return HTML(to_html(self, spacing=spacing, debug=debug)) + ans['multiindex'] = self.make_test_dataframe(num_rows=10, num_columns=5, + num_index_levels=3, num_column_levels=1, + column_name_length=4, title=True, sep=' ', + index_types=['int', 'str']) + ans['multicolumns'] = self.make_test_dataframe(num_rows=10, num_columns=5, + num_index_levels=1, num_column_levels=3, + column_name_length=4, title=True, sep=' ', + index_types=['int', 'str']) + + ans['complex'] = self.make_test_dataframe(num_rows=20, num_columns=10, + num_index_levels=3, num_column_levels=3, + column_name_length=4, + index_types=['int', 'str']) + + return ans
{r}{r}
{c}{c}