categorical histogram: the unique bar re-counts singletons already shown as top values #1083

Activity

1. paddymul

   added

   Histograms

   on Oct 6, 2026
2. paddymul commented on Oct 7, 2026

   

   paddymul

   on Oct 7, 2026

   CollaboratorAuthor

   More actions

   Probe script for this issue. Save it as probe_1083.py in a buckaroo checkout and run uv run python probe_1083.py. The output below is from main at e88ace9. Each case prints the histogram from pandas (pd), polars (pl) and xorq (xo): the bar kinds, the sum of the population bars, the sum of the categorical bars and the NA bar, then each bar as name:kind=value, and below that the histogram_bins range, min, max, nan_per and distinct_count.

   probe_1083.py

   """Histogram probe: runs columns through the pandas, polars and xorq stat pipelines and prints each histogram.
   
   Each line is [bar kinds, population sum, categorical sum, NA, duplicate names] then the bars as name:kind=value.
   """
   import logging
   import warnings
   from collections import Counter
   
   logging.disable(logging.CRITICAL)
   warnings.filterwarnings('ignore')
   
   import numpy as np
   import pandas as pd
   import polars as pl
   import pyarrow as pa
   import xorq.api as xo
   
   from buckaroo.customizations.pd_stats_v2 import PD_ANALYSIS_V2
   from buckaroo.customizations.pl_stats_v2 import PL_ANALYSIS_V2
   from buckaroo.customizations.xorq_stats_v2 import XORQ_STATS_V2
   from buckaroo.pluggable_analysis_framework.stat_pipeline import StatPipeline
   from buckaroo.pluggable_analysis_framework.xorq_stat_pipeline import XorqStatPipeline
   
   rng = np.random.default_rng(0)
   
   
   def summarize(h):
       if not isinstance(h, list):
           return f'NOT-A-LIST {type(h).__name__}'
       if len(h) == 0:
           return 'EMPTY'
       kinds = Counter(k for b in h for k in b if k != 'name')
       parts = []
       for b in h:
           k = next((k for k in b if k != 'name'), '?')
           v = b.get(k)
           parts.append(f"{b['name']}:{k[:3]}={v:g}" if isinstance(v, (int, float, np.number)) else f"{b['name']}:{k}={v}")
       dups = [n for n, c in Counter(b['name'] for b in h).items() if c > 1]
       pop = sum(b.get('population', 0) for b in h)
       cat = sum(b.get(k, 0) for b in h for k in ('cat_pop', 'unique', 'longtail'))
       na = sum(b.get('NA', 0) for b in h)
       return f"[{dict(kinds)} pop={pop:g} cat={cat:g} NA={na:g}{' DUPNAMES=' + str(dups) if dups else ''}] " + ' | '.join(parts)
   
   
   def bins_s(b):
       if not b:
           return 'bins=[]'
       arr = np.asarray(b, dtype=float)
       return f'bins={len(b)} [{arr[0]:.6g}..{arr[-1]:.6g}]' + ('' if np.isfinite(arr).all() else ' NONFINITE')
   
   
   def run(backend, values, pd_dtype=None, pl_dtype=None, pa_type=None):
       if backend == 'pd':
           sd, errs = StatPipeline(PD_ANALYSIS_V2, unit_test=False).process_df(pd.DataFrame({'a': pd.Series(values, dtype=pd_dtype)}))
       elif backend == 'pl':
           sd, errs = StatPipeline(PL_ANALYSIS_V2, unit_test=False).process_df(pl.DataFrame({'a': pl.Series('a', values, dtype=pl_dtype, strict=False)}))
       else:
           table = xo.memtable(pa.table({'a': pa.array(values, type=pa_type)}), name='t')
           sd, errs = XorqStatPipeline(XORQ_STATS_V2, unit_test=False).process_table(table)
       return sd.get('a', {}), sorted({f'{e.stat_key}:{type(e.error).__name__}' for e in errs})
   
   
   def case(name, values, pd_dtype=None, pl_dtype=None, pa_type=None, backends=('pd', 'pl', 'xo')):
       print(f'\n### {name}')
       for b in backends:
           try:
               c, errs = run(b, values, pd_dtype, pl_dtype, pa_type)
           except Exception as e:
               print(f'  {b}: RAISED {type(e).__name__}: {str(e)[:150]}')
               continue
           print(f"  {b}: {summarize(c.get('histogram'))}")
           print(f"      {bins_s(c.get('histogram_bins'))} min={c.get('min')!r:.30} max={c.get('max')!r:.30} "
                 f"nan_per={c.get('nan_per')} distinct={c.get('distinct_count')}" + (f' ERRS={errs}' if errs else ''))
   
   case('counts 5, 3 then 8 singletons', ['a'] * 5 + ['b'] * 3 + [f's{i}' for i in range(8)], pa_type=pa.string())
   case('single row', [1.5], pa_type=pa.float64())
   case('all unique strings (1000)', [f'id{i}' for i in range(1000)], pa_type=pa.string())
   case('datetimes, all unique', list(pd.date_range('2024-01-01', periods=50, freq='h')), backends=('pd', 'pl'))
   case('pandas categorical with unused categories', pd.Categorical(['x', 'y', 'x'], categories=['x', 'y'] + [f'unused{i}' for i in range(10)]), backends=('pd',))

   Output at e88ace9

   ### counts 5, 3 then 8 singletons
     pd: [{'cat_pop': 7, 'unique': 1} pop=0 cat=130 NA=0] a:cat=31 | b:cat=19 | s0:cat=6 | s1:cat=6 | s2:cat=6 | s3:cat=6 | s4:cat=6 | unique:uni=50
         bins=[] min=nan max=nan nan_per=0.0 distinct=10
     pl: [{'cat_pop': 7, 'unique': 1} pop=0 cat=130 NA=0] a:cat=31 | b:cat=19 | s0:cat=6 | s1:cat=6 | s2:cat=6 | s3:cat=6 | s4:cat=6 | unique:uni=50
         bins=[] min=nan max=nan nan_per=0.0 distinct=10
     xo: [{'cat_pop': 10} pop=0 cat=99.6 NA=0] a:cat=31.2 | b:cat=18.8 | s4:cat=6.2 | s7:cat=6.2 | s0:cat=6.2 | s6:cat=6.2 | s3:cat=6.2 | s1:cat=6.2 | s2:cat=6.2 | s5:cat=6.2
         bins=[] min=None max=None nan_per=0.0 distinct=10
   
   ### single row
     pd: [{'cat_pop': 1, 'unique': 1} pop=0 cat=200 NA=0] 1.5:cat=100 | unique:uni=100
         bins=[] min=np.float64(1.5) max=np.float64(1.5) nan_per=0.0 distinct=1
     pl: [{'cat_pop': 1, 'unique': 1} pop=0 cat=200 NA=0] 1.5:cat=100 | unique:uni=100
         bins=[] min=1.5 max=1.5 nan_per=0.0 distinct=1
     xo: EMPTY
         bins=[] min=1.5 max=1.5 nan_per=0.0 distinct=None
   
   ### all unique strings (1000)
     pd: [{'unique': 1} pop=0 cat=100 NA=0] unique:uni=100
         bins=[] min=nan max=nan nan_per=0.0 distinct=1000
     pl: [{'unique': 1} pop=0 cat=100 NA=0] unique:uni=100
         bins=[] min=nan max=nan nan_per=0.0 distinct=1000
     xo: [{'cat_pop': 10} pop=0 cat=100 NA=0] id38:cat=10 | id37:cat=10 | id44:cat=10 | id47:cat=10 | id54:cat=10 | id79:cat=10 | id63:cat=10 | id82:cat=10 | id69:cat=10 | id108:cat=10
         bins=[] min=None max=None nan_per=0.0 distinct=1006
   
   ### datetimes, all unique
     pd: [{'cat_pop': 7, 'unique': 1} pop=0 cat=114 NA=0] 2024-01-01 00:00:00:cat=2 | 2024-01-02 13:00:00:cat=2 | 2024-01-02 03:00:00:cat=2 | 2024-01-02 04:00:00:cat=2 | 2024-01-02 05:00:00:cat=2 | 2024-01-02 06:00:00:cat=2 | 2024-01-02 07:00:00:cat=2 | unique:uni=100
         bins=[] min=nan max=nan nan_per=0.0 distinct=50
     pl: [{'cat_pop': 7, 'unique': 1} pop=0 cat=114 NA=0] 2024-01-01 00:00:00:cat=2 | 2024-01-01 01:00:00:cat=2 | 2024-01-01 02:00:00:cat=2 | 2024-01-01 03:00:00:cat=2 | 2024-01-01 04:00:00:cat=2 | 2024-01-01 05:00:00:cat=2 | 2024-01-01 06:00:00:cat=2 | unique:uni=100
         bins=[] min=nan max=nan nan_per=0.0 distinct=50
   
   ### pandas categorical with unused categories
     pd: [{'cat_pop': 2, 'unique': 1} pop=0 cat=133 NA=0] x:cat=67 | y:cat=33 | unique:uni=33
         bins=[] min=nan max=nan nan_per=0.0 distinct=12