xorq stats: numeric histogram leaves out empty buckets, so gaps in the data disappear #1087

Activity

1. paddymul

   added

   Histograms

   on Oct 6, 2026
2. paddymul commented on Oct 7, 2026

   

   paddymul

   on Oct 7, 2026

   CollaboratorAuthor

   More actions

   Probe script for this issue. Save it as probe_1087.py in a buckaroo checkout and run uv run python probe_1087.py. The output below is from main at e88ace9. Each case prints the histogram from pandas (pd), polars (pl) and xorq (xo): the bar kinds, the sum of the population bars, the sum of the categorical bars and the NA bar, then each bar as name:kind=value, and below that the histogram_bins range, min, max, nan_per and distinct_count.

   probe_1087.py

   """Histogram probe: runs columns through the pandas, polars and xorq stat pipelines and prints each histogram.
   
   Each line is [bar kinds, population sum, categorical sum, NA, duplicate names] then the bars as name:kind=value.
   """
   import logging
   import warnings
   from collections import Counter
   
   logging.disable(logging.CRITICAL)
   warnings.filterwarnings('ignore')
   
   import numpy as np
   import pandas as pd
   import polars as pl
   import pyarrow as pa
   import xorq.api as xo
   
   from buckaroo.customizations.pd_stats_v2 import PD_ANALYSIS_V2
   from buckaroo.customizations.pl_stats_v2 import PL_ANALYSIS_V2
   from buckaroo.customizations.xorq_stats_v2 import XORQ_STATS_V2
   from buckaroo.pluggable_analysis_framework.stat_pipeline import StatPipeline
   from buckaroo.pluggable_analysis_framework.xorq_stat_pipeline import XorqStatPipeline
   
   rng = np.random.default_rng(0)
   
   
   def summarize(h):
       if not isinstance(h, list):
           return f'NOT-A-LIST {type(h).__name__}'
       if len(h) == 0:
           return 'EMPTY'
       kinds = Counter(k for b in h for k in b if k != 'name')
       parts = []
       for b in h:
           k = next((k for k in b if k != 'name'), '?')
           v = b.get(k)
           parts.append(f"{b['name']}:{k[:3]}={v:g}" if isinstance(v, (int, float, np.number)) else f"{b['name']}:{k}={v}")
       dups = [n for n, c in Counter(b['name'] for b in h).items() if c > 1]
       pop = sum(b.get('population', 0) for b in h)
       cat = sum(b.get(k, 0) for b in h for k in ('cat_pop', 'unique', 'longtail'))
       na = sum(b.get('NA', 0) for b in h)
       return f"[{dict(kinds)} pop={pop:g} cat={cat:g} NA={na:g}{' DUPNAMES=' + str(dups) if dups else ''}] " + ' | '.join(parts)
   
   
   def bins_s(b):
       if not b:
           return 'bins=[]'
       arr = np.asarray(b, dtype=float)
       return f'bins={len(b)} [{arr[0]:.6g}..{arr[-1]:.6g}]' + ('' if np.isfinite(arr).all() else ' NONFINITE')
   
   
   def run(backend, values, pd_dtype=None, pl_dtype=None, pa_type=None):
       if backend == 'pd':
           sd, errs = StatPipeline(PD_ANALYSIS_V2, unit_test=False).process_df(pd.DataFrame({'a': pd.Series(values, dtype=pd_dtype)}))
       elif backend == 'pl':
           sd, errs = StatPipeline(PL_ANALYSIS_V2, unit_test=False).process_df(pl.DataFrame({'a': pl.Series('a', values, dtype=pl_dtype, strict=False)}))
       else:
           table = xo.memtable(pa.table({'a': pa.array(values, type=pa_type)}), name='t')
           sd, errs = XorqStatPipeline(XORQ_STATS_V2, unit_test=False).process_table(table)
       return sd.get('a', {}), sorted({f'{e.stat_key}:{type(e.error).__name__}' for e in errs})
   
   
   def case(name, values, pd_dtype=None, pl_dtype=None, pa_type=None, backends=('pd', 'pl', 'xo')):
       print(f'\n### {name}')
       for b in backends:
           try:
               c, errs = run(b, values, pd_dtype, pl_dtype, pa_type)
           except Exception as e:
               print(f'  {b}: RAISED {type(e).__name__}: {str(e)[:150]}')
               continue
           print(f"  {b}: {summarize(c.get('histogram'))}")
           print(f"      {bins_s(c.get('histogram_bins'))} min={c.get('min')!r:.30} max={c.get('max')!r:.30} "
                 f"nan_per={c.get('nan_per')} distinct={c.get('distinct_count')}" + (f' ERRS={errs}' if errs else ''))
   
   case('bimodal 0..10 and 90..100', [float(i) for i in range(11)] * 10 + [float(i) for i in range(90, 101)] * 10, pa_type=pa.float64())
   case('int64 extremes beside 0..99', [-2**63, 2**63 - 1] + list(range(100)), pd_dtype='int64', pl_dtype=pl.Int64, pa_type=pa.int64())
   case('no meat: 400 ones + 10..50', [1] * 400 + [10, 20, 30, 40, 50], pa_type=pa.int64())
   case('5 distinct floats', [0.0, 0.25, 0.5, 0.75, 1.0] * 20, pa_type=pa.float64())

   Output at e88ace9

   ### bimodal 0..10 and 90..100
     pd: [{'tail': 2, 'population': 10} pop=100 cat=0 NA=0] 0–0:tai=1 | 1–10.8:pop=50 | 10.8–20.6:pop=0 | 20.6–30.4:pop=0 | 30.4–40.2:pop=0 | 40.2–50:pop=0 | 50–59.8:pop=0 | 59.8–69.6:pop=0 | 69.6–79.4:pop=0 | 79.4–89.2:pop=0 | 89.2–99:pop=50 | 100–100:tai=1
         bins=11 [1..99] min=np.float64(0.0) max=np.float64(100.0) nan_per=0.0 distinct=22
     pl: [{'tail': 2, 'population': 10} pop=100 cat=0 NA=0] 0–0:tai=1 | 1–10.8:pop=50 | 10.8–20.6:pop=0 | 20.6–30.4:pop=0 | 30.4–40.2:pop=0 | 40.2–50:pop=0 | 50–59.8:pop=0 | 59.8–69.6:pop=0 | 69.6–79.4:pop=0 | 79.4–89.2:pop=0 | 89.2–99:pop=50 | 100–100:tai=1
         bins=11 [1..99] min=0.0 max=100.0 nan_per=0.0 distinct=22
     xo: [{'population': 3} pop=100 cat=0 NA=0] 0–10:pop=45.5 | 10–20:pop=4.5 | 90–100:pop=50
         bins=11 [0..100] min=0.0 max=100.0 nan_per=0.0 distinct=None
   
   ### int64 extremes beside 0..99
     pd: [{'tail': 2, 'population': 10} pop=98 cat=0 NA=0] 0–0:tai=1 | 1–10.7:pop=10 | 10.7–20.4:pop=10 | 20.4–30.1:pop=10 | 30.1–39.8:pop=9 | 39.8–49.5:pop=10 | 49.5–59.2:pop=10 | 59.2–68.9:pop=9 | 68.9–78.6:pop=10 | 78.6–88.3:pop=10 | 88.3–98:pop=10 | 99–9223372.04T:tai=1
         bins=11 [1..98] min=np.int64(-9223372036854775808) max=np.int64(9223372036854775807) nan_per=0.0 distinct=102
     pl: [{'tail': 2, 'population': 10} pop=98 cat=0 NA=0] -9223372.04T<>0:tai=1 | 1–10.7:pop=10 | 10.7–20.4:pop=10 | 20.4–30.1:pop=10 | 30.1–39.8:pop=9 | 39.8–49.5:pop=10 | 49.5–59.2:pop=10 | 59.2–68.9:pop=9 | 68.9–78.6:pop=10 | 78.6–88.3:pop=10 | 88.3–98:pop=10 | 99–9223372.04T:tai=1
         bins=11 [1..98] min=-9223372036854775808 max=9223372036854775807 nan_per=0.0 distinct=102
     xo: [{'population': 3} pop=100 cat=0 NA=0] -9223372T<>-7378698T:pop=1 | 0T–1844674T:pop=98 | 7378698T–9223372T:pop=1
         bins=11 [-9.22337e+18..9.22337e+18] min=-9223372036854775808 max=9223372036854775807 nan_per=0.0 distinct=102
   
   ### no meat: 400 ones + 10..50
     pd: [{'cat_pop': 1, 'unique': 1} pop=0 cat=100 NA=0] 1:cat=99 | unique:uni=1
         bins=[] min=np.int64(1) max=np.int64(50) nan_per=0.0 distinct=6
     pl: [{'cat_pop': 1, 'unique': 1} pop=0 cat=100 NA=0] 1:cat=99 | unique:uni=1
         bins=[] min=1 max=50 nan_per=0.0 distinct=6
     xo: [{'population': 6} pop=99.8 cat=0 NA=0] 1–5.9:pop=98.8 | 5.9–10.8:pop=0.2 | 15.7–20.6:pop=0.2 | 25.5–30.4:pop=0.2 | 35.3–40.2:pop=0.2 | 45.1–50:pop=0.2
         bins=11 [1..50] min=1 max=50 nan_per=0.0 distinct=6
   
   ### 5 distinct floats
     pd: [{'cat_pop': 5} pop=0 cat=100 NA=0] 0.0:cat=20 | 0.25:cat=20 | 0.5:cat=20 | 0.75:cat=20 | 1.0:cat=20
         bins=11 [0.25..0.75] min=np.float64(0.0) max=np.float64(1.0) nan_per=0.0 distinct=5
     pl: [{'cat_pop': 5} pop=0 cat=100 NA=0] 0.0:cat=20 | 0.25:cat=20 | 0.5:cat=20 | 0.75:cat=20 | 1.0:cat=20
         bins=11 [0.25..0.75] min=0.0 max=1.0 nan_per=0.0 distinct=5
     xo: [{'population': 5} pop=100 cat=0 NA=0] 0–0.1:pop=20 | 0.2–0.3:pop=20 | 0.5–0.6:pop=20 | 0.7–0.8:pop=20 | 0.9–1:pop=20
         bins=11 [0..1] min=0.0 max=1.0 nan_per=0.0 distinct=None