hydra: KLL CDF and UnivMon sum rows; KLL cell merge buffer in the footprint

README.md

@@ -451,7 +451,10 @@ The file can be found [under configs directory](./configs/datagen/hydra_columns.
 
 **`--group-columns`** (Hydra rows only): which label columns the subpopulation is taken over, as comma-separated indices into the columns before the value column, e.g. `1` or `0,1`; default `0`.
 Any non-empty subset works, since the grid stores every subset of a record's labels; it picks the one the `hydra-*` row (`lib` and `polars`) asks and scores.
-With `--metrics accuracy`, the record also summarises each scored group's error in the row's own metric (relative error; mean rank error for `hydra-kll`): `err_mean`, `err_p50`, `err_p90`, `err_max`, `groups_scored`, plus `schema_width` (`d`), `records` (`N`) and `fanned_mass` (`N·(2^d − 1)`).
+With `--metrics accuracy`, the record also summarises each scored group's error in the row's own metric (relative error; mean rank error for `hydra-kll`; mean absolute CDF error for `hydra-kll-cdf`): `err_mean`, `err_p50`, `err_p90`, `err_max`, `groups_scored`, plus `schema_width` (`d`), `records` (`N`) and `fanned_mass` (`N·(2^d − 1)`).
+
+Two Hydra rows ask a second statistic of an existing cell: `hydra-kll-cdf` asks the KLL grid `Cdf(x)`, the share of a group at or below `x`, at each group's own values at `phi` = 0.01..0.99; `hydra-univmon-sum` inserts each value counted by itself (i64 or u64, within an `i32` count), so the UnivMon grid's L1 is the group's sum.
+Both have a `polars` exact baseline.
 
 **`--per-group-out PATH`** (Hydra rows only): write those per-group errors as a CSV, `group_key,n_q,error`, one row per group (`n_q` is its record count, `group_key` its tagged labels such as `label0:a;label1:x`, with `\`, `:` and `;` in a label escaped by `\`).
 Every group is listed, so the `n_q` sum to the stream's records; a group that cannot be scored (its true statistic is zero, e.g. zero entropy) has an empty `error` and is left out of the `err_*` summary.

aqpbm-cli/src/rows/hydra.rs

@@ -253,3 +253,65 @@ pub(crate) fn row_hydra_kll_polars(
         other => Err(no_build_at(req, other)),
     }
 }
+
+pub(crate) fn row_hydra_kll_cdf_lib(
+    req: &Requirement,
+    description: &TableDescription,
+    table: GeneratedTable,
+    want: &[(Operation, Metric)],
+) -> Result<Measurements, RunError> {
+    macro_rules! at {
+        ($t:ty) => {
+            subpop_cdf_row::<$t>(
+                req,
+                description,
+                table,
+                want,
+                hks::insert_hydra_kll::<$t>,
+                hks::insert_step_hydra_kll::<$t>,
+                hks::query_hydra_kll_cdf::<$t>,
+                Some((
+                    hks::merge_hydra_kll::<$t>,
+                    hks::merge_step_hydra_kll::<$t>,
+                    hks::merge_query_hydra_kll_cdf::<$t>,
+                )),
+                None,
+            )
+        };
+    }
+    match req.width {
+        Dtype::I64 => at!(i64),
+        Dtype::U64 => at!(u64),
+        Dtype::F64 => at!(f64),
+        other => Err(no_build_at(req, other)),
+    }
+}
+
+pub(crate) fn row_hydra_kll_cdf_polars(
+    req: &Requirement,
+    description: &TableDescription,
+    table: GeneratedTable,
+    want: &[(Operation, Metric)],
+) -> Result<Measurements, RunError> {
+    macro_rules! at {
+        ($t:ty) => {
+            subpop_cdf_row::<$t>(
+                req,
+                description,
+                table,
+                want,
+                hkp::insert_polars_subpop_quantile::<$t>,
+                hkp::insert_step_polars_subpop_quantile::<$t>,
+                hkp::query_polars_subpop_cdf::<$t>,
+                None,
+                Some(hkp::prepare_polars_subpop_quantile::<$t>),
+            )
+        };
+    }
+    match req.width {
+        Dtype::I64 => at!(i64),
+        Dtype::U64 => at!(u64),
+        Dtype::F64 => at!(f64),
+        other => Err(no_build_at(req, other)),
+    }
+}

aqpbm-cli/src/rows/hydra_univmon.rs

@@ -272,3 +272,83 @@ pub(crate) fn row_hydra_univmon_entropy_polars(
         Dtype::Str => at!(String),
     }
 }
+
+/// The sum rows insert each value as its own integer count, so they build at
+/// the integer widths only.
+fn no_sum_at(req: &Requirement, got: Dtype) -> RunError {
+    RunError::Sketch(format!(
+        "{}/{} counts each value by itself, an integer weight, so builds at i64 or u64; \
+         --dtype {} is not an integer",
+        req.variant,
+        req.library,
+        got.name(),
+    ))
+}
+
+pub(crate) fn row_hydra_univmon_sum_lib(
+    req: &Requirement,
+    description: &TableDescription,
+    table: GeneratedTable,
+    want: &[(Operation, Metric)],
+) -> Result<Measurements, RunError> {
+    macro_rules! at {
+        ($t:ty) => {
+            subpop_vector_row::<$t, _>(
+                req,
+                description,
+                table,
+                want,
+                SubpopSumGT {
+                    group_columns: group_columns(req, description)?,
+                    value_column: value_column(description),
+                },
+                hus::insert_hydra_univmon_sum::<$t>,
+                hus::insert_step_hydra_univmon_sum::<$t>,
+                hus::query_hydra_univmon_sum::<$t>,
+                Some((
+                    hus::merge_hydra_univmon_sum::<$t>,
+                    hus::merge_step_hydra_univmon_sum::<$t>,
+                    hus::merge_query_hydra_univmon_sum::<$t>,
+                )),
+                None,
+            )
+        };
+    }
+    match req.width {
+        Dtype::I64 => at!(i64),
+        Dtype::U64 => at!(u64),
+        other => Err(no_sum_at(req, other)),
+    }
+}
+
+pub(crate) fn row_hydra_univmon_sum_polars(
+    req: &Requirement,
+    description: &TableDescription,
+    table: GeneratedTable,
+    want: &[(Operation, Metric)],
+) -> Result<Measurements, RunError> {
+    macro_rules! at {
+        ($t:ty) => {
+            subpop_vector_row::<$t, _>(
+                req,
+                description,
+                table,
+                want,
+                SubpopSumGT {
+                    group_columns: group_columns(req, description)?,
+                    value_column: value_column(description),
+                },
+                hup::insert_polars_subpop_sum::<$t>,
+                hup::insert_step_polars_subpop_sum::<$t>,
+                hup::query_polars_subpop_sum::<$t>,
+                None,
+                Some(hup::prepare_polars_subpop_sum::<$t>),
+            )
+        };
+    }
+    match req.width {
+        Dtype::I64 => at!(i64),
+        Dtype::U64 => at!(u64),
+        other => Err(no_sum_at(req, other)),
+    }
+}

aqpbm-cli/src/rows/mod.rs

@@ -23,8 +23,8 @@ use aqpbm_core::accuracy::keyed::{
 };
 use aqpbm_core::accuracy::quantile::{RankErrorGT, RelativeValueErrorGT};
 use aqpbm_core::accuracy::subpopulation::{
-    GroupKey, SubpopCardinalityGT, SubpopEntropyGT, SubpopFrequencyGT, SubpopL1NormGT,
-    SubpopL2NormGT, SubpopRankErrorGT,
+    GroupKey, SubpopCardinalityGT, SubpopCdfErrorGT, SubpopEntropyGT, SubpopFrequencyGT,
+    SubpopL1NormGT, SubpopL2NormGT, SubpopRankErrorGT, SubpopSumGT,
 };
 use aqpbm_core::accuracy::topk::TopkGT;
 use aqpbm_core::accuracy::Score;
@@ -127,6 +127,8 @@ fn binding(variant: &str, library: &str) -> Option<RowBinding> {
         ("hydra-hll", "polars") => row_hydra_hll_polars,
         ("hydra-kll", "lib") => row_hydra_kll_lib,
         ("hydra-kll", "polars") => row_hydra_kll_polars,
+        ("hydra-kll-cdf", "lib") => row_hydra_kll_cdf_lib,
+        ("hydra-kll-cdf", "polars") => row_hydra_kll_cdf_polars,
         ("hydra-univmon-cardinality", "lib") => row_hydra_univmon_cardinality_lib,
         ("hydra-univmon-cardinality", "polars") => row_hydra_univmon_cardinality_polars,
         ("hydra-univmon-l1-norm", "lib") => row_hydra_univmon_l1_norm_lib,
@@ -135,6 +137,8 @@ fn binding(variant: &str, library: &str) -> Option<RowBinding> {
         ("hydra-univmon-l2-norm", "polars") => row_hydra_univmon_l2_norm_polars,
         ("hydra-univmon-entropy", "lib") => row_hydra_univmon_entropy_lib,
         ("hydra-univmon-entropy", "polars") => row_hydra_univmon_entropy_polars,
+        ("hydra-univmon-sum", "lib") => row_hydra_univmon_sum_lib,
+        ("hydra-univmon-sum", "polars") => row_hydra_univmon_sum_polars,
         ("univmon-cardinality", "lib") => row_univmon_cardinality_lib,
         ("univmon-l1-norm", "lib") => row_univmon_l1_norm_lib,
         ("univmon-l1-norm", "oxide") => row_univmon_l1_norm_oxide,

aqpbm-cli/src/rows/statistic.rs

@@ -259,6 +259,38 @@ pub(super) fn subpop_quantile_row<V: ColumnItem + 'static>(
     )
 }
 
+/// A row answering **subpopulation CDF**: the share of a group at or below a
+/// value, scored in absolute CDF error.
+#[allow(clippy::too_many_arguments, clippy::type_complexity)]
+pub(super) fn subpop_cdf_row<V: ColumnItem + 'static>(
+    req: &Requirement,
+    description: &TableDescription,
+    table: GeneratedTable,
+    want: &[(Operation, Metric)],
+    insert: InsertBody<(String, V)>,
+    insert_step: InsertStepBody<(String, V)>,
+    query: QueryBody<(String, V), (GroupKey, f64), f64>,
+    merge: Option<Folds<(String, V), (GroupKey, f64), f64>>,
+    prepare: Option<PrepareBody<(String, V)>>,
+) -> Result<Measurements, RunError> {
+    scored_row(
+        req,
+        description,
+        table,
+        want,
+        SubpopCdfErrorGT {
+            group_columns: group_columns(req, description)?,
+            value_column: value_column(description),
+        },
+        peel_labeled::<V>,
+        insert,
+        insert_step,
+        query,
+        merge,
+        prepare,
+    )
+}
+
 #[allow(clippy::too_many_arguments)]
 pub(super) fn keyed_row<K: ColumnItem, G>(
     req: &Requirement,

aqpbm-cli/tests/hydra_grouping.rs

@@ -74,11 +74,17 @@ fn any_label_subset_is_asked_where_it_was_inserted() {
         ("hydra-cs", "rows=3 cols=1024 cell_rows=3 cell_cols=256", "i64"),
         ("hydra-hll", "rows=3 cols=1024", "i64"),
         ("hydra-kll", "rows=3 cols=1024 cell_k=4096", "f64"),
+        ("hydra-kll-cdf", "rows=3 cols=1024 cell_k=4096", "f64"),
         (
             "hydra-univmon-l1-norm",
             "rows=3 cols=1024 cell_heap_size=64 cell_sketch_row=3 cell_sketch_col=256 cell_layer_size=4",
             "i64",
         ),
+        (
+            "hydra-univmon-sum",
+            "rows=3 cols=1024 cell_heap_size=64 cell_sketch_row=3 cell_sketch_col=256 cell_layer_size=4",
+            "i64",
+        ),
     ];
     for (variant, config, dtype) in rows {
         for group_columns in ["1", "0,1"] {
@@ -161,3 +167,26 @@ fn per_group_out_without_accuracy_is_refused() {
     assert!(err.contains("none was asked for"), "{err}");
     assert!(!csv.exists(), "no file is left behind");
 }
+
+/// The sum rows count each value by itself, an integer weight: an `f64`
+/// value column is refused by name, at both libraries.
+#[test]
+fn a_float_sum_is_refused_by_name() {
+    let spec = spec("f64");
+    for library in ["lib", "polars"] {
+        let out = Command::new(env!("CARGO_BIN_EXE_approxbench"))
+            .args(["sketchbench", "--variant", "hydra-univmon-sum"])
+            .args(["--library", library, "--dtype", "f64"])
+            .args(["--spec", spec.to_str().expect("utf-8 path")])
+            .args(["--operations", "query", "--metrics", "accuracy"])
+            .args(["--runs", "1", "--warmup-runs", "0"])
+            .output()
+            .expect("approxbench runs");
+        assert!(!out.status.success(), "{library}");
+        let err = String::from_utf8_lossy(&out.stderr);
+        assert!(
+            err.contains("f64") && err.contains("integer"),
+            "{library}: {err}"
+        );
+    }
+}