feat(bench): exact-delta-set row and HydraKLL in the rqe-optimizer cost export

aqpbm-cli/src/rows/exact.rs

@@ -1,7 +1,7 @@
 use super::*;
 use aqpbm_core::accuracy::aggregate::{Aggregate, AggregateGT};
 use sketch_bench::wrappers::cs_heap as csh;
-use sketch_bench::wrappers::exact::{self as ex, Cell, Increase, Max, Min, Sum};
+use sketch_bench::wrappers::exact::{self as ex, Cell, Increase, Max, Min, Present, Sum};
 use sketch_bench::wrappers::quantile_value::ToF64;
 use sketch_bench::wrappers::univmon::sketchlib as ul;
 
@@ -92,6 +92,15 @@ pub(crate) fn row_exact_increase(
     exact_row::<Increase>(req, description, table, want, Aggregate::Increase)
 }
 
+pub(crate) fn row_exact_delta_set(
+    req: &Requirement,
+    description: &TableDescription,
+    table: GeneratedTable,
+    want: &[(Operation, Metric)],
+) -> Result<Measurements, RunError> {
+    exact_row::<Present>(req, description, table, want, Aggregate::Presence)
+}
+
 // -------- more top-k: CountSketch + heap, UnivMon --------
 
 pub(crate) fn row_countsketch_heap_topk_fastpath_vector2d_lib(

aqpbm-cli/src/rows/mod.rs

@@ -153,6 +153,7 @@ fn binding(variant: &str, library: &str) -> Option<RowBinding> {
         ("exact-min", "exact") => row_exact_min,
         ("exact-max", "exact") => row_exact_max,
         ("exact-increase", "exact") => row_exact_increase,
+        ("exact-delta-set", "exact") => row_exact_delta_set,
         _ => return None,
     })
 }

aqpbm-core/src/accuracy/aggregate.rs

@@ -25,6 +25,10 @@ pub enum Aggregate {
     /// Counter increase, as ASAPQuery's `IncreaseAccumulator` answers it
     /// unbounded: last minus first, plus the value before each drop (a reset).
     Increase,
+    /// 1 for every group seen: the key set ASAPQuery's `DeltaSetAggregator`
+    /// tracks, as one value per key so a missing key is scored. Its infinite
+    /// error serializes as `null`, which `atomic-costs` skips the row for.
+    Presence,
 }
 
 impl Aggregate {
@@ -42,6 +46,7 @@ impl Aggregate {
                     .sum();
                 values.last().unwrap_or(&0.0) - values.first().unwrap_or(&0.0) + resets
             }
+            Aggregate::Presence => 1.0,
         }
     }
 }

configs/datagen/hydra_kll_disjoint_labels.yaml

@@ -0,0 +1,36 @@
+# `hydra_columns_f64.yaml` with a separate alphabet per label column, so a key1
+# value can never render the same as a key2 value (#74). On the shared alphabet
+# a key1 query also picks up the matching key2 group, and that collision shows
+# up as rank error; here the `polars` baseline scores zero and any rank error
+# is the sketch's. `export_rqe_optimizer_costs.sh` prices `hydra-kll` on it.
+
+column_num: 3
+column_label: [key1, key2, value]
+row_num: 200000
+
+column_spec:
+  # key1: the column the comparator groups by.
+  - data_type: string
+    distribution:
+      kind: uniform
+      lower_bound: 0.0
+      upper_bound: 200.0
+      seed: 1
+    string: {alphabet: "abcdefghijklm"}
+
+  # key2: a second label, skewed, so the subset fan-out is not uniform.
+  - data_type: string
+    distribution:
+      kind: zipf
+      skewness: 1.1
+      population_size: 50
+      seed: 2
+    string: {alphabet: "nopqrstuvwxyz"}
+
+  # value: KLL orders these inside each subpopulation.
+  - data_type: f64
+    distribution:
+      kind: zipf
+      skewness: 1.2
+      population_size: 1000
+      seed: 3

scripts/export_rqe_optimizer_costs.sh

@@ -41,91 +41,101 @@ CARDINALITY=100000
 SEED=42
 MERGE_SHARDS=16
 
+# The swept parameters, every one of them. Exact accumulators take none.
 HLL_PRECISIONS=(12 14)
 KLL_KS=(200 500)
 DD_ALPHAS=(0.01 0.02)
 CMS_HEAP_ROWS=(3 5)
 CMS_HEAP_COLS=(2048)
-
-# Zipf is deliberate. Uniform data makes request-rate relative error
-# meaningless in the rare-key tail and makes top-k accuracy degenerate.
-point() {
-    local variant=$1 config=$2 comparator=$3
-    echo "  $variant ($config)" >&2
+UNIVMON_CONFIGS=(
+    "heap_size=1000 sketch_row=5 sketch_col=2048 layer_size=8"
+    "heap_size=500 sketch_row=3 sketch_col=1024 layer_size=6"
+)
+HYDRA_KLL_CONFIGS=(
+    "rows=3 cols=128 cell_k=200"
+    "rows=3 cols=256 cell_k=200"
+    "rows=3 cols=512 cell_k=200"
+    "rows=5 cols=256 cell_k=200"
+    "rows=3 cols=256 cell_k=500"
+)
+
+# One row: a cost pass, then an accuracy pass, over the same data. Arguments
+# after the comparator name the data (`--dataset ...` or `--spec ...`, plus
+# `--dtype`). An empty config omits `--config` (the exact accumulators).
+measure() {
+    local variant=$1 library=$2 config=$3 comparator=$4
+    shift 4
+    local config_args=()
+    [[ -n "$config" ]] && config_args=(--config "$config")
+    echo "  $variant${config:+ ($config)}" >&2
 
     # Cost goes first. Secondary query timings accompany both passes, and
     # flattening retains the first one; this preserves the five-sample cost
     # timings that atomic-costs pairs with query throughput.
     "$BINARY" sketchbench \
-        --variant "$variant" --library lib --config "$config" \
+        --variant "$variant" --library "$library" "${config_args[@]}" \
         --operations insert,query,merge --metrics throughput,cpu,memory \
         --merge-shards "$MERGE_SHARDS" --runs "$RUNS" --warmup-runs "$WARMUP" \
-        --dataset zipf --zipf-s 1.1 --size "$SIZE" --cardinality "$CARDINALITY" \
-        --dtype i64 --seed "$SEED" --report "$RAW_JSONL"
+        "$@" --seed "$SEED" --report "$RAW_JSONL"
     "$BINARY" sketchbench \
-        --variant "$variant" --library lib --config "$config" \
+        --variant "$variant" --library "$library" "${config_args[@]}" \
         --operations query --metrics accuracy --comparator "$comparator" \
         --runs "$RUNS" --warmup-runs "$WARMUP" \
-        --dataset zipf --zipf-s 1.1 --size "$SIZE" --cardinality "$CARDINALITY" \
-        --dtype i64 --seed "$SEED" --report "$RAW_JSONL"
+        "$@" --seed "$SEED" --report "$RAW_JSONL"
 }
 
-# Exact accumulators take no --config, live under library "exact", and ingest
-# grouped records: label columns, then an i64 value. Increase reads counters.
-point_exact() {
-    local variant=$1 comparator=$2 spec=$3
-    echo "  $variant" >&2
-    "$BINARY" sketchbench \
-        --variant "$variant" --library exact \
-        --operations insert,query,merge --metrics throughput,cpu,memory \
-        --merge-shards "$MERGE_SHARDS" --runs "$RUNS" --warmup-runs "$WARMUP" \
-        --spec "$spec" --dtype i64 --seed "$SEED" --report "$RAW_JSONL"
-    "$BINARY" sketchbench \
-        --variant "$variant" --library exact \
-        --operations query --metrics accuracy --comparator "$comparator" \
-        --runs "$RUNS" --warmup-runs "$WARMUP" \
-        --spec "$spec" --dtype i64 --seed "$SEED" --report "$RAW_JSONL"
+# One-column sketches on inline data. Zipf is deliberate. Uniform data makes
+# request-rate relative error meaningless in the rare-key tail and makes top-k
+# accuracy degenerate.
+point() {
+    local variant=$1 config=$2 comparator=$3
+    measure "$variant" lib "$config" "$comparator" \
+        --dataset zipf --zipf-s 1.1 --size "$SIZE" --cardinality "$CARDINALITY" \
+        --dtype i64
 }
 
-# UnivMon consumes key/value pairs. The inline dataset path is one-column, so
-# use the tracked two-column spec whose key column is u64.
-point_univmon() {
-    local config=$1
-    echo "  univmon-cardinality ($config)" >&2
-    "$BINARY" sketchbench \
-        --variant univmon-cardinality --library lib --config "$config" \
-        --operations insert,query,merge --metrics throughput,cpu,memory \
-        --merge-shards "$MERGE_SHARDS" --runs "$RUNS" --warmup-runs "$WARMUP" \
-        --spec configs/datagen/univmon_columns.yaml --dtype u64 --seed "$SEED" \
-        --report "$RAW_JSONL"
-    "$BINARY" sketchbench \
-        --variant univmon-cardinality --library lib --config "$config" \
-        --operations query --metrics accuracy --comparator keyed-cardinality \
-        --runs "$RUNS" --warmup-runs "$WARMUP" \
-        --spec configs/datagen/univmon_columns.yaml --dtype u64 --seed "$SEED" \
-        --report "$RAW_JSONL"
+# Rows that need a multi-column spec: grouped records for the exact
+# accumulators and Hydra, key/value pairs for UnivMon cardinality.
+point_spec() {
+    local variant=$1 library=$2 config=$3 comparator=$4 spec=$5 dtype=$6
+    measure "$variant" "$library" "$config" "$comparator" --spec "$spec" --dtype "$dtype"
 }
 
-echo "==> Exact accumulators: sum, min, max, increase" >&2
-point_exact exact-sum sum-or-count configs/datagen/hydra_columns.yaml
-point_exact exact-min min configs/datagen/hydra_columns.yaml
-point_exact exact-max max configs/datagen/hydra_columns.yaml
-point_exact exact-increase rate-or-increase configs/datagen/counter_columns.yaml
-
-echo "==> Quantiles: KLL and DDSketch" >&2
+# Exact accumulators live under library "exact" and take no config. Increase
+# reads counters; the rest read grouped i64 records.
+echo "==> Exact accumulators: sum, min, max, increase, delta set" >&2
+point_spec exact-sum exact "" sum-or-count configs/datagen/hydra_columns.yaml i64
+point_spec exact-min exact "" min configs/datagen/hydra_columns.yaml i64
+point_spec exact-max exact "" max configs/datagen/hydra_columns.yaml i64
+point_spec exact-increase exact "" rate-or-increase configs/datagen/counter_columns.yaml i64
+point_spec exact-delta-set exact "" key-set configs/datagen/hydra_columns.yaml i64
+
+# HydraKLL rows are whole-sketch: one instance serves every group, and memory
+# and merge cost are not divided per group (`subpop-rank-error` reports no
+# `groups_per_instance`). The optimizer prices a row as card(G) x per-instance
+# cost, so these rows must not become candidates until #142 lands. Cost and
+# accuracy hold at this spec's group count only. The spec gives each label its
+# own alphabet, so rank error is the sketch's, not label collisions' (#74).
+echo "==> Quantiles: KLL, DDSketch, HydraKLL" >&2
 for k in "${KLL_KS[@]}"; do
     point kll-percall "k=$k" rank-error
 done
 for alpha in "${DD_ALPHAS[@]}"; do
     point dd "alpha=$alpha" rank-error
 done
+for config in "${HYDRA_KLL_CONFIGS[@]}"; do
+    point_spec hydra-kll lib "$config" subpop-rank-error \
+        configs/datagen/hydra_kll_disjoint_labels.yaml f64
+done
 
 echo "==> Cardinality: HLL and UnivMon" >&2
 for lg_k in "${HLL_PRECISIONS[@]}"; do
     point hll "lg_k=$lg_k" cardinality
 done
-point_univmon "heap_size=1000 sketch_row=5 sketch_col=2048 layer_size=8"
-point_univmon "heap_size=500 sketch_row=3 sketch_col=1024 layer_size=6"
+for config in "${UNIVMON_CONFIGS[@]}"; do
+    point_spec univmon-cardinality lib "$config" keyed-cardinality \
+        configs/datagen/univmon_columns.yaml u64
+done
 
 echo "==> Top-k: CMS heap, CountSketch heap, UnivMon" >&2
 for rows in "${CMS_HEAP_ROWS[@]}"; do
@@ -134,13 +144,21 @@ for rows in "${CMS_HEAP_ROWS[@]}"; do
         point countsketch-heap-topk-fastpath-vector2d "rows=$rows cols=$cols" topk
     done
 done
-point univmon-topk "heap_size=1000 sketch_row=5 sketch_col=2048 layer_size=8" topk
-point univmon-topk "heap_size=500 sketch_row=3 sketch_col=1024 layer_size=6" topk
+for config in "${UNIVMON_CONFIGS[@]}"; do
+    point univmon-topk "$config" topk
+done
 
 echo "==> Flattening cost + accuracy passes..." >&2
 "$BINARY" flatten "$RAW_JSONL" --output "$GRID_JSONL"
 
-echo "==> Reducing to atomic-cost table (expect 18 row(s), 0 skipped)..." >&2
-"$BINARY" atomic-costs "$GRID_JSONL" --output "$TABLE_JSON"
+echo "==> Reducing to atomic-cost table (expect 24 row(s), 0 skipped)..." >&2
+# A skipped row is a failed measurement, e.g. an exact row that missed a group
+# scores an infinite error, which JSON holds as null. Fail rather than publish
+# a table that is silently short.
+summary=$("$BINARY" atomic-costs "$GRID_JSONL" --output "$TABLE_JSON" 2>&1 | tee /dev/stderr | tail -n 1)
+if [[ "$summary" != *", 0 skipped" ]]; then
+    echo "ERROR: atomic-costs skipped rows; see the reasons above." >&2
+    exit 1
+fi
 
 echo "Done. $TABLE_JSON" >&2

sketch-bench/src/registry.rs

@@ -1003,6 +1003,22 @@ pub const REGISTRY: &[SketchId] = &[
             .union(MetricsMask::MEMORY)
             .union(MetricsMask::ACCURACY),
     },
+    SketchId {
+        algorithm: "exact",
+        variant: "exact-delta-set",
+        library: "exact",
+        description: "key set per window, ASAPQuery DeltaSetAggregator (no churn: added only)",
+        capability: Capability::KeySet,
+        comparator: Some("key-set"),
+        operations: OperationMask::INSERT
+            .union(OperationMask::QUERY)
+            .union(OperationMask::MERGE),
+        metrics: MetricsMask::THROUGHPUT
+            .union(MetricsMask::LATENCY)
+            .union(MetricsMask::CPU)
+            .union(MetricsMask::MEMORY)
+            .union(MetricsMask::ACCURACY),
+    },
 ];
 
 // ---------- what the frontend asks ----------
@@ -1353,6 +1369,7 @@ mod tests {
             "min",
             "max",
             "rate-or-increase",
+            "key-set",
         ];
         for e in REGISTRY {
             if let Some(name) = e.comparator {

sketch-bench/src/request.rs

@@ -27,6 +27,7 @@ pub enum Capability {
     Min,
     Max,
     RateOrIncrease,
+    KeySet,
     #[default]
     None,
 }
@@ -54,6 +55,7 @@ impl Capability {
             Capability::Min => "min",
             Capability::Max => "max",
             Capability::RateOrIncrease => "rate-or-increase",
+            Capability::KeySet => "key-set",
             Capability::None => "none",
         }
     }

sketch-bench/src/wrappers/exact.rs

@@ -184,6 +184,23 @@ impl Cell for Increase {
     }
 }
 
+/// `DeltaSetAggregatorAccumulator`'s `added` set: a key and nothing else.
+/// Keys are never removed here (no churn between windows), so `removed`
+/// stays empty and the chronological merge is a union.
+#[derive(Clone)]
+pub struct Present;
+
+impl Cell for Present {
+    fn first(_v: f64, _ts: i64) -> Self {
+        Present
+    }
+    fn insert(&mut self, _v: f64, _ts: i64) {}
+    fn merge(&mut self, _other: &Self) {}
+    fn value(&self) -> f64 {
+        1.0
+    }
+}
+
 /// One accumulator holding every group.
 pub struct Multiple<C> {
     by_group: HashMap<Vec<String>, C>,