feat(index): per-query index cache hit/miss stats

java/lance-jni/src/blocking_scanner.rs

@@ -637,7 +637,7 @@ fn inner_count_rows(env: &mut JNIEnv, j_scanner: JObject) -> Result<u64> {
 }
 
 const SCAN_STATS_CLASS: &str = "org/lance/ipc/ScanStats";
-const SCAN_STATS_CONSTRUCTOR_SIG: &str = "(JJJJJJLjava/util/Map;Ljava/util/Map;)V";
+const SCAN_STATS_CONSTRUCTOR_SIG: &str = "(JJJJJJJJLjava/util/Map;Ljava/util/Map;)V";
 
 fn export_usize_map<'a>(env: &mut JNIEnv<'a>, map: &HashMap<String, usize>) -> Result<JObject<'a>> {
     let hash_map = env.new_object("java/util/HashMap", "()V", &[])?;
@@ -669,6 +669,8 @@ impl IntoJava for &ExecutionSummaryCounts {
                 JValueGen::Long(self.indices_loaded as i64),
                 JValueGen::Long(self.parts_loaded as i64),
                 JValueGen::Long(self.index_comparisons as i64),
+                JValueGen::Long(self.index_cache_hits() as i64),
+                JValueGen::Long(self.index_cache_misses() as i64),
                 JValueGen::Object(&all_counts),
                 JValueGen::Object(&all_times),
             ],

java/src/main/java/org/lance/ipc/ScanStats.java

@@ -30,6 +30,13 @@ public final class ScanStats {
   private final long indicesLoaded;
   private final long partsLoaded;
   private final long indexComparisons;
+
+  /** Number of index cache page lookups served from memory in this scan. */
+  private final long indexCacheHits;
+
+  /** Number of index cache page lookups that had to load from storage in this scan. */
+  private final long indexCacheMisses;
+
   private final Map<String, Long> allCounts;
   private final Map<String, Long> allTimes;
 
@@ -40,6 +47,8 @@ public ScanStats(
       long indicesLoaded,
       long partsLoaded,
       long indexComparisons,
+      long indexCacheHits,
+      long indexCacheMisses,
       Map<String, Long> allCounts,
       Map<String, Long> allTimes) {
     this.iops = iops;
@@ -48,10 +57,42 @@ public ScanStats(
     this.indicesLoaded = indicesLoaded;
     this.partsLoaded = partsLoaded;
     this.indexComparisons = indexComparisons;
+    this.indexCacheHits = indexCacheHits;
+    this.indexCacheMisses = indexCacheMisses;
     this.allCounts = freezeMap(allCounts);
     this.allTimes = freezeMap(allTimes);
   }
 
+  /**
+   * Backwards-compatible constructor kept for existing callers that predate the addition of
+   * per-query index cache statistics. New code should use the 10-argument constructor that also
+   * accepts {@code indexCacheHits} and {@code indexCacheMisses}.
+   *
+   * @deprecated Use {@link #ScanStats(long, long, long, long, long, long, long, long, Map, Map)}.
+   */
+  @Deprecated
+  public ScanStats(
+      long iops,
+      long requests,
+      long bytesRead,
+      long indicesLoaded,
+      long partsLoaded,
+      long indexComparisons,
+      Map<String, Long> allCounts,
+      Map<String, Long> allTimes) {
+    this(
+        iops,
+        requests,
+        bytesRead,
+        indicesLoaded,
+        partsLoaded,
+        indexComparisons,
+        0L,
+        0L,
+        allCounts,
+        allTimes);
+  }
+
   private static <K, V> Map<K, V> freezeMap(Map<K, V> map) {
     if (map == null || map.isEmpty()) {
       return Collections.emptyMap();
@@ -83,6 +124,41 @@ public long getIndexComparisons() {
     return indexComparisons;
   }
 
+  /**
+   * Number of index cache page lookups where the loader was not executed in this scan.
+   *
+   * <p>Counts both true cache hits on already-populated entries and coalesced concurrent loads (a
+   * follower attached to another caller's in-flight load).
+   *
+   * <p>Instrumented boundaries in this release: BTree, IVF v2 (write-cache scan path), inverted
+   * posting list (grouped and per-token) and its per-token metadata, inverted phrase positions,
+   * bitmap (Equals / Range / IsIn), ngram, rtree.
+   *
+   * <p>Caveats:
+   *
+   * <ul>
+   *   <li>IVF v2 streaming scans and legacy v1 IVF partitions bypass the cache by design and are
+   *       therefore reported as a miss on every call.
+   *   <li>A cold posting-list lookup on the grouped inverted layout can record up to two misses
+   *       (group + per-token metadata) for a single term.
+   * </ul>
+   *
+   * <p>Uninstrumented paths (HNSW graph pages, quantizer codebooks) do not contribute to either
+   * counter. See the sibling {@link #getIndexCacheMisses()} for the paired counter.
+   */
+  public long getIndexCacheHits() {
+    return indexCacheHits;
+  }
+
+  /**
+   * Number of index cache page lookups where the loader ran in this scan (the page was not resident
+   * and had to be materialised, typically from storage). See {@link #getIndexCacheHits()} for the
+   * paired counter and the list of instrumented boundaries.
+   */
+  public long getIndexCacheMisses() {
+    return indexCacheMisses;
+  }
+
   public Map<String, Long> getAllCounts() {
     return allCounts;
   }
@@ -106,6 +182,8 @@ public boolean equals(Object o) {
         && indicesLoaded == that.indicesLoaded
         && partsLoaded == that.partsLoaded
         && indexComparisons == that.indexComparisons
+        && indexCacheHits == that.indexCacheHits
+        && indexCacheMisses == that.indexCacheMisses
         && Objects.equals(allCounts, that.allCounts)
         && Objects.equals(allTimes, that.allTimes);
   }
@@ -119,6 +197,8 @@ public int hashCode() {
         indicesLoaded,
         partsLoaded,
         indexComparisons,
+        indexCacheHits,
+        indexCacheMisses,
         allCounts,
         allTimes);
   }
@@ -138,6 +218,10 @@ public String toString() {
         + partsLoaded
         + ", indexComparisons="
         + indexComparisons
+        + ", indexCacheHits="
+        + indexCacheHits
+        + ", indexCacheMisses="
+        + indexCacheMisses
         + ", allCounts="
         + allCounts
         + ", allTimes="

java/src/test/java/org/lance/ScannerTest.java

@@ -212,6 +212,10 @@ void testDatasetScannerStats(@TempDir Path tempDir) throws Exception {
           assertTrue(statsOpt.isPresent());
           ScanStats stats = statsOpt.get();
           assertTrue(stats.getBytesRead() > 0 || !stats.getAllCounts().isEmpty());
+          // Even without an index on this dataset, the two new counters must
+          // still marshal through JNI and default to zero rather than throwing.
+          assertTrue(stats.getIndexCacheHits() >= 0);
+          assertTrue(stats.getIndexCacheMisses() >= 0);
         }
       }
     }

python/python/lance/lance/__init__.pyi

@@ -892,6 +892,31 @@ class ScanStatistics:
     indices_loaded: int
     parts_loaded: int
     index_comparisons: int
+    index_cache_hits: int
+    """Number of index cache page lookups where the loader was not executed
+    in this scan. Counts both true cache hits on already-populated entries
+    and coalesced concurrent loads (a follower attached to another caller's
+    in-flight load).
+
+    Instrumented boundaries in this release: BTree, IVF v2 (write-cache scan
+    path), inverted posting list (grouped and per-token) and its per-token
+    metadata, inverted phrase positions, bitmap (Equals / Range / IsIn),
+    ngram, rtree.
+
+    Caveats:
+
+    * IVF v2 streaming scans and legacy v1 IVF partitions bypass the cache
+      by design and are therefore reported as a miss on every call.
+    * A cold posting-list lookup on the grouped inverted layout can record
+      up to two misses (group + per-token metadata) for a single term.
+
+    Uninstrumented paths (HNSW graph pages, quantizer codebooks) do not
+    contribute to either counter."""
+    index_cache_misses: int
+    """Number of index cache page lookups where the loader ran (the page was
+    not resident and had to be materialised, typically from storage). See
+    the sibling ``index_cache_hits`` for the paired counter and the list of
+    instrumented boundaries."""
     all_counts: Dict[
         str, int
     ]  # Additional metrics for debugging purposes. Subject to change.

python/python/tests/test_scalar_index.py

@@ -3346,6 +3346,166 @@ def scan_stats_callback(stats: lance.ScanStatistics):
     assert scan_stats.parts_loaded == 0
 
 
+def test_btree_index_cache_hit_miss_stats(tmp_path: Path):
+    """Cold scan reports index cache misses; warm scan reports hits.
+
+    ScanStatistics.index_cache_{hits,misses} are populated at page-level cache
+    boundaries. On a freshly-loaded dataset the BTree page fetch must be a
+    miss; a second scan against the same in-memory Dataset re-uses the cached
+    page and therefore reports a hit with zero misses.
+    """
+    scan_stats = None
+
+    def scan_stats_callback(stats: lance.ScanStatistics):
+        nonlocal scan_stats
+        scan_stats = stats
+
+    test_table = pa.table({"val": list(range(1000))})
+    ds = lance.write_dataset(test_table, tmp_path)
+    ds.create_scalar_index("val", index_type="BTREE")
+
+    # Reopen so the session cache starts cold. A single-key point lookup on
+    # a small dataset resolves to exactly one BTree page, so cold/warm counts
+    # are deterministic 1/0 and 0/1.
+    ds = lance.dataset(tmp_path)
+    ds.scanner(filter="val = 42", scan_stats_callback=scan_stats_callback).to_table()
+    assert scan_stats is not None
+    assert scan_stats.index_cache_misses == 1
+    assert scan_stats.index_cache_hits == 0
+
+    # Same Dataset, warm cache — no new page loads, only hits.
+    ds.scanner(filter="val = 42", scan_stats_callback=scan_stats_callback).to_table()
+    assert scan_stats.index_cache_hits == 1
+    assert scan_stats.index_cache_misses == 0
+
+
+def test_bitmap_index_cache_hit_miss_stats(tmp_path: Path):
+    """Bitmap Range/IN queries report cold misses and warm hits; a value
+    that is not in the index never reaches the loader and must not count.
+
+    Guards against the ``BitmapIndex::search`` regressions where the
+    ``Range`` / ``IsIn`` branches used to drop the ``MetricsCollector`` (so
+    every lookup was silently ``0/0``), and where an equality on a value
+    absent from ``index_map`` recorded a spurious miss before short-circuiting
+    to the empty result.
+    """
+    scan_stats = None
+
+    def scan_stats_callback(stats: lance.ScanStatistics):
+        nonlocal scan_stats
+        scan_stats = stats
+
+    test_table = pa.table({"color": ["red", "green", "blue", "yellow"] * 25})
+    ds = lance.write_dataset(test_table, tmp_path)
+    ds.create_scalar_index("color", index_type="BITMAP")
+
+    # Reopen so the session cache starts cold.
+    ds = lance.dataset(tmp_path)
+    ds.scanner(
+        filter="color IN ('red', 'blue')", scan_stats_callback=scan_stats_callback
+    ).to_table()
+    assert scan_stats is not None
+    assert scan_stats.index_cache_misses == 2
+    assert scan_stats.index_cache_hits == 0
+
+    ds.scanner(
+        filter="color IN ('red', 'blue')", scan_stats_callback=scan_stats_callback
+    ).to_table()
+    assert scan_stats.index_cache_hits == 2
+    assert scan_stats.index_cache_misses == 0
+
+    # A value that is not in the index short-circuits before the loader and
+    # must not touch either counter.
+    ds.scanner(
+        filter="color = 'purple'", scan_stats_callback=scan_stats_callback
+    ).to_table()
+    assert scan_stats.index_cache_hits == 0
+    assert scan_stats.index_cache_misses == 0
+
+
+def test_phrase_query_cache_hit_miss_stats(tmp_path: Path):
+    """Phrase-query fallback populates ``PositionKey``; that boundary must
+    show up in per-query cache statistics.
+
+    Guards against ``read_positions`` silently using the non-metric
+    ``get_or_insert_with_key`` API — before this fix, a warm phrase query
+    would report zero hits for the phrase-position cache slot even though
+    the loader was skipped.
+
+    The cold path can already record a few hits (``bm25_stats_for_terms``
+    populates ``PostingMetadataKey``, which is then re-read on the
+    posting-list path as a cross-boundary hit), so the cold assertion is
+    ``misses > hits`` rather than a strict zero.
+    """
+    scan_stats = None
+
+    def scan_stats_callback(stats: lance.ScanStatistics):
+        nonlocal scan_stats
+        scan_stats = stats
+
+    test_table = pa.table(
+        {"text": ["quick brown fox jumps over lazy dog" for _ in range(50)]}
+    )
+    ds = lance.write_dataset(test_table, tmp_path)
+    ds.create_scalar_index("text", index_type="INVERTED", with_position=True)
+
+    ds = lance.dataset(tmp_path)
+    ds.scanner(
+        scan_stats_callback=scan_stats_callback,
+        full_text_query='"quick brown"',
+    ).to_table()
+    assert scan_stats is not None
+    assert scan_stats.index_cache_misses > 0
+    assert scan_stats.index_cache_misses > scan_stats.index_cache_hits
+
+    ds.scanner(
+        scan_stats_callback=scan_stats_callback,
+        full_text_query='"quick brown"',
+    ).to_table()
+    assert scan_stats.index_cache_hits > 0
+    assert scan_stats.index_cache_misses == 0
+
+
+def test_fts_index_cache_hit_miss_stats(tmp_path: Path):
+    """Cold FTS scan reports misses; warm FTS scan reports hits.
+
+    Guards the wrapper-forwarding fix in ``FtsIndexMetrics``: previously the
+    two new cache-hit/miss trait methods had default no-op implementations
+    that swallowed FTS-side events, so cache activity was reported as ``0/0``
+    even for hot inverted-index scans.
+
+    The cold path can still record a few hits when the same cache key is
+    read across boundaries in one query (e.g. ``bm25_stats_for_terms``
+    populates ``PostingMetadataKey`` before ``posting_list`` re-reads it),
+    so the cold assertion is ``misses > hits`` rather than a strict zero.
+    """
+    scan_stats = None
+
+    def scan_stats_callback(stats: lance.ScanStatistics):
+        nonlocal scan_stats
+        scan_stats = stats
+
+    test_table = pa.table({"fts": ["word" for _ in range(100)]})
+    ds = lance.write_dataset(test_table, tmp_path)
+    ds.create_scalar_index("fts", index_type="INVERTED")
+
+    # Reopen so the session cache starts cold.
+    ds = lance.dataset(tmp_path)
+    ds.scanner(
+        scan_stats_callback=scan_stats_callback, full_text_query="word"
+    ).to_table()
+    assert scan_stats is not None
+    assert scan_stats.index_cache_misses > 0
+    assert scan_stats.index_cache_misses > scan_stats.index_cache_hits
+
+    # Same Dataset, warm cache — posting-list / metadata reads must now hit.
+    ds.scanner(
+        scan_stats_callback=scan_stats_callback, full_text_query="word"
+    ).to_table()
+    assert scan_stats.index_cache_hits > 0
+    assert scan_stats.index_cache_misses == 0
+
+
 def test_fts_backward_v0_27_0(tmp_path: Path):
     path = (
         Path(__file__).parent.parent.parent.parent