fix: read legacy compacted LSM keys across shared leaves

engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexCompacted.java

@@ -377,7 +377,14 @@ private LSMTreeIndexUnderlyingCompactedSeriesCursor searchInCurrentPage(boolean
       int pageInSeries = resultInRootPage.keyIndex;
 
       if (resultInRootPage.found) {
-        if (pageInSeries >= rootPageCount)
+        if (ascendingOrder) {
+          // Start at the first matching leaf, plus its possible shared predecessor for files written before the overflow
+          // safeguard. A non-matching predecessor advances to the next page in the page-level lookup below.
+          final int firstMatchingRootEntry = resultInRootPage.valueBeginPositions != null
+              ? resultInRootPage.keyIndex - resultInRootPage.valueBeginPositions.length + 1
+              : resultInRootPage.keyIndex;
+          pageInSeries = Math.max(0, firstMatchingRootEntry - 1);
+        } else if (pageInSeries >= rootPageCount)
           // LAST ITEM + FOUND = IT'S THE LAST ELEMENT OF THE LAST PAGE
           --pageInSeries;
       } else
@@ -485,6 +492,9 @@ protected void searchInCompactedIndex(final Object[] originalKeys, final Object[
       if (!resultInRootPage.outside) {
         // IT'S IN PAGE RANGE
         int pageInSeries = resultInRootPage.keyIndex;
+        final int firstMatchingRootEntry = resultInRootPage.found && resultInRootPage.valueBeginPositions != null
+            ? resultInRootPage.keyIndex - resultInRootPage.valueBeginPositions.length + 1
+            : -1;
 
         if (resultInRootPage.found) {
           if (pageInSeries >= rootPageCount)
@@ -520,6 +530,21 @@ protected void searchInCompactedIndex(final Object[] originalKeys, final Object[
               removedKeys, deletedRIDs))
             return;
         }
+
+        // Compacted files written before the shared-leaf writer safeguard can store the first chunk of an overflowing key on
+        // the leaf that ends with the preceding key. Later chunks have root entries for the searched key, but that first chunk
+        // is reachable only through the immediately preceding leaf. The result set removes any overlap.
+        if (firstMatchingRootEntry > 0) {
+          final int precedingPageNum = rootPage.getPageId().getPageNumber() + firstMatchingRootEntry;
+          final BasePage precedingPage = database.getTransaction()
+              .getPage(new PageId(database, file.getFileId(), precedingPageNum), pageSize);
+          final Binary precedingPageBuffer = new Binary(precedingPage.slice());
+          final int precedingCount = getCount(precedingPage);
+
+          if (!lookupInPageAndAddInResultset(precedingPage, precedingPageBuffer, precedingCount, originalKeys, convertedKeys,
+              limit, set, removedKeys, deletedRIDs))
+            return;
+        }
       }
 
       --pageNumber;

engine/src/test/java/com/arcadedb/index/LegacySharedLeafIndexCompatibilityTest.java

@@ -0,0 +1,107 @@
+/*
+ * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com)
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package com.arcadedb.index;
+
+import com.arcadedb.database.Database;
+import com.arcadedb.database.DatabaseFactory;
+import com.arcadedb.query.sql.executor.ResultSet;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Verifies exact lookup compatibility with compacted files written before the shared-leaf writer safeguard.
+ *
+ * <p>The synthetic fixture contains 30,000 records with the same composite key. It was generated with the safeguard
+ * temporarily disabled; an unpatched reader returns only 28,857 records from its compacted index.</p>
+ */
+class LegacySharedLeafIndexCompatibilityTest {
+  private static final int EXPECTED_DUPLICATES = 30_000;
+
+  @TempDir
+  Path tempDir;
+
+  private void extractFixture() throws IOException {
+    try (InputStream resource = getClass().getResourceAsStream("/com/arcadedb/index/legacy-shared-leaf-fixture.fixture")) {
+      assertThat(resource).as("legacy compacted-index fixture").isNotNull();
+      try (ZipInputStream zip = new ZipInputStream(resource)) {
+        for (ZipEntry entry; (entry = zip.getNextEntry()) != null; ) {
+          final Path target = tempDir.resolve(entry.getName()).normalize();
+          assertThat(target.startsWith(tempDir.normalize())).as("zip entry stays under the test directory").isTrue();
+          if (entry.isDirectory())
+            Files.createDirectories(target);
+          else {
+            Files.createDirectories(target.getParent());
+            Files.copy(zip, target, StandardCopyOption.REPLACE_EXISTING);
+          }
+        }
+      }
+    }
+  }
+
+  private long count(final IndexCursor cursor) {
+    long count = 0;
+    while (cursor.hasNext()) {
+      cursor.next();
+      count++;
+    }
+    return count;
+  }
+
+  private void assertCounts(final Database database) {
+    try (final ResultSet scan = database.query("sql", "SELECT count(*) AS c FROM Tok WHERE word.trim() = 'dup'")) {
+      assertThat(((Number) scan.next().getProperty("c")).longValue()).as("full scan count").isEqualTo(EXPECTED_DUPLICATES);
+    }
+
+    try (final ResultSet indexed = database.query("sql", "SELECT count(*) AS c FROM Tok WHERE word = 'dup' AND lang = 'xx'")) {
+      assertThat(((Number) indexed.next().getProperty("c")).longValue()).as("legacy compacted-index lookup count")
+          .isEqualTo(EXPECTED_DUPLICATES);
+    }
+
+    final TypeIndex index = database.getSchema().getType("Tok").getIndexByProperties("word", "lang");
+    final Object[] fullKey = { "dup", "xx" };
+    assertThat(count(index.range(true, fullKey, true, fullKey, true))).as("ascending full-key range count")
+        .isEqualTo(EXPECTED_DUPLICATES);
+    assertThat(count(index.range(false, fullKey, true, fullKey, true))).as("descending full-key range count")
+        .isEqualTo(EXPECTED_DUPLICATES);
+  }
+
+  @Test
+  void lookupsReadFirstChunkFromSharedPrecedingLeaf() throws IOException {
+    extractFixture();
+
+    try (DatabaseFactory factory = new DatabaseFactory(tempDir.toString())) {
+      try (Database database = factory.open()) {
+        assertCounts(database);
+      }
+      try (Database reopened = factory.open()) {
+        assertCounts(reopened);
+      }
+    }
+  }
+}