diff --git a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexCompacted.java b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexCompacted.java index e5d8af5dd2..e65107271b 100644 --- a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexCompacted.java +++ b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexCompacted.java @@ -377,7 +377,14 @@ private LSMTreeIndexUnderlyingCompactedSeriesCursor searchInCurrentPage(boolean int pageInSeries = resultInRootPage.keyIndex; if (resultInRootPage.found) { - if (pageInSeries >= rootPageCount) + if (ascendingOrder) { + // Start at the first matching leaf, plus its possible shared predecessor for files written before the overflow + // safeguard. A non-matching predecessor advances to the next page in the page-level lookup below. + final int firstMatchingRootEntry = resultInRootPage.valueBeginPositions != null + ? resultInRootPage.keyIndex - resultInRootPage.valueBeginPositions.length + 1 + : resultInRootPage.keyIndex; + pageInSeries = Math.max(0, firstMatchingRootEntry - 1); + } else if (pageInSeries >= rootPageCount) // LAST ITEM + FOUND = IT'S THE LAST ELEMENT OF THE LAST PAGE --pageInSeries; } else @@ -485,6 +492,9 @@ protected void searchInCompactedIndex(final Object[] originalKeys, final Object[ if (!resultInRootPage.outside) { // IT'S IN PAGE RANGE int pageInSeries = resultInRootPage.keyIndex; + final int firstMatchingRootEntry = resultInRootPage.found && resultInRootPage.valueBeginPositions != null + ? resultInRootPage.keyIndex - resultInRootPage.valueBeginPositions.length + 1 + : -1; if (resultInRootPage.found) { if (pageInSeries >= rootPageCount) @@ -520,6 +530,21 @@ protected void searchInCompactedIndex(final Object[] originalKeys, final Object[ removedKeys, deletedRIDs)) return; } + + // Compacted files written before the shared-leaf writer safeguard can store the first chunk of an overflowing key on + // the leaf that ends with the preceding key. Later chunks have root entries for the searched key, but that first chunk + // is reachable only through the immediately preceding leaf. The result set removes any overlap. + if (firstMatchingRootEntry > 0) { + final int precedingPageNum = rootPage.getPageId().getPageNumber() + firstMatchingRootEntry; + final BasePage precedingPage = database.getTransaction() + .getPage(new PageId(database, file.getFileId(), precedingPageNum), pageSize); + final Binary precedingPageBuffer = new Binary(precedingPage.slice()); + final int precedingCount = getCount(precedingPage); + + if (!lookupInPageAndAddInResultset(precedingPage, precedingPageBuffer, precedingCount, originalKeys, convertedKeys, + limit, set, removedKeys, deletedRIDs)) + return; + } } --pageNumber; diff --git a/engine/src/test/java/com/arcadedb/index/LegacySharedLeafIndexCompatibilityTest.java b/engine/src/test/java/com/arcadedb/index/LegacySharedLeafIndexCompatibilityTest.java new file mode 100644 index 0000000000..27a3888af6 --- /dev/null +++ b/engine/src/test/java/com/arcadedb/index/LegacySharedLeafIndexCompatibilityTest.java @@ -0,0 +1,107 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.index; + +import com.arcadedb.database.Database; +import com.arcadedb.database.DatabaseFactory; +import com.arcadedb.query.sql.executor.ResultSet; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Verifies exact lookup compatibility with compacted files written before the shared-leaf writer safeguard. + * + *

The synthetic fixture contains 30,000 records with the same composite key. It was generated with the safeguard + * temporarily disabled; an unpatched reader returns only 28,857 records from its compacted index.

+ */ +class LegacySharedLeafIndexCompatibilityTest { + private static final int EXPECTED_DUPLICATES = 30_000; + + @TempDir + Path tempDir; + + private void extractFixture() throws IOException { + try (InputStream resource = getClass().getResourceAsStream("/com/arcadedb/index/legacy-shared-leaf-fixture.fixture")) { + assertThat(resource).as("legacy compacted-index fixture").isNotNull(); + try (ZipInputStream zip = new ZipInputStream(resource)) { + for (ZipEntry entry; (entry = zip.getNextEntry()) != null; ) { + final Path target = tempDir.resolve(entry.getName()).normalize(); + assertThat(target.startsWith(tempDir.normalize())).as("zip entry stays under the test directory").isTrue(); + if (entry.isDirectory()) + Files.createDirectories(target); + else { + Files.createDirectories(target.getParent()); + Files.copy(zip, target, StandardCopyOption.REPLACE_EXISTING); + } + } + } + } + } + + private long count(final IndexCursor cursor) { + long count = 0; + while (cursor.hasNext()) { + cursor.next(); + count++; + } + return count; + } + + private void assertCounts(final Database database) { + try (final ResultSet scan = database.query("sql", "SELECT count(*) AS c FROM Tok WHERE word.trim() = 'dup'")) { + assertThat(((Number) scan.next().getProperty("c")).longValue()).as("full scan count").isEqualTo(EXPECTED_DUPLICATES); + } + + try (final ResultSet indexed = database.query("sql", "SELECT count(*) AS c FROM Tok WHERE word = 'dup' AND lang = 'xx'")) { + assertThat(((Number) indexed.next().getProperty("c")).longValue()).as("legacy compacted-index lookup count") + .isEqualTo(EXPECTED_DUPLICATES); + } + + final TypeIndex index = database.getSchema().getType("Tok").getIndexByProperties("word", "lang"); + final Object[] fullKey = { "dup", "xx" }; + assertThat(count(index.range(true, fullKey, true, fullKey, true))).as("ascending full-key range count") + .isEqualTo(EXPECTED_DUPLICATES); + assertThat(count(index.range(false, fullKey, true, fullKey, true))).as("descending full-key range count") + .isEqualTo(EXPECTED_DUPLICATES); + } + + @Test + void lookupsReadFirstChunkFromSharedPrecedingLeaf() throws IOException { + extractFixture(); + + try (DatabaseFactory factory = new DatabaseFactory(tempDir.toString())) { + try (Database database = factory.open()) { + assertCounts(database); + } + try (Database reopened = factory.open()) { + assertCounts(reopened); + } + } + } +} diff --git a/engine/src/test/resources/com/arcadedb/index/legacy-shared-leaf-fixture.fixture b/engine/src/test/resources/com/arcadedb/index/legacy-shared-leaf-fixture.fixture new file mode 100644 index 0000000000..b8392567e8 Binary files /dev/null and b/engine/src/test/resources/com/arcadedb/index/legacy-shared-leaf-fixture.fixture differ