diff --git a/engine/src/main/antlr4/com/arcadedb/query/sql/grammar/SQLParser.g4 b/engine/src/main/antlr4/com/arcadedb/query/sql/grammar/SQLParser.g4 index 2d1df1db03..aaa2010c07 100644 --- a/engine/src/main/antlr4/com/arcadedb/query/sql/grammar/SQLParser.g4 +++ b/engine/src/main/antlr4/com/arcadedb/query/sql/grammar/SQLParser.g4 @@ -136,6 +136,9 @@ statement // Index Management | rebuildIndexStatement # rebuildIndexStmt + // Type Re-serialisation (e.g. relocate property values after toggling EXTERNAL flag) + | REBUILD TYPE rebuildTypeBody # rebuildTypeStmt + // Transaction Statements | beginStatement # beginStmt | commitStatement # commitStmt @@ -868,6 +871,16 @@ rebuildIndexStatement : REBUILD INDEX (identifier | STAR) (WITH identifier EQ expression (COMMA identifier EQ expression)*)? ; +/** + * REBUILD TYPE statement + * Re-serialises every record of a type so that schema changes (e.g. toggling a property's EXTERNAL flag) are applied + * to existing records on disk. POLYMORPHIC additionally walks subtypes. + * Syntax: REBUILD TYPE typeName [POLYMORPHIC] + */ +rebuildTypeBody + : typeName=identifier POLYMORPHIC? (WITH settingKey+=identifier EQ settingValue+=expression (COMMA settingKey+=identifier EQ settingValue+=expression)*)? + ; + // ============================================================================ // CONTROL FLOW STATEMENTS // ============================================================================ diff --git a/engine/src/main/java/com/arcadedb/GlobalConfiguration.java b/engine/src/main/java/com/arcadedb/GlobalConfiguration.java index 06c0ef41c4..f3808fc445 100644 --- a/engine/src/main/java/com/arcadedb/GlobalConfiguration.java +++ b/engine/src/main/java/com/arcadedb/GlobalConfiguration.java @@ -210,6 +210,14 @@ public Object call(final Object value) { BUCKET_DEFAULT_PAGE_SIZE("arcadedb.bucketDefaultPageSize", SCOPE.DATABASE, "Default page size in bytes for buckets. Default is 64KB", Integer.class, 65_536), + EXTERNAL_PROPERTY_BUCKET_DEFAULT_PAGE_SIZE("arcadedb.externalPropertyBucketDefaultPageSize", SCOPE.DATABASE, + "Default page size in bytes for paired external-property buckets. They hold heavy property payloads (vector embeddings, large strings, embedded JSON) so the default is larger than for primary buckets to reduce multi-page chunking. Matches the LSM-index default (256KB)", + Integer.class, 262_144), + + EXTERNAL_PROPERTY_BUCKET_PATH("arcadedb.externalPropertyBucketPath", SCOPE.DATABASE, + "Filesystem directory where new paired external-property buckets are created. If empty (default), external buckets sit alongside primary buckets in the database directory. Set to a path on cheaper/slower storage (HDD, network mount) to tier the heavy payloads away from the topology files. The directory must exist and be writable. Existing external buckets are not relocated when this changes.", + String.class, ""), + BUCKET_REUSE_SPACE_MODE("arcadedb.bucketReuseSpaceMode", SCOPE.DATABASE, "How to reuse space in pages. 'high' = more space saved, but slower opening and update/delete time. 'medium' to still reuse space without the initial scan at opening time. 'low' for faster performance, but less space reused. Default is 'high'", String.class, "high", Set.of("low", "medium", "high")), diff --git a/engine/src/main/java/com/arcadedb/compression/CompressionFactory.java b/engine/src/main/java/com/arcadedb/compression/CompressionFactory.java index 7208bb1eb7..83fe1b5a22 100644 --- a/engine/src/main/java/com/arcadedb/compression/CompressionFactory.java +++ b/engine/src/main/java/com/arcadedb/compression/CompressionFactory.java @@ -27,4 +27,13 @@ public class CompressionFactory { public static Compression getDefault() { return defaultImplementation; } + + /** + * Returns the shared {@link LZ4Compression} instance. Exposes the concrete type for callers that need + * encoder variants beyond the {@link Compression} interface (e.g. {@link LZ4Compression#compressMax} for + * the LZ4 HC tier used by EXTERNAL property storage). + */ + public static LZ4Compression getLZ4() { + return defaultImplementation; + } } diff --git a/engine/src/main/java/com/arcadedb/compression/LZ4Compression.java b/engine/src/main/java/com/arcadedb/compression/LZ4Compression.java index 4eeeba1d82..161b13dd0a 100644 --- a/engine/src/main/java/com/arcadedb/compression/LZ4Compression.java +++ b/engine/src/main/java/com/arcadedb/compression/LZ4Compression.java @@ -26,16 +26,24 @@ import java.util.*; /** - * Compression implementation that uses the popular LZ4 algorithm. + * Compression implementation that uses the popular LZ4 algorithm. Two compressors are exposed: the default + * {@code fast} encoder (block size optimised for throughput) and an on-demand {@code max} (LZ4 HC) encoder + * that spends more CPU on a deeper match search to produce a smaller output. Both share the same LZ4 byte + * format and the same decoder, so {@link #decompress(byte[], int)} works regardless of which encoder produced + * the input. */ public class LZ4Compression implements Compression { private static final byte[] EMPTY_BYTES = new byte[0]; private static final Binary EMPTY_BINARY = new Binary(EMPTY_BYTES); + private final LZ4Factory factory; private final LZ4Compressor compressor; private final LZ4FastDecompressor decompressor; + // High-compression encoder is built on first use: it allocates ~256KB of internal state and is rarely needed + // on the read path, so we don't want to pay for it when only the fast encoder is in use. + private volatile LZ4Compressor maxCompressor; public LZ4Compression() { - final LZ4Factory factory = LZ4Factory.fastestInstance(); + this.factory = LZ4Factory.fastestInstance(); this.compressor = factory.fastCompressor(); this.decompressor = factory.fastDecompressor(); } @@ -50,6 +58,28 @@ public byte[] compress(final byte[] data) { return compressed; } + /** + * Maximum-compression encoder (LZ4 HC). Compresses 8-20x slower than {@link #compress(byte[])} but produces + * meaningfully smaller output (~10pp on text); decompression is the same speed. Use for write-once / + * read-many EXTERNAL payloads where bucket file size dominates. + */ + public byte[] compressMax(final byte[] data) { + LZ4Compressor c = maxCompressor; + if (c == null) { + synchronized (this) { + c = maxCompressor; + if (c == null) + c = maxCompressor = factory.highCompressor(); + } + } + final int maxCompressedLength = c.maxCompressedLength(data.length); + byte[] compressed = new byte[maxCompressedLength]; + final int compressedLength = c.compress(data, 0, data.length, compressed, 0, maxCompressedLength); + if (compressedLength != maxCompressedLength) + compressed = Arrays.copyOf(compressed, compressedLength); + return compressed; + } + @Override public Binary compress(final Binary data) { final int decompressedLength = data.size() - data.position(); diff --git a/engine/src/main/java/com/arcadedb/database/BaseDocument.java b/engine/src/main/java/com/arcadedb/database/BaseDocument.java index 2a869266d9..53b3008f30 100644 --- a/engine/src/main/java/com/arcadedb/database/BaseDocument.java +++ b/engine/src/main/java/com/arcadedb/database/BaseDocument.java @@ -39,7 +39,7 @@ import java.util.List; import java.util.Map; -public abstract class BaseDocument extends BaseRecord implements Document, Serializable, Externalizable { +public abstract class BaseDocument extends BaseRecord implements Document, DocumentInternal, Serializable, Externalizable { protected final DocumentType type; protected int propertiesStartingPosition = 1; @@ -157,6 +157,11 @@ public DocumentType getType() { return type; } + @Override + public int getPropertiesStartingPosition() { + return propertiesStartingPosition; + } + public String getTypeName() { return type.getName(); } diff --git a/engine/src/main/java/com/arcadedb/database/DocumentInternal.java b/engine/src/main/java/com/arcadedb/database/DocumentInternal.java new file mode 100644 index 0000000000..fc53004db8 --- /dev/null +++ b/engine/src/main/java/com/arcadedb/database/DocumentInternal.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.database; + +import com.arcadedb.utility.ExcludeFromJacocoGeneratedReport; + +/** + * Internal contract for {@link BaseDocument} that exposes serialization-format details to engine modules + * (notably {@code com.arcadedb.serializer.BinarySerializer}) without leaking those details into the public + * {@link Document} API. Code that consumes ArcadeDB as a library should never cast to this interface; cross-version + * stability is not guaranteed. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +@ExcludeFromJacocoGeneratedReport +public interface DocumentInternal { + /** + * Offset (in bytes) where the property header begins inside the record's binary buffer. Required by the + * serializer to scan for {@code TYPE_EXTERNAL} pointers without re-parsing the bucket-level record framing. + */ + int getPropertiesStartingPosition(); +} diff --git a/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java b/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java new file mode 100644 index 0000000000..85a4a6df70 --- /dev/null +++ b/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java @@ -0,0 +1,73 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.database; + +import com.arcadedb.serializer.json.JSONObject; + +/** + * Opaque payload record for EXTERNAL property values. Buffer = [RECORD_TYPE][value type][value bytes]. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public class ExternalValueRecord extends BaseRecord implements RecordInternal { + // BaseRecord-subclass type tags (first byte of every record buffer): + // Document = 0 (Document.java) + // Vertex = 1 (Vertex.java) + // Edge = 2 (Edge.java) + // EdgeSegment = 3 (EdgeSegment.java / MutableEdgeSegment.java) + // EmbeddedDocument = 4 (EmbeddedDocument.java) + // ExternalValueRecord = 5 (this class - paired-bucket payload for EXTERNAL property values) + // LightEdge = 6 (LightEdge.java) + // Add new values at the end and update this list. + public static final byte RECORD_TYPE = 5; + + public ExternalValueRecord(final Database database, final RID rid, final Binary buffer) { + super(database, rid, buffer); + } + + @Override + public byte getRecordType() { + return RECORD_TYPE; + } + + @Override + public void setIdentity(final RID rid) { + this.rid = upgradeRID(rid); + } + + @Override + public void unsetDirty() { + // NO-OP: BUFFER IS BUILT FRESH ON EACH SERIALIZE + } + + public Binary getContent() { + return buffer; + } + + @Override + public JSONObject toJSON(final boolean includeMetadata) { + // ExternalValueRecord is engine-internal infrastructure. It's only reachable through a primary record's + // TYPE_EXTERNAL pointer, and every public read path (Document.toJSON, REST handlers, Studio) walks through + // BinarySerializer.deserializeProperty which materialises the value inline. If a generic record dumper hits + // this method directly, it's a bug: failing loud surfaces it instead of returning silent empty data. + throw new UnsupportedOperationException( + "ExternalValueRecord is an internal payload of an EXTERNAL property and has no JSON representation; " + + "read the value through its owning Document instead of dereferencing the external RID directly"); + } +} diff --git a/engine/src/main/java/com/arcadedb/database/LocalDatabase.java b/engine/src/main/java/com/arcadedb/database/LocalDatabase.java index bd4e59cb39..298462102e 100644 --- a/engine/src/main/java/com/arcadedb/database/LocalDatabase.java +++ b/engine/src/main/java/com/arcadedb/database/LocalDatabase.java @@ -962,8 +962,15 @@ public void createRecordNoLock(final Record record, final String bucketName, fin if (bucketName == null && record instanceof Document doc) bucket = (LocalBucket) doc.getType().getBucketIdByRecord(doc, DatabaseContext.INSTANCE.getContext(databasePath).asyncMode); - else + else { bucket = (LocalBucket) schema.getBucketByName(bucketName); + // Reject direct writes to internal buckets (e.g. paired external-property buckets). They are infrastructure + // for the serializer, not user data containers; allowing user DML to write here would corrupt the schema's + // accounting of which records are real records vs. payload blobs. + if (bucket.getPurpose() != LocalBucket.Purpose.PRIMARY) + throw new IllegalArgumentException( + "Bucket '" + bucketName + "' is internal (purpose=" + bucket.getPurpose() + ") and cannot be written to directly"); + } ((RecordInternal) record).setIdentity(bucket.createRecord(record, discardRecordAfter)); @@ -1124,8 +1131,12 @@ public void deleteRecordNoLock(final Record record) { try { final LocalBucket bucket = schema.getBucketById(record.getIdentity().getBucketId()); - if (record instanceof Document document) + if (record instanceof Document document) { indexer.deleteDocument(document); + // Cascade-delete EXTERNAL property values living in paired external buckets. This must run BEFORE the primary + // record is deleted, so the buffer is still readable. Both deletes ride the same transaction. + cascadeDeleteExternalValues(document); + } if (record instanceof Edge edge) { graphEngine.deleteEdge(edge); @@ -1155,6 +1166,27 @@ public void deleteRecordNoLock(final Record record) { } } + /** + * Deletes all external-bucket records referenced by the given document's TYPE_EXTERNAL property pointers, in the same + * transaction as the primary delete. No-op if the type has no EXTERNAL properties or the document was not loaded with + * a buffer. + */ + private void cascadeDeleteExternalValues(final Document document) { + if (!(document.getType() instanceof LocalDocumentType localType)) + return; + if (!localType.hasExternalProperties()) + return; + final Map externalRids = serializer.findExistingExternalRids(this, document); + for (final RID extRid : externalRids.values()) { + final LocalBucket externalBucket = schema.getBucketById(extRid.getBucketId(), false); + if (externalBucket != null) { + externalBucket.deleteRecord(extRid); + // Keep the external bucket's count consistent (mirrors the +1 in BinarySerializer.writeExternalPropertyValue). + getTransaction().updateBucketRecordDelta(externalBucket.getFileId(), -1); + } + } + } + @Override public boolean isTransactionActive() { final Transaction tx = getTransactionIfExists(); @@ -1436,6 +1468,14 @@ public String getName() { return name; } + /** Override root + dbName subdir, or null if not configured. The subdir prevents collisions across databases. */ + public String resolveExternalBucketPath() { + final String configured = configuration.getValueAsString(GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH); + if (configured == null || configured.isEmpty()) + return null; + return configured + File.separator + name; + } + @Override public ComponentFile.MODE getMode() { return mode; @@ -2028,7 +2068,7 @@ private void openInternal() { DatabaseContext.INSTANCE.init(this); setLockingEnabled(configuration.getValueAsBoolean(GlobalConfiguration.BACKUP_ENABLED)); - fileManager = new FileManager(databasePath, mode, SUPPORTED_FILE_EXT); + fileManager = new FileManager(databasePath, mode, SUPPORTED_FILE_EXT, resolveExternalBucketPath()); transactionManager = new TransactionManager(wrappedDatabaseInstance); open = true; diff --git a/engine/src/main/java/com/arcadedb/database/MutableDocument.java b/engine/src/main/java/com/arcadedb/database/MutableDocument.java index 8c20ac01cd..c10b18d00d 100644 --- a/engine/src/main/java/com/arcadedb/database/MutableDocument.java +++ b/engine/src/main/java/com/arcadedb/database/MutableDocument.java @@ -70,6 +70,15 @@ public boolean isDirty() { return dirty; } + /** + * Forces the next save to re-serialize this record even if no property has changed. Used by maintenance commands + * like `REBUILD TYPE` to apply schema layout changes (e.g. EXTERNAL flag toggles) to existing records on disk. + */ + public MutableDocument markDirty() { + dirty = true; + return this; + } + @Override public void setBuffer(final Binary buffer) { super.setBuffer(buffer); diff --git a/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java b/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java index 67e31ea66b..f0b4fcad02 100644 --- a/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java +++ b/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java @@ -20,6 +20,7 @@ import com.arcadedb.database.Database; import com.arcadedb.database.DatabaseInternal; +import com.arcadedb.database.Document; import com.arcadedb.database.RID; import com.arcadedb.database.Record; import com.arcadedb.graph.GraphDatabaseChecker; @@ -28,10 +29,12 @@ import com.arcadedb.index.lsm.LSMTreeIndexAbstract; import com.arcadedb.log.LogManager; import com.arcadedb.schema.DocumentType; +import com.arcadedb.schema.LocalDocumentType; import com.arcadedb.schema.LocalEdgeType; import com.arcadedb.schema.LocalVertexType; import com.arcadedb.schema.Schema; import com.arcadedb.serializer.json.JSONObject; +import com.arcadedb.utility.LongHashSet; import java.io.*; import java.util.*; @@ -71,6 +74,8 @@ public Map check() { checkBuckets(result); + checkExternalProperties(); + final Set affectedBuckets = new HashSet<>(); for (final RID rid : (Collection) result.get("corruptedRecords")) if (rid != null) @@ -254,6 +259,116 @@ public DatabaseChecker setCompress(final boolean compress) { return this; } + /** Detects (and on FIX deletes) external-property records that are no longer referenced by any primary record. */ + private void checkExternalProperties() { + if (verboseLevel > 0) + LogManager.instance().log(this, Level.INFO, "Checking external-property buckets..."); + + final List warnings = new ArrayList<>(); + final Set orphanedExternalRecords = new LinkedHashSet<>(); + long fixedCount = 0L; + + // For every external bucket, build the set of positions actually referenced from primary records of its + // owning type. Orphan = an external record whose position is NOT in that set. + // + // We use LongHashSet (open-addressing primitive long set) instead of HashSet to skip Long-boxing on + // every add/contains and shrink the per-entry footprint from ~48 B (Long box + HashMap.Node + array slot) + // to ~8 B + load-factor slack. On a database with 10M referenced positions this is the difference between + // ~50 MB and ~10 MB of CHECK DATABASE working set. A future streaming variant could sort both bucket scans + // by position and walk them in lockstep, dropping the heap footprint to O(1) at the cost of a sort. + final Map referencedByExtBucketId = new HashMap<>(); + final Map extBucketsToCheck = new HashMap<>(); + + for (final DocumentType type : database.getSchema().getTypes()) { + if (!(type instanceof LocalDocumentType ldt) || !ldt.hasExternalProperties()) + continue; + if (types != null && !types.isEmpty() && !types.contains(type.getName())) + continue; + + for (final Bucket primaryBucket : type.getBuckets(false)) { + final Integer extBucketId = ldt.getExternalBucketIdFor(primaryBucket.getFileId()); + if (extBucketId == null) + continue; + final LocalBucket extBucket = (LocalBucket) database.getSchema().getBucketById(extBucketId); + extBucketsToCheck.put(extBucketId, extBucket); + final LongHashSet referenced = referencedByExtBucketId.computeIfAbsent(extBucketId, k -> new LongHashSet()); + + primaryBucket.scan((rid, view) -> { + try { + final Document record = (Document) database.getRecordFactory().newImmutableRecord(database, type, rid, view, null); + for (final RID extRid : database.getSerializer().findExistingExternalRids(database, record).values()) + referenced.add(extRid.getPosition()); + } catch (final Exception e) { + warnings.add("primary record " + rid + " could not be parsed for external pointer scan: " + e.getMessage()); + } + return true; + }, null); + } + } + + for (final Map.Entry entry : extBucketsToCheck.entrySet()) { + final LocalBucket extBucket = entry.getValue(); + final LongHashSet referenced = referencedByExtBucketId.get(entry.getKey()); + + final List orphans = new ArrayList<>(); + extBucket.scan((rid, view) -> { + if (!referenced.contains(rid.getPosition())) + orphans.add(rid); + return true; + }, null); + + orphanedExternalRecords.addAll(orphans); + + if (fix && !orphans.isEmpty()) { + final boolean startedNewTx = !database.isTransactionActive(); + if (startedNewTx) + database.begin(); + // All-or-nothing per bucket: any failure rolls back the whole batch (or, if a caller-supplied tx is + // already open, throws so the caller can decide). We never commit a partially-cleaned bucket. + boolean anyFailure = false; + long localFixed = 0L; + for (final RID orphan : orphans) { + try { + extBucket.deleteRecord(orphan); + // Mirror the accounting in LocalDatabase.cascadeDeleteExternalValues so count() stays consistent. + database.getTransaction().updateBucketRecordDelta(extBucket.getFileId(), -1); + localFixed++; + } catch (final Exception e) { + warnings.add("could not delete orphan external record " + orphan + ": " + e.getMessage()); + anyFailure = true; + break; + } + } + if (startedNewTx) { + if (anyFailure) + database.rollback(); + else { + database.commit(); + fixedCount += localFixed; + } + } else if (anyFailure) { + throw new com.arcadedb.exception.DatabaseOperationException( + "Failed to delete orphan external records in bucket '" + extBucket.getName() + + "'; aborting CHECK DATABASE FIX so the caller's transaction is not silently committed in a partial state"); + } else { + fixedCount += localFixed; + } + } + } + + result.put("orphanedExternalRecords", (long) orphanedExternalRecords.size()); + result.put("orphanedExternalRecordsFixed", fixedCount); + // BinarySerializer.findExistingExternalRids() catches parse failures and returns an empty map, which + // means the caller skips orphan-cleanup for that record. Surfacing the JVM-cumulative count here lets the + // operator notice corruption-driven leak rates climbing without having to grep WARN logs. The counter is + // process-static, so re-runs of CHECK report the running total since startup. + result.put("externalRidScanFailuresCumulative", + com.arcadedb.serializer.BinarySerializer.getExternalRidScanFailures()); + ((LinkedHashSet) result.get("warnings")).addAll(warnings); + if (fix) + ((LinkedHashSet) result.get("deletedRecordsAfterFix")).addAll(orphanedExternalRecords); + } + private void checkBuckets(final Map result) { if (verboseLevel > 0) LogManager.instance().log(this, Level.INFO, "Checking buckets..."); diff --git a/engine/src/main/java/com/arcadedb/engine/FileManager.java b/engine/src/main/java/com/arcadedb/engine/FileManager.java index 5dfface981..7350e88529 100644 --- a/engine/src/main/java/com/arcadedb/engine/FileManager.java +++ b/engine/src/main/java/com/arcadedb/engine/FileManager.java @@ -68,6 +68,17 @@ public static class FileManagerStats { } public FileManager(final String path, final ComponentFile.MODE mode, final Set supportedFileExt) { + this(path, mode, supportedFileExt, null); + } + + /** + * @param path primary database directory; created if missing. + * @param extraScanPath optional secondary directory to scan for additional component files (e.g. paired + * external-property buckets that have been tiered to a different disk via + * {@code arcadedb.externalPropertyBucketPath}). May be null/empty. + */ + public FileManager(final String path, final ComponentFile.MODE mode, final Set supportedFileExt, + final String extraScanPath) { this.mode = mode; final File dbDirectory = new File(path); @@ -83,18 +94,35 @@ public FileManager(final String path, final ComponentFile.MODE mode, final Set supportedFileExt) { + final File[] entries = dir.listFiles(); + if (entries == null) + return; + for (final File f : entries) { + // Compute the extension from the file name (not the full path) so a database directory containing dots + // (e.g. /home/u/my.db/bucket1) doesn't accidentally find the dot in the directory name. + final String fileName = f.getName(); + final int lastDot = fileName.lastIndexOf("."); + if (lastDot < 0) + continue; + final String fileExt = fileName.substring(lastDot + 1); + if (!supportedFileExt.contains(fileExt)) + continue; + try { + final ComponentFile file = new PaginatedComponentFile(f.getAbsolutePath(), mode); + registerFile(file); + } catch (final FileNotFoundException e) { + LogManager.instance().log(this, Level.WARNING, "Cannot load file '%s'", null, f); } } } diff --git a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java index ea8bbe268a..720d297da1 100644 --- a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java +++ b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java @@ -21,6 +21,7 @@ import com.arcadedb.GlobalConfiguration; import com.arcadedb.database.Binary; import com.arcadedb.database.DatabaseInternal; +import com.arcadedb.database.Document; import com.arcadedb.database.RID; import com.arcadedb.database.Record; import com.arcadedb.database.RecordEventsRegistry; @@ -71,6 +72,13 @@ public class LocalBucket extends PaginatedComponent implements Bucket { public static final String BUCKET_EXT = "bucket"; public static final int CURRENT_VERSION = 0; + // Bucket file-format version 1 is reserved for paired external-property buckets. They hold heavier records, + // so the page-slot table is sized down to 256 (vs 2048 at v0). The bucket-page header (computed by + // contentHeaderSize = PAGE_RECORD_TABLE_OFFSET + maxSlots*4) shrinks from 8194 bytes at v0 to 1026 bytes at + // v1 - reclaiming ~7KB of header per page that becomes available for value blobs. Sized to host typical + // 1-2KB records (especially with compression enabled). + public static final int EXTERNAL_BUCKET_VERSION = 1; + private static final int DEF_MAX_RECORDS_IN_PAGE_V1 = 256; public static final long RECORD_PLACEHOLDER_POINTER = -1L; // USE -1 AS SIZE TO STORE A PLACEHOLDER (THAT POINTS TO A RECORD ON ANOTHER PAGE) public static final long FIRST_CHUNK = -2L; // USE -2 TO MARK THE FIRST CHUNK OF A BIG RECORD. FOLLOWS THE CHUNK SIZE AND THE POINTER TO THE NEXT CHUNK public static final long NEXT_CHUNK = -3L; // USE -3 TO MARK THE SECOND AND FURTHER CHUNK THAT IS PART OF A BIG RECORD THAT DOES NOT FIT A PAGE. FOLLOWS THE CHUNK SIZE AND THE POINTER TO THE NEXT CHUNK OR 0 IF THE CURRENT CHUNK IS THE LAST (NO FURTHER CHUNKS) @@ -87,8 +95,25 @@ public class LocalBucket extends PaginatedComponent implements Bucket { private static final int GATHER_STATS_MIN_SPACE_PERC = 10; private static final int SPARE_SPACE_FOR_GROWTH = 32; protected final int contentHeaderSize; - private final int maxRecordsInPage = DEF_MAX_RECORDS_IN_PAGE; + private final int maxRecordsInPage; private final AtomicLong cachedRecordCount = new AtomicLong(-1); + + /** + * Bucket purpose tag. Declared up here (next to the {@link #purpose} field that uses it) so the enum is the + * first thing a reader sees alongside the other bucket-level constants - the alternative was burying it + * mid-file between unrelated state, which made the EXTERNAL property contract hard to discover. + */ + public enum Purpose { + /** Bucket holding the primary records of a type (vertex/edge/document). Targetable by user DML. */ + PRIMARY, + /** Paired infrastructure bucket holding externalised property values. NOT targetable by user DML. */ + EXTERNAL_PROPERTY + } + + // Buckets are PRIMARY by default (they hold the primary records of a type and are user-targetable via DML). + // Internal kinds (e.g. EXTERNAL_PROPERTY) hold serializer infrastructure that user-facing DML must not target. + // The purpose is persisted in schema.json (per-type) and restored at load time, see LocalDocumentType. + private Purpose purpose = Purpose.PRIMARY; // pageId → free-space-bytes. TreeMap ordering is unused (verified by grep), so a primitive // open-addressing map saves memory and avoids Integer boxing on every read/write/remove on // the page-allocation hot path. Bounded by MAX_PAGES_GATHER_STATS (100). Single-threaded @@ -130,9 +155,11 @@ public PaginatedComponent createOnLoad(final DatabaseInternal database, final St public LocalBucket(final DatabaseInternal database, final String name, final String filePath, final ComponentFile.MODE mode, final int pageSize, final int version) throws IOException { super(database, name, filePath, BUCKET_EXT, mode, pageSize, version); + this.maxRecordsInPage = maxRecordsInPageForVersion(version); this.contentHeaderSize = PAGE_RECORD_TABLE_OFFSET + (maxRecordsInPage * INT_SERIALIZED_SIZE); this.cachedRecordCount.set(0); this.reuseSpaceMode = REUSE_SPACE_MODE.valueOf(GlobalConfiguration.BUCKET_REUSE_SPACE_MODE.getValueAsString().toUpperCase()); + this.purpose = purposeForVersion(version); } /** @@ -141,12 +168,24 @@ public LocalBucket(final DatabaseInternal database, final String name, final Str public LocalBucket(final DatabaseInternal database, final String name, final String filePath, final int id, final ComponentFile.MODE mode, final int pageSize, final int version) throws IOException { super(database, name, filePath, id, mode, pageSize, version); + this.maxRecordsInPage = maxRecordsInPageForVersion(version); contentHeaderSize = PAGE_RECORD_TABLE_OFFSET + (maxRecordsInPage * INT_SERIALIZED_SIZE); this.reuseSpaceMode = REUSE_SPACE_MODE.valueOf(GlobalConfiguration.BUCKET_REUSE_SPACE_MODE.getValueAsString().toUpperCase()); + // Derive purpose from the bucket file version - the version itself is persisted in the on-disk file name + // (e.g. Doc_0_ext...v1.bucket), so this assignment is reliable as soon as the LocalBucket + // is constructed - long before LocalDocumentType.restoreExternalBuckets() runs. That closes the gap where + // a write path firing between FileManager scan and schema-load completion would have seen purpose=PRIMARY + // by default and bypassed the user-DML guard. Schema JSON still maps primary->external by name (which is + // an orthogonal concern), but the write guard now no longer depends on schema-load ordering. + this.purpose = purposeForVersion(version); if (this.reuseSpaceMode.ordinal() >= REUSE_SPACE_MODE.HIGH.ordinal()) gatherPageStatistics(); } + private static Purpose purposeForVersion(final int version) { + return version >= EXTERNAL_BUCKET_VERSION ? Purpose.EXTERNAL_PROPERTY : Purpose.PRIMARY; + } + @Override public void close() { super.close(); @@ -157,9 +196,27 @@ public int getMaxRecordsInPage() { return maxRecordsInPage; } + /** Slot-table sizing for the bucket file format version. v0=2048 (legacy), v1=256 (paired external buckets). */ + private static int maxRecordsInPageForVersion(final int version) { + return version >= EXTERNAL_BUCKET_VERSION ? DEF_MAX_RECORDS_IN_PAGE_V1 : DEF_MAX_RECORDS_IN_PAGE; + } + + public Purpose getPurpose() { + return purpose; + } + + public void setPurpose(final Purpose purpose) { + this.purpose = purpose; + } + @Override public RID createRecord(final Record record, final boolean discardRecordAfter) { database.checkPermissionsOnFile(fileId, SecurityDatabaseUser.ACCESS.CREATE_RECORD); + // Provisional identity for Document records (and subtypes: vertex, edge) so the serializer can resolve the + // target primary bucket for EXTERNAL property handling. EdgeSegment, ExternalValueRecord, and other internal + // record types do not need this and must not be touched. + if (record.getIdentity() == null && record instanceof Document && record instanceof RecordInternal ri) + ri.setIdentity(RID.create(database, fileId, -1L)); return createRecordInternal(record, false, discardRecordAfter); } diff --git a/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java b/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java index 7d676b9d35..11b7654fc7 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java +++ b/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java @@ -34,7 +34,9 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; @@ -5727,6 +5729,37 @@ public RebuildIndexStatement visitRebuildIndexStatement(final SQLParser.RebuildI return stmt; } + /** + * Visit REBUILD TYPE statement. + * Grammar: REBUILD TYPE typeName [POLYMORPHIC] [WITH key = expression (, key = expression)*] + */ + @Override + public RebuildTypeStatement visitRebuildTypeStmt(final SQLParser.RebuildTypeStmtContext ctx) { + final RebuildTypeStatement stmt = new RebuildTypeStatement(-1); + final SQLParser.RebuildTypeBodyContext bodyCtx = ctx.rebuildTypeBody(); + // Grammar uses explicit labels (typeName=, settingKey+=, settingValue+=) so a future grammar tweak that + // introduces another identifier slot won't silently shift index-based bindings here. + stmt.typeName = (Identifier) visit(bodyCtx.typeName); + stmt.polymorphic = bodyCtx.POLYMORPHIC() != null; + if (bodyCtx.WITH() != null) { + // Track which settingKey strings we've already seen so a duplicate (e.g. WITH batchSize=1, batchSize=2) + // doesn't silently shadow the first via Map.put. Bare Expression equality on the key would only catch + // exact AST matches; using the lowercased identifier string also catches case variants. + final Set seenKeys = new HashSet<>(); + for (int i = 0; i < bodyCtx.settingValue.size(); i++) { + final Identifier keyId = (Identifier) visit(bodyCtx.settingKey.get(i)); + final String keyName = keyId.getStringValue().toLowerCase(Locale.ENGLISH); + if (!seenKeys.add(keyName)) + throw new CommandSQLParsingException( + "REBUILD TYPE WITH clause has duplicate setting '" + keyId.getStringValue() + + "'. Each setting must appear at most once."); + final Expression value = (Expression) visit(bodyCtx.settingValue.get(i)); + stmt.settings.put(new Expression(keyId), value); + } + } + return stmt; + } + // DDL STATEMENTS - ALTER /** diff --git a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaBucketDetailStep.java b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaBucketDetailStep.java index 02945fb486..3f196290d0 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaBucketDetailStep.java +++ b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaBucketDetailStep.java @@ -57,6 +57,7 @@ public ResultSet syncPull(final CommandContext context, final int nRecords) thro r.setProperty("fileId", bucket.getFileId()); r.setProperty("pageSize", bucket.getPageSize()); r.setProperty("totalPages", bucket.getTotalPages()); + r.setProperty("purpose", bucket.getPurpose().name()); final Map checkResult = bucket.check(0, false); for (final Map.Entry entry : checkResult.entrySet()) diff --git a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaBucketsStep.java b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaBucketsStep.java index 2510bc3cd0..415963209e 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaBucketsStep.java +++ b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaBucketsStep.java @@ -19,6 +19,7 @@ package com.arcadedb.query.sql.executor; import com.arcadedb.engine.Bucket; +import com.arcadedb.engine.LocalBucket; import com.arcadedb.exception.TimeoutException; import com.arcadedb.schema.Schema; @@ -60,6 +61,10 @@ public ResultSet syncPull(final CommandContext context, final int nRecords) thro r.setProperty("name", bucket.getName()); r.setProperty("fileId", bucket.getFileId()); r.setProperty("records", context.getDatabase().countBucket(bucketName)); + // The bucket's purpose lets tooling (Studio etc.) hide or label internal buckets like paired + // external-property buckets. Filter via `WHERE purpose = 'PRIMARY'` to see only user-targetable ones. + if (bucket instanceof LocalBucket lb) + r.setProperty("purpose", lb.getPurpose().name()); context.setVariable("current", r); } diff --git a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java index 308a55e549..e563a74083 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java +++ b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java @@ -19,6 +19,7 @@ package com.arcadedb.query.sql.executor; import com.arcadedb.database.Document; +import com.arcadedb.engine.Bucket; import com.arcadedb.engine.timeseries.ColumnDefinition; import com.arcadedb.engine.timeseries.DownsamplingTier; import com.arcadedb.engine.timeseries.TimeSeriesEngine; @@ -28,6 +29,7 @@ import com.arcadedb.graph.Vertex; import com.arcadedb.index.Index; import com.arcadedb.schema.DocumentType; +import com.arcadedb.schema.LocalDocumentType; import com.arcadedb.schema.LocalTimeSeriesType; import com.arcadedb.schema.Schema; @@ -89,6 +91,22 @@ else if (type.getType() == Edge.RECORD_TYPE) r.setProperty("buckets", type.getBuckets(false).stream().map((b) -> b.getName()).collect(Collectors.toList())); r.setProperty("bucketSelectionStrategy", type.getBucketSelectionStrategy().getName()); + // Expose the primary->external bucket mapping for types that have any EXTERNAL property. Lets tooling + // (Studio etc.) tell the user where the externalised values for each primary bucket are stored. + if (type instanceof LocalDocumentType ldt) { + final Map extMap = new HashMap<>(); + for (final Bucket b : type.getBuckets(false)) { + final Integer extId = ldt.getExternalBucketIdFor(b.getFileId()); + if (extId != null) { + final Bucket extBucket = context.getDatabase().getSchema().getBucketById(extId); + if (extBucket != null) + extMap.put(b.getName(), extBucket.getName()); + } + } + if (!extMap.isEmpty()) + r.setProperty("externalBuckets", extMap); + } + final List parents = type.getSuperTypes().stream().map(pt -> pt.getName()).collect(Collectors.toList()); r.setProperty("parentTypes", parents); @@ -109,6 +127,13 @@ else if (type.getType() == Edge.RECORD_TYPE) propRes.setProperty("notNull", property.isNotNull()); if (property.isHidden()) propRes.setProperty("hidden", property.isHidden()); + if (property.isExternal()) + propRes.setProperty("external", property.isExternal()); + // Only emit the compression policy when explicitly set to a non-default value, mirroring the + // toJSON convention. Studio renders the badge tooltip and a small label off this field. + final String cmp = property.getCompression(); + if (cmp != null && !"none".equalsIgnoreCase(cmp)) + propRes.setProperty("compression", cmp); if (property.getMin() != null) propRes.setProperty("min", property.getMin()); if (property.getMax() != null) diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java b/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java index ae3902bf30..a684b833b9 100755 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java @@ -88,6 +88,12 @@ public ResultSet executeDDL(final CommandContext context) { } else if (setting.equalsIgnoreCase("hidden")) { oldValue = property.isHidden(); property.setHidden((boolean) finalValue); + } else if (setting.equalsIgnoreCase("external")) { + oldValue = property.isExternal(); + property.setExternal((boolean) finalValue); + } else if (setting.equalsIgnoreCase("compression")) { + oldValue = property.getCompression(); + property.setCompression(String.valueOf(finalValue)); } else if (setting.equalsIgnoreCase("max")) { oldValue = property.getMax(); property.setMax("" + finalValue); diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java b/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java index 61b35f6a7b..d50a3b39c7 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java @@ -71,6 +71,10 @@ public Object setOnProperty(final Property internalProp, final CommandContext co internalProp.setNotNull((boolean) attrValue); } else if (attrName.equalsIgnoreCase("hidden")) { internalProp.setHidden((boolean) attrValue); + } else if (attrName.equalsIgnoreCase("external")) { + internalProp.setExternal((boolean) attrValue); + } else if (attrName.equalsIgnoreCase("compression")) { + internalProp.setCompression(String.valueOf(attrValue)); } else if (attrName.equalsIgnoreCase("max")) { internalProp.setMax("" + attrValue); } else if (attrName.equalsIgnoreCase("min")) { diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java b/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java new file mode 100644 index 0000000000..b204b9234c --- /dev/null +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java @@ -0,0 +1,232 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.query.sql.parser; + +import com.arcadedb.database.Database; +import com.arcadedb.database.MutableDocument; +import com.arcadedb.exception.CommandExecutionException; +import com.arcadedb.exception.CommandSQLParsingException; +import com.arcadedb.log.LogManager; +import com.arcadedb.query.sql.executor.CommandContext; +import com.arcadedb.query.sql.executor.InternalResultSet; +import com.arcadedb.query.sql.executor.ResultInternal; +import com.arcadedb.query.sql.executor.ResultSet; +import com.arcadedb.schema.DocumentType; +import com.arcadedb.schema.LocalDocumentType; +import com.arcadedb.schema.Schema; + +import java.util.*; + +/** + * REBUILD TYPE typeName [POLYMORPHIC] [WITH batchSize = N] - re-serialises records to apply schema layout changes. + *

+ * Behaviour under Raft HA. This statement extends {@link DDLStatement}, so + * {@code Statement.isDDL()} returns {@code true}. {@code RaftReplicatedDatabase.command()} therefore forwards + * any REBUILD TYPE issued on a follower to the leader via {@code forwardCommandToLeaderViaRaft}; the leader is + * the only node that runs the loop locally. Each in-loop {@code db.commit()} produces one Raft log entry; + * followers replay them in order. A follower that crashes mid-replay can resync from the leader's snapshot + * (the half-migrated intermediate state is recoverable by re-running REBUILD on the leader). + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public class RebuildTypeStatement extends DDLStatement { + private static final int DEFAULT_BATCH_SIZE = 10_000; + + public Identifier typeName; + public boolean polymorphic = false; + public final Map settings = new HashMap<>(); + + public RebuildTypeStatement(final int id) { + super(id); + } + + @Override + public ResultSet executeDDL(final CommandContext context) { + final Database db = context.getDatabase(); + final Schema schema = db.getSchema(); + final DocumentType type = schema.getType(typeName.getStringValue()); + if (type == null) + throw new CommandExecutionException("Type not found: " + typeName.getStringValue()); + + int batchSize = DEFAULT_BATCH_SIZE; + for (final Map.Entry e : settings.entrySet()) { + final String key = e.getKey().toString(); + if (key.equalsIgnoreCase("batchSize")) { + final Object raw = e.getValue().value; + try { + batchSize = Integer.parseInt(raw == null ? "null" : raw.toString()); + } catch (final NumberFormatException nfe) { + throw new CommandSQLParsingException( + "REBUILD TYPE setting 'batchSize' must be a positive integer, got: " + raw); + } + // batchSize is the modulus for the in-rebuild commit cadence (count[0] % batchSize == 0). A zero or + // negative value would either ArithmeticException (mod-zero) or never trip the commit branch (modulo by + // a negative still works but the boundary semantics are nonsensical). Reject up front. + if (batchSize <= 0) + throw new CommandSQLParsingException( + "REBUILD TYPE setting 'batchSize' must be a positive integer, got: " + batchSize); + } else + throw new CommandSQLParsingException( + "Unrecognized setting '" + key + "' in REBUILD TYPE statement (supported: batchSize)"); + } + final int finalBatchSize = batchSize; + + final long[] count = { 0L }; + // Records committed in earlier batches and therefore NOT rolled back by a later failure: needed so we can + // tell the caller exactly how many rows are already in the new layout when the rebuild aborts mid-stream. + final long[] committedBefore = { 0L }; + final boolean implicitTx = !db.isTransactionActive(); + if (implicitTx) + db.begin(); + + final long startNanos = System.nanoTime(); + try { + db.scanType(typeName.getStringValue(), polymorphic, rec -> { + final MutableDocument m = (MutableDocument) rec.modify(); + m.markDirty(); + m.save(); + count[0]++; + // Batch only when we own the transaction. Committing inside a caller-supplied TX would leak the user's + // writes prematurely and leave the trailing batch uncommitted. + if (implicitTx && count[0] % finalBatchSize == 0) { + db.commit(); + committedBefore[0] = count[0]; + // Per-batch progress line so an operator running REBUILD on a 100M-record type can observe forward + // motion via the server log instead of staring at a frozen prompt for many minutes. Logged at INFO + // so it's on by default but easy to silence by raising the level for this class. + final long elapsedMs = (System.nanoTime() - startNanos) / 1_000_000L; + LogManager.instance().log(this, java.util.logging.Level.INFO, + "REBUILD TYPE '%s': %,d records re-serialised so far (last batch=%,d, elapsed=%,d ms)", + null, typeName.getStringValue(), count[0], finalBatchSize, elapsedMs); + db.begin(); + } + return true; + }); + + if (implicitTx) + db.commit(); + final long elapsedMs = (System.nanoTime() - startNanos) / 1_000_000L; + LogManager.instance().log(this, java.util.logging.Level.INFO, + "REBUILD TYPE '%s' completed: %,d records re-serialised in %,d ms", + null, typeName.getStringValue(), count[0], elapsedMs); + } catch (Exception e) { + if (implicitTx && db.isTransactionActive()) + db.rollback(); + // REBUILD TYPE is NOT atomic across batch boundaries: every db.commit() above already persisted those + // records in the new layout. After rollback, only the in-flight batch is reverted; the type is left + // half-migrated. Surface the boundary clearly so the operator knows to re-run REBUILD TYPE to finish. + final long migratedAndKept = committedBefore[0]; + final long rolledBack = count[0] - committedBefore[0]; + throw new CommandExecutionException( + "Error on rebuilding type '" + typeName.getStringValue() + "' after " + count[0] + " records (" + + migratedAndKept + " committed in earlier batches and remain in the new layout, " + rolledBack + + " rolled back from the in-flight batch). REBUILD TYPE is NOT atomic across batches; re-run the" + + " command to migrate the remaining records once the underlying issue is fixed." + + " If the failed batch left external-bucket blobs that no primary record references (typical when the" + + " rebuild crashes between writing the external value and re-serializing the primary record), run" + + " CHECK DATABASE FIX to delete the orphaned external records.", e); + } + + // If the rebuild was triggered to revert a property from EXTERNAL to inline (i.e. the type no longer has + // any EXTERNAL property), the previously-paired external buckets are now empty (orphan-cleanup in + // serializeProperties deleted every external blob during the re-save). Drop them so they don't accumulate + // across toggle cycles, and persist the cleared mapping. + // Reclaim paired external buckets only if REBUILD owned the transaction. With a caller-supplied tx the + // queued record updates haven't flushed yet (LocalDatabase.updateRecord defers serialization to commit), + // so the orphan cleanup hasn't run and the buckets are still non-empty. The caller can re-run REBUILD + // outside their tx to trigger reclaim, or invoke it explicitly. + boolean reclaimSkipped = false; + if (type instanceof LocalDocumentType ldt && !ldt.hasExternalProperties()) { + if (implicitTx) + ldt.reclaimEmptyExternalBuckets(); + else if (ldt.hasExternalBuckets()) + // Caller-supplied tx with empty-EXTERNAL state and paired buckets still around: we cannot drop them + // here (records haven't flushed), so surface the situation in the result + log so the operator knows + // to re-run REBUILD outside a transaction to reclaim the buckets. + reclaimSkipped = true; + } + + final ResultInternal result = new ResultInternal(db); + result.setProperty("operation", "rebuild type"); + result.setProperty("typeName", typeName.getStringValue()); + result.setProperty("polymorphic", polymorphic); + result.setProperty("recordsRebuilt", count[0]); + if (reclaimSkipped) { + final String warning = "REBUILD TYPE " + typeName.getStringValue() + + " ran inside a caller-supplied transaction; paired external buckets were NOT reclaimed because " + + "queued record updates haven't been flushed yet. Re-run REBUILD TYPE outside a transaction (or " + + "commit and re-run) to drop the now-empty paired buckets."; + result.setProperty("warning", warning); + LogManager.instance().log(this, java.util.logging.Level.WARNING, warning); + } + final InternalResultSet rs = new InternalResultSet(); + rs.add(result); + return rs; + } + + @Override + public void toString(final Map params, final StringBuilder builder) { + builder.append("REBUILD TYPE "); + typeName.toString(params, builder); + if (polymorphic) + builder.append(" POLYMORPHIC"); + if (!settings.isEmpty()) { + builder.append(" WITH "); + boolean first = true; + for (final Map.Entry e : settings.entrySet()) { + if (!first) + builder.append(", "); + e.getKey().toString(params, builder); + builder.append(" = "); + e.getValue().toString(params, builder); + first = false; + } + } + } + + @Override + public RebuildTypeStatement copy() { + final RebuildTypeStatement result = new RebuildTypeStatement(-1); + result.typeName = typeName == null ? null : typeName.copy(); + result.polymorphic = polymorphic; + for (final Map.Entry e : settings.entrySet()) + result.settings.put(e.getKey().copy(), e.getValue().copy()); + return result; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final RebuildTypeStatement that = (RebuildTypeStatement) o; + return polymorphic == that.polymorphic + && Objects.equals(typeName, that.typeName) + && Objects.equals(settings, that.settings); + } + + @Override + public int hashCode() { + int result = typeName != null ? typeName.hashCode() : 0; + result = 31 * result + (polymorphic ? 1 : 0); + result = 31 * result + settings.hashCode(); + return result; + } +} diff --git a/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java b/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java index 21bf2348a6..aba1122a55 100644 --- a/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java +++ b/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java @@ -39,6 +39,18 @@ public abstract class AbstractProperty implements Property { protected boolean mandatory = false; protected boolean notNull = false; protected boolean hidden = false; + // Volatile because BinarySerializer.serializeProperties reads isExternal() outside the schema write lock + // on the per-record write hot path. The schema lock serialises mutations, but a reader that came in just + // before setExternal() flipped the bit must observe the latest value to route the value through the + // correct write path (inline vs paired bucket). volatile is the cheapest correctness fix and matches the + // memory-model role of {@link LocalDocumentType#ownExternalPropertyCount}'s atomic. + protected volatile boolean external = false; + // Compression policy for EXTERNAL property values: "none" | "fast" | "max" | "auto" (legacy alias: "lz4" -> "fast"). + // STORAGE CONVENTION: null means "none" (the default), so toJSON omits the key. Read access MUST go through + // getCompression(), which materialises null as the literal string "none". LocalProperty.setCompression + // normalises "none" / null / "" all to null on write. Direct field reads from outside this class would see + // null when the user wrote "none"; always use the getter. + protected String compression = null; protected String max = null; protected String min = null; protected String regexp = null; @@ -142,6 +154,16 @@ public boolean isHidden() { return hidden; } + @Override + public boolean isExternal() { + return external; + } + + @Override + public String getCompression() { + return compression == null ? "none" : compression; + } + @Override public String getMax() { return max; @@ -188,6 +210,10 @@ public JSONObject toJSON() { json.put("notNull", notNull); if (hidden) json.put("hidden", hidden); + if (external) + json.put("external", external); + if (compression != null && !"none".equalsIgnoreCase(compression)) + json.put("compression", compression); if (max != null) json.put("max", max); if (min != null) diff --git a/engine/src/main/java/com/arcadedb/schema/DocumentType.java b/engine/src/main/java/com/arcadedb/schema/DocumentType.java index 361dbd2350..2b3716ac5a 100644 --- a/engine/src/main/java/com/arcadedb/schema/DocumentType.java +++ b/engine/src/main/java/com/arcadedb/schema/DocumentType.java @@ -100,6 +100,10 @@ default Property createProperty(String propName, JSONObject prop) { p.setNotNull(prop.getBoolean("notNull")); if (prop.has("hidden")) p.setHidden(prop.getBoolean("hidden")); + if (prop.has("external")) + p.setExternal(prop.getBoolean("external")); + if (prop.has("compression")) + p.setCompression(prop.getString("compression")); if (prop.has("max")) p.setMax(prop.getString("max")); if (prop.has("min")) diff --git a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java index 4eabd66908..95cdbb0775 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java @@ -18,8 +18,10 @@ */ package com.arcadedb.schema; +import com.arcadedb.GlobalConfiguration; import com.arcadedb.database.DatabaseInternal; import com.arcadedb.database.Document; +import com.arcadedb.database.LocalDatabase; import com.arcadedb.database.MutableDocument; import com.arcadedb.database.RecordEvents; import com.arcadedb.database.RecordEventsRegistry; @@ -44,6 +46,7 @@ import java.io.*; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.*; import java.util.logging.*; public class LocalDocumentType implements DocumentType { @@ -63,6 +66,13 @@ public class LocalDocumentType implements DocumentType { protected List cachedPolymorphicBucketIds = new ArrayList<>(); // PRE COMPILED LIST TO SPEED UP RUN-TIME OPERATIONS protected BucketSelectionStrategy bucketSelectionStrategy = new RoundRobinBucketSelectionStrategy(); protected Set propertiesWithDefaultDefined = Collections.emptySet(); + // Map: primary bucket id -> external bucket id. Populated lazily when the first EXTERNAL property is + // set on the type, and persisted in schema.json under the per-type "externalBuckets" key. + protected final Map externalBucketIdByPrimaryBucketId = new ConcurrentHashMap<>(); + // Cached count of OWN properties (not inherited) currently flagged EXTERNAL. Avoids O(N) scans of + // getPolymorphicProperties() on hot paths (cascadeDeleteExternalValues, addBucketInternal, addSuperType). + // Maintained by LocalProperty.setExternal, dropProperty, and the schema-load path. + final AtomicInteger ownExternalPropertyCount = new AtomicInteger(0); public LocalDocumentType(final LocalSchema schema, final String name) { this.schema = schema; @@ -503,7 +513,11 @@ public Property dropProperty(final String propertyName) { } return recordFileChanges(() -> { - return properties.remove(propertyName); + final Property removed = properties.remove(propertyName); + // Keep the EXTERNAL counter consistent so hasExternalProperties() stays O(1). + if (removed != null && removed.isExternal()) + ownExternalPropertyCount.decrementAndGet(); + return removed; }); } @@ -962,6 +976,205 @@ protected void addBucketInternal(final Bucket bucket) { } }); } + + // IF THE TYPE ALREADY HAS EXTERNAL PROPERTIES, ENSURE A PAIRED EXTERNAL BUCKET FOR THIS NEW PRIMARY BUCKET + if (hasExternalProperties()) + ensureExternalBucketFor((LocalBucket) bucket); + } + + /** Polymorphic: counts inherited EXTERNAL properties too. O(1) on own count + O(depth) on supertype walk. */ + public boolean hasExternalProperties() { + if (ownExternalPropertyCount.get() > 0) + return true; + for (final LocalDocumentType st : superTypes) + if (st.hasExternalProperties()) + return true; + return false; + } + + public Integer getExternalBucketIdFor(final int primaryBucketId) { + return externalBucketIdByPrimaryBucketId.get(primaryBucketId); + } + + /** True when this type still owns at least one paired external-property bucket. */ + public boolean hasExternalBuckets() { + return !externalBucketIdByPrimaryBucketId.isEmpty(); + } + + public void ensureExternalBuckets() { + for (final Bucket b : buckets) + ensureExternalBucketFor((LocalBucket) b); + } + + /** Recurses into subtypes: records of a subtype live in subtype primary buckets, so each needs its own paired ext. */ + public void ensureExternalBucketsRecursive() { + ensureExternalBuckets(); + for (final LocalDocumentType sub : subTypes) + sub.ensureExternalBucketsRecursive(); + } + + private void ensureExternalBucketFor(final LocalBucket primary) { + // Atomic check-and-create: two threads racing through ensureExternalBuckets()/addBucketInternal() must not + // both attempt to allocate the paired _ext bucket and trip schema.createBucket's "already exists" guard. + externalBucketIdByPrimaryBucketId.computeIfAbsent(primary.getFileId(), pid -> { + final String extName = primary.getName() + "_ext"; + final LocalBucket external; + if (schema.bucketMap.containsKey(extName)) { + external = schema.bucketMap.get(extName); + // Refuse to adopt a bucket that is already registered as the primary bucket of some user type. + // {@code bucketId2TypeMap} is the authoritative source of "this bucket is a user type's primary + // bucket": it is rebuilt from each type's {@code getBuckets(false)} list (primary buckets only; + // adopted external buckets are NOT in this map, they live in per-type externalBucketIdByPrimaryBucketId). + // We therefore do NOT consult {@link LocalBucket#getPurpose}: purpose is transient and reset to PRIMARY + // on load, and depending on it would create a fragile ordering requirement against + // restoreExternalBuckets() running first. {@code computeIfAbsent} above already short-circuits when + // this type's own ext bucket is already adopted, so any hit here means the candidate name belongs to + // a different type's primary bucket, full stop. + if (schema.getTypeByBucketId(external.getFileId()) != null) + throw new SchemaException( + "Cannot adopt bucket '" + extName + "' as the external-property bucket for type '" + name + + "': it is already a primary bucket of another user type. The paired external bucket is named" + + " _ext, so a primary bucket called '" + extName + "' (or one whose name" + + " ends in '_ext' that matches another type's primary bucket) collides with this convention." + + " Resolve by renaming the conflicting primary bucket via 'ALTER BUCKET " + extName + + " NAME ...' so its name no longer ends in '_ext' or no longer collides, then retry the EXTERNAL" + + " property change."); + } else { + // External buckets get larger pages (256KB vs 64KB primary), a smaller slot table (256 vs 2048: file-format + // version EXTERNAL_BUCKET_VERSION), and optional placement on cheaper-storage tier via + // resolveExternalBucketPath() which returns /. + final int pageSize = schema.getDatabase().getConfiguration() + .getValueAsInteger(GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_DEFAULT_PAGE_SIZE); + final String overridePath = ((LocalDatabase) schema.getDatabase()).resolveExternalBucketPath(); + external = schema.createBucket(extName, pageSize, overridePath, LocalBucket.EXTERNAL_BUCKET_VERSION); + } + external.setPurpose(LocalBucket.Purpose.EXTERNAL_PROPERTY); + return external.getFileId(); + }); + } + + /** + * Drops paired external-property buckets that no longer back any record (typical case: a REBUILD TYPE that + * moved every value back inline because the EXTERNAL flag was just toggled off). Skips buckets that still + * hold records so we never lose data; those point at persistent corruption and need investigation. + *

+ * Preconditions (caller-enforced). + *

    + *
  1. {@code !hasExternalProperties()} - the type no longer has any EXTERNAL property to write into the + * bucket. Without this, a concurrent insert could legitimately write into the bucket between the + * {@code count() == 0} check and {@code dropBucket}.
    2. + *
  2. No transaction is active (or any active transaction has been committed before this call). The + * {@code count()} read consults pageManager state that has not yet flushed dirty pages from a still-open + * transaction, so an open tx can hide records from this method and lead to a data-losing drop.
    3. + *
+ * The current production caller ({@code RebuildTypeStatement}) only invokes this on the + * {@code implicitTx == true} path, after committing its own transaction, so both preconditions are met. + * Any new caller MUST honour both, or restructure to take a schema-level write lock that blocks inserts + * for the duration of the count-then-drop sequence. + */ + public void reclaimEmptyExternalBuckets() { + // Fail-fast on precondition #1 from the javadoc. A no-op return would silently mask the bug; throwing + // surfaces it during development AND production. The cost is one O(1) atomic read on a hot path that + // only runs after schema mutations and REBUILD TYPE, so it is negligible. + if (hasExternalProperties()) + throw new IllegalStateException( + "reclaimEmptyExternalBuckets() requires !hasExternalProperties() but type '" + name + "' still has at" + + " least one EXTERNAL property. Calling this with EXTERNAL properties present would race with" + + " concurrent inserts that legitimately write into the bucket between the count() check and" + + " dropBucket(). Drop the EXTERNAL flag on every property of the type before reclaiming."); + // The TOCTOU window between count() == 0 and dropBucket() is closed when the caller honours the second + // precondition (no active transaction). With the EXTERNAL flag off, no new code path will write into + // these buckets; with no in-flight tx, no queued update can have a record waiting to flush either. + // Use an explicit throw instead of `assert`: production JVMs run without -ea, and a violation here can + // lose data (we'd drop a bucket that has a queued tx record about to flush into it). Same pattern as + // the hasExternalProperties() guard two lines above. + if (((LocalDatabase) schema.getDatabase()).isTransactionActive()) + throw new IllegalStateException( + "reclaimEmptyExternalBuckets() requires no active transaction but one is open on database '" + + schema.getDatabase().getName() + "'. Queued record updates have not flushed yet, so the" + + " count() check could miss records about to land in the bucket and we'd drop it under them." + + " Commit (or rollback) the active transaction before calling reclaimEmptyExternalBuckets()."); + final List toDrop = new ArrayList<>(); + for (final Map.Entry entry : externalBucketIdByPrimaryBucketId.entrySet()) { + final LocalBucket extBucket = schema.getBucketById(entry.getValue(), false); + if (extBucket == null || extBucket.count() == 0L) + toDrop.add(entry.getKey()); + } + for (final Integer primaryBucketId : toDrop) { + final Integer extBucketId = externalBucketIdByPrimaryBucketId.remove(primaryBucketId); + if (extBucketId == null) + continue; + final LocalBucket extBucket = schema.getBucketById(extBucketId, false); + if (extBucket != null) + schema.dropBucket(extBucket.getName()); + } + schema.saveConfiguration(); + } + + /** + * Re-applies EXTERNAL_PROPERTY purpose (transient on {@link LocalBucket}) and rebuilds the map from JSON at + * load time. After processing the JSON-driven entries, runs a name-based heuristic sweep over this type's + * primary buckets: for any primary bucket whose paired '_ext' sibling exists in the schema's + * bucketMap but was missing from the JSON (corruption, partial migration, JSON edited by hand), adopt the + * sibling as the external bucket and tag its purpose. Without this fallback the {@code purpose} field would + * default to {@code PRIMARY}, the DML write guard ({@code LocalDatabase.createRecordNoLock}) would let users + * target the bucket directly, and an INSERT could corrupt internal payload bytes. Adoption is refused for + * any '_ext' bucket that bucketId2TypeMap already claims as another type's primary bucket. + */ + void restoreExternalBuckets(final Map primaryNameToExternalName) { + externalBucketIdByPrimaryBucketId.clear(); + for (final Map.Entry entry : primaryNameToExternalName.entrySet()) { + final LocalBucket primary = schema.bucketMap.get(entry.getKey()); + final LocalBucket external = schema.bucketMap.get(entry.getValue()); + if (primary == null) { + LogManager.instance() + .log(this, Level.WARNING, "Cannot restore external bucket mapping for type '%s': primary bucket '%s' not found", + null, name, entry.getKey()); + continue; + } + if (external == null) { + // Tiered bucket file not found: usually means arcadedb.externalPropertyBucketPath is unset on this restart + // but was set when the bucket was created. Reads of EXTERNAL properties for records in this primary + // bucket would silently fail - so we surface the configuration mismatch loudly and explicitly. + LogManager.instance().log(this, Level.SEVERE, + "Cannot find external bucket '%s' for type '%s' primary bucket '%s'. If the bucket was tiered to a " + + "secondary path, set 'arcadedb.externalPropertyBucketPath' to the same value used at creation " + + "time before reopening the database. EXTERNAL property reads on this type will fail until fixed.", + null, entry.getValue(), name, entry.getKey()); + continue; + } + external.setPurpose(LocalBucket.Purpose.EXTERNAL_PROPERTY); + externalBucketIdByPrimaryBucketId.put(primary.getFileId(), external.getFileId()); + } + + // Heuristic recovery: for every primary bucket of this type that does NOT yet have a paired entry in + // externalBucketIdByPrimaryBucketId, look for '_ext' in bucketMap and adopt it. Defends + // against schema.json missing the externalBuckets key (corruption, migration from an older snapshot, or + // a hand-edit). Refuses to adopt if the candidate is already registered as another type's primary + // bucket (bucketId2TypeMap is authoritative; the field-level Purpose is transient and unreliable here). + for (final Bucket primaryBucket : buckets) { + if (externalBucketIdByPrimaryBucketId.containsKey(primaryBucket.getFileId())) + continue; + final String candidateName = primaryBucket.getName() + "_ext"; + final LocalBucket candidate = schema.bucketMap.get(candidateName); + if (candidate == null) + continue; + if (schema.getTypeByBucketId(candidate.getFileId()) != null) { + // candidate is some other type's primary bucket; refuse to repurpose, just log so the operator sees it. + LogManager.instance().log(this, Level.WARNING, + "Heuristic recovery for type '%s': bucket '%s' looks like a paired external bucket by name but is" + + " already a primary bucket of another user type. Skipping adoption.", + null, name, candidateName); + continue; + } + candidate.setPurpose(LocalBucket.Purpose.EXTERNAL_PROPERTY); + externalBucketIdByPrimaryBucketId.put(primaryBucket.getFileId(), candidate.getFileId()); + LogManager.instance().log(this, Level.WARNING, + "Heuristic recovery for type '%s': adopted bucket '%s' as the external-property bucket for primary" + + " '%s'. The schema.json was missing the matching externalBuckets entry; it will be re-saved on" + + " the next schema mutation.", + null, name, candidateName, primaryBucket.getName()); + } } protected void removeBucketInternal(final Bucket bucket) { @@ -1078,6 +1291,28 @@ else if (this instanceof LocalEdgeType edgeType) { type.put("buckets", buckets); + if (!externalBucketIdByPrimaryBucketId.isEmpty()) { + // PRIMARY BUCKET NAME -> EXTERNAL BUCKET NAME. NAMES (NOT IDS) ARE PERSISTED FOR HUMAN READABILITY AND + // BECAUSE FILE IDS CAN BE REMAPPED ON FILE MIGRATION (LocalSchema.migratedFileIds). + // NAME DEPENDENCY: this serialised mapping is keyed by string. ArcadeDB does not currently expose a + // user-level RENAME BUCKET command, so the names are stable in practice. If a future feature lets a + // user rename a bucket, the rename code MUST update both sides of this mapping (or it will go stale on + // restart, and restoreExternalBuckets() will log SEVERE for the missing entry). Same constraint applies + // to the external bucket itself: its file name carries the primary's name + "_ext" suffix. + // TODO(rename-bucket): when a user-level RENAME BUCKET is introduced (see LocalSchema), it MUST also + // (a) re-key this map's primary entry, (b) rename the paired '_ext' bucket file to + // '_ext' to keep the naming convention consistent, and (c) re-save the schema so the JSON + // mirrors the new state. A grep for "TODO(rename-bucket)" surfaces every site that needs updating. + final JSONObject extBuckets = new JSONObject(); + for (final Map.Entry e : externalBucketIdByPrimaryBucketId.entrySet()) { + final LocalBucket primary = schema.getBucketById(e.getKey(), false); + final LocalBucket external = schema.getBucketById(e.getValue(), false); + if (primary != null && external != null) + extBuckets.put(primary.getName(), external.getName()); + } + type.put("externalBuckets", extBuckets); + } + type.put("aliases", aliases); final JSONObject properties = new JSONObject(); @@ -1149,6 +1384,11 @@ DocumentType addSuperType(final DocumentType superType, final boolean createInde // UPDATE THE LIST OF POLYMORPHIC BUCKETS TREE embeddedSuperType.updatePolymorphicBucketsCache(true, cachedPolymorphicBuckets, cachedPolymorphicBucketIds); + // IF THE NEWLY-LINKED SUPERTYPE HAS ANY EXTERNAL PROPERTY (OWN OR INHERITED), THIS SUBTYPE MUST OWN PAIRED + // EXTERNAL BUCKETS FOR ITS OWN PRIMARY BUCKETS, BECAUSE RECORDS OF THIS SUBTYPE LIVE IN THIS SUBTYPE'S BUCKETS. + if (embeddedSuperType.hasExternalProperties()) + ensureExternalBucketsRecursive(); + // CREATE INDEXES AUTOMATICALLY ON PROPERTIES DEFINED IN SUPER TYPES final Collection indexes = new ArrayList<>(getAllIndexes(true)); indexes.removeAll(indexesByProperties.values()); diff --git a/engine/src/main/java/com/arcadedb/schema/LocalProperty.java b/engine/src/main/java/com/arcadedb/schema/LocalProperty.java index a017cfabdc..aa68c81d98 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalProperty.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalProperty.java @@ -122,6 +122,55 @@ public Property setHidden(final boolean hidden) { return this; } + @Override + public Property setExternal(final boolean external) { + final boolean changed = !Objects.equals(this.external, external); + if (changed) { + final LocalDocumentType localOwner = (LocalDocumentType) owner; + if (external) { + // ORDERING IS LOAD-BEARING: paired buckets must exist BEFORE any reader can observe + // ownExternalPropertyCount > 0. hasExternalProperties() short-circuits on the counter, and the + // serializer's write path uses that to decide whether to route a value through the external bucket. + // If we incremented first and another thread saw count > 0 before ensureExternalBucketsRecursive() + // ran, it could try to write into a bucket that doesn't exist yet. Schema mutations are typically + // serialised by the schema lock, but this ordering keeps the property setter safe even without it. + localOwner.ensureExternalBucketsRecursive(); + this.external = external; + localOwner.ownExternalPropertyCount.incrementAndGet(); + } else { + // Flip the flag first so concurrent serialisations stop writing externally; the counter is the gate + // hasExternalProperties() consults, so decrement last to keep the "count > 0 implies bucket exists" + // invariant directional. + this.external = external; + localOwner.ownExternalPropertyCount.decrementAndGet(); + } + owner.getSchema().getEmbedded().saveConfiguration(); + } + return this; + } + + @Override + public Property setCompression(final String compression) { + final String normalized; + if (compression == null || compression.isEmpty() || "none".equalsIgnoreCase(compression)) + normalized = null; + else if ("fast".equalsIgnoreCase(compression) || "max".equalsIgnoreCase(compression) + || "auto".equalsIgnoreCase(compression)) + normalized = compression.toLowerCase(Locale.ENGLISH); + else if ("lz4".equalsIgnoreCase(compression)) + // Backwards alias: "lz4" was the original name for the fast tier; map it to "fast" so existing schema + // configs keep working without touching the schema.json on disk. + normalized = "fast"; + else + throw new IllegalArgumentException( + "Unsupported compression '" + compression + "' (supported: none, fast, max, auto)"); + if (!Objects.equals(this.compression, normalized)) { + this.compression = normalized; + owner.getSchema().getEmbedded().saveConfiguration(); + } + return this; + } + @Override public Property setMax(final String max) { final boolean changed = !Objects.equals(this.max, max); diff --git a/engine/src/main/java/com/arcadedb/schema/LocalSchema.java b/engine/src/main/java/com/arcadedb/schema/LocalSchema.java index dcc2a53adb..26c9eee18a 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalSchema.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalSchema.java @@ -372,15 +372,48 @@ public LocalBucket createBucket(final String bucketName) { } public LocalBucket createBucket(final String bucketName, final int pageSize) { + return createBucket(bucketName, pageSize, databasePath, LocalBucket.CURRENT_VERSION); + } + + /** Creates the bucket file under {@code parentDirectory} instead of the database directory; null/empty falls back. */ + public LocalBucket createBucket(final String bucketName, final int pageSize, final String parentDirectory) { + return createBucket(bucketName, pageSize, parentDirectory, LocalBucket.CURRENT_VERSION); + } + + /** + * Full overload: creates a bucket with an explicit file-format version. Paired external-property buckets pass + * {@link LocalBucket#EXTERNAL_BUCKET_VERSION} so they get the smaller (256-slot) page-slot table appropriate for + * heavy payloads; everything else uses {@link LocalBucket#CURRENT_VERSION}. + */ + public LocalBucket createBucket(final String bucketName, final int pageSize, final String parentDirectory, final int version) { database.checkPermissionsOnDatabase(SecurityDatabaseUser.DATABASE_ACCESS.UPDATE_SCHEMA); if (bucketMap.containsKey(bucketName)) throw new SchemaException("Cannot create bucket '" + bucketName + "' because already exists"); + // Discoverability warning for the EXTERNAL property naming convention. The engine creates paired buckets + // as '_ext' with file-format version EXTERNAL_BUCKET_VERSION, so version == CURRENT_VERSION (0) + // here means a user-driven CREATE BUCKET. A user bucket named '*_ext' will collide if a primary bucket + // with the matching prefix later gains an EXTERNAL property; ensureExternalBucketFor() rejects with a + // SchemaException at that point. Surfacing the constraint at create time is much cheaper than debugging + // the later failure. + if (version == LocalBucket.CURRENT_VERSION && bucketName.endsWith("_ext")) + LogManager.instance().log(this, Level.WARNING, + "Bucket name '%s' ends with '_ext'. The engine reserves the '_ext' suffix for paired" + + " EXTERNAL-property buckets. If a primary bucket whose name + '_ext' equals this name later" + + " gains an EXTERNAL property, that property change will fail with a SchemaException. Consider" + + " renaming this bucket to avoid the collision.", + null, bucketName); + + final String dir = (parentDirectory == null || parentDirectory.isEmpty()) ? databasePath : parentDirectory; + return recordFileChanges(() -> { try { - final LocalBucket bucket = new LocalBucket(database, bucketName, databasePath + File.separator + bucketName, - ComponentFile.MODE.READ_WRITE, pageSize, LocalBucket.CURRENT_VERSION); + final File parent = new File(dir); + if (!parent.exists() && !parent.mkdirs()) + throw new SchemaException("Cannot create directory '" + dir + "' for bucket '" + bucketName + "'"); + final LocalBucket bucket = new LocalBucket(database, bucketName, dir + File.separator + bucketName, + ComponentFile.MODE.READ_WRITE, pageSize, version); registerFile((Component) bucket); bucketMap.put(bucketName, bucket); @@ -1478,6 +1511,20 @@ protected synchronized void readConfiguration() { } } + // RESTORE THE primaryBucket -> externalBucket MAP BEFORE PROPERTIES ARE LOADED, SO THAT setExternal(true) ON A + // PROPERTY DOES NOT TRY TO LAZY-CREATE BUCKETS THAT ALREADY EXIST. Always call restoreExternalBuckets + // - even when the JSON has no externalBuckets key - so the name-based heuristic inside it can adopt + // any orphan '_ext' files that exist on disk but were lost from the JSON (partial corruption, + // migration from an older snapshot, etc.). Without that pass the affected buckets would default to + // purpose=PRIMARY and our DML write guard would let users target them. + final Map primaryToExternal = new HashMap<>(); + if (schemaType.has("externalBuckets")) { + final JSONObject extBuckets = schemaType.getJSONObject("externalBuckets"); + for (final String primaryName : extBuckets.keySet()) + primaryToExternal.put(primaryName, extBuckets.getString(primaryName)); + } + type.restoreExternalBuckets(primaryToExternal); + type.custom.clear(); if (schemaType.has("custom")) type.custom.putAll(schemaType.getJSONObject("custom").toMap()); diff --git a/engine/src/main/java/com/arcadedb/schema/Property.java b/engine/src/main/java/com/arcadedb/schema/Property.java index f9791bce15..12ffb93c8f 100644 --- a/engine/src/main/java/com/arcadedb/schema/Property.java +++ b/engine/src/main/java/com/arcadedb/schema/Property.java @@ -76,6 +76,41 @@ public interface Property { boolean isHidden(); + /** + * When true, the property's value is stored in a paired external bucket instead of inline in the primary + * record. The main record carries only a small TYPE_EXTERNAL pointer, so traversal-only queries that don't + * project this property never load its bytes. Best for vector embeddings, large strings, embedded JSON, and + * full-text payloads. + *

+ * Null semantics. {@code set("field", null)} on an EXTERNAL property does NOT consume external bucket + * space: the serializer writes an inline TYPE_NULL byte and releases any pre-existing paired blob via the + * orphan-cleanup pass. {@code set(field, null)} and {@code remove(field)} are therefore equivalent in terms + * of paired-bucket storage; they differ only in whether the property header slot is retained (set-null + * keeps it, remove drops it). Reads return null in both cases. + */ + Property setExternal(boolean external); + + boolean isExternal(); + + /** + * Compression policy for an EXTERNAL property's value: + *

    + *
  • {@code none} (default) - store raw.
    • + *
  • {@code fast} - LZ4 fast encoder. ~1.2-1.5x faster compress than Snappy on text, identical decompress + * speed regardless of tier. Best default when writes are frequent.
    • + *
  • {@code max} - LZ4 HC encoder. ~10pp smaller output than {@code fast}, 8-20x slower compress; + * decompression speed is the same as {@code fast}. Best for write-once / read-many payloads.
    • + *
  • {@code auto} - try {@code fast}; keep compressed only when it saves more than 10% of bytes. + * Cost: on no-win records (e.g. dense float32 embeddings) the work is "compress, measure, throw + * it away, fall back to raw". You pay one wasted LZ4 compress + one extra byte-array copy per record + * compared to {@code none}. Use {@code none} explicitly when the workload tips no-win consistently.
    • + *
+ * The legacy alias {@code lz4} is accepted and stored as {@code fast}. Ignored for non-EXTERNAL properties. + */ + Property setCompression(String compression); + + String getCompression(); + Property setMax(String max); String getMax(); diff --git a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java index dcde91ad2a..d9a20baf71 100644 --- a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java +++ b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java @@ -20,6 +20,8 @@ import com.arcadedb.ContextConfiguration; import com.arcadedb.GlobalConfiguration; +import com.arcadedb.compression.CompressionFactory; +import com.arcadedb.compression.LZ4Compression; import com.arcadedb.database.BaseRecord; import com.arcadedb.database.Binary; import com.arcadedb.database.DataEncryption; @@ -30,11 +32,14 @@ import com.arcadedb.database.EmbeddedDocument; import com.arcadedb.database.EmbeddedModifier; import com.arcadedb.database.EmbeddedModifierProperty; +import com.arcadedb.database.ExternalValueRecord; import com.arcadedb.database.Identifiable; import com.arcadedb.database.MutableDocument; import com.arcadedb.database.RID; import com.arcadedb.database.Record; +import com.arcadedb.engine.Bucket; import com.arcadedb.engine.Dictionary; +import com.arcadedb.engine.LocalBucket; import com.arcadedb.exception.SerializationException; import com.arcadedb.graph.Edge; import com.arcadedb.graph.EdgeSegment; @@ -45,7 +50,10 @@ import com.arcadedb.log.LogManager; import com.arcadedb.query.sql.executor.Result; import com.arcadedb.function.sql.geo.GeoUtils; +import com.arcadedb.database.BaseDocument; +import com.arcadedb.database.DocumentInternal; import com.arcadedb.schema.DocumentType; +import com.arcadedb.schema.LocalDocumentType; import com.arcadedb.schema.Property; import com.arcadedb.serializer.json.JSONObject; import com.arcadedb.utility.DateUtils; @@ -63,6 +71,7 @@ import java.time.*; import java.time.temporal.*; import java.util.*; +import java.util.concurrent.atomic.*; import java.util.logging.*; /** @@ -81,6 +90,20 @@ public class BinarySerializer { // Cached WKT writer for fast Point serialization (avoid recreating writer for each Point) private static volatile ShapeWriter cachedWktWriter; + /** + * Process-cumulative count of records whose OLD buffer could not be parsed by + * {@link #findExistingExternalRids}. Each parse failure means the record's external blobs (if any) were + * NOT discovered and could be left as orphans in the paired bucket. CHECK DATABASE surfaces this counter + * so an operator can see corruption-driven leaks accumulating without waiting for the next bucket scan. + * Static so the value survives instance churn and tracks the JVM lifetime. + */ + private static final AtomicLong externalRidScanFailures = new AtomicLong(0); + + /** Returns the JVM-cumulative count of {@link #findExistingExternalRids} parse failures since process start. */ + public static long getExternalRidScanFailures() { + return externalRidScanFailures.get(); + } + public BinarySerializer(final ContextConfiguration configuration) throws ClassNotFoundException { setDateImplementation(configuration.getValue(GlobalConfiguration.DATE_IMPLEMENTATION)); setDateTimeImplementation(configuration.getValue(GlobalConfiguration.DATE_TIME_IMPLEMENTATION)); @@ -92,6 +115,7 @@ public Binary serialize(final DatabaseInternal database, final Record record) { case Vertex.RECORD_TYPE -> serializeVertex(database, (MutableVertex) record); case Edge.RECORD_TYPE -> serializeEdge(database, (MutableEdge) record); case EdgeSegment.RECORD_TYPE -> serializeEdgeContainer((EdgeSegment) record); + case ExternalValueRecord.RECORD_TYPE -> ((ExternalValueRecord) record).getContent(); default -> throw new IllegalArgumentException("Cannot serialize a record of type=" + record.getRecordType()); }; } @@ -267,7 +291,15 @@ else if (properties == 0) final EmbeddedModifierProperty propertyModifier = embeddedModifier != null ? new EmbeddedModifierProperty(embeddedModifier.getOwner(), propertyName) : null; - final Object propertyValue = deserializeValue(database, buffer, type, propertyModifier); + final Object propertyValue; + if (isExternalType(type)) { + final int extBucketId = (int) buffer.getNumber(); + final long extPosition = buffer.getNumber(); + propertyValue = readExternalValue((DatabaseInternal) database, extBucketId, extPosition, propertyModifier, + isExternalCompressedType(type)); + } else { + propertyValue = deserializeValue(database, buffer, type, propertyModifier); + } values.put(propertyName, propertyValue); } catch (Exception e) { @@ -341,6 +373,13 @@ else if (properties == 0) final EmbeddedModifierProperty propertyModifier = embeddedModifier != null ? new EmbeddedModifierProperty(embeddedModifier.getOwner(), fieldName) : null; + if (isExternalType(type)) { + final int extBucketId = (int) buffer.getNumber(); + final long extPosition = buffer.getNumber(); + return readExternalValue((DatabaseInternal) database, extBucketId, extPosition, propertyModifier, + isExternalCompressedType(type)); + } + return deserializeValue(database, buffer, type, propertyModifier); } } catch (Exception e) { @@ -822,6 +861,12 @@ public Binary serializeProperties(final Database database, final Document record final Map properties = record.propertiesAsMap(); final Dictionary dictionary = database.getSchema().getDictionary(); final DocumentType documentType = record.getType(); + // For records being UPDATED, look up existing external RIDs from the old buffer so we can update the external bucket + // record in place rather than allocating a new one. New records (no identity yet) get an empty map. + final Map existingExternalRids = findExistingExternalRids(database, record); + // Track which existing external RIDs we re-used (kept) so we can delete the rest as orphans below. An entry is + // orphaned when the property is no longer EXTERNAL (toggled off via ALTER), was renamed, or was dropped entirely. + final Set consumedExternalProperties = existingExternalRids.isEmpty() ? null : new HashSet<>(); // Pre-resolve types so the property count matches what is actually written. // Skipping an invalid property after writing its nameId would desync the header on read. @@ -862,17 +907,65 @@ public Binary serializeProperties(final Database database, final Document record final int startContentPosition = content.position(); - if (value instanceof String stringValue && type == BinaryTypes.TYPE_STRING) { - final int id = dictionary.getIdByName(stringValue, false); - if (id > -1) { - // WRITE THE COMPRESSED STRING - type = BinaryTypes.TYPE_COMPRESSED_STRING; - value = id; + final Property propertyDef = documentType.getPropertyIfExists(propertyName); + if (propertyDef != null && propertyDef.isExternal()) { + // NULL is not externalised. set("field", null) is treated semantically the same as remove("field") + // for storage purposes: we write a TYPE_NULL byte INLINE in the main record and let the orphan- + // cleanup pass at the end of this method delete any pre-existing external blob (the property is + // intentionally NOT added to consumedExternalProperties below, so orphan cleanup runs). This keeps + // user mental model intuitive (null means "no payload") and avoids charging external-bucket space + // for null markers, which on a paired-bucket layout would otherwise force one external record per + // null-valued record. + if (value == null) { + content.putByte(BinaryTypes.TYPE_NULL); + } else { + // Externalised property: write the value to the paired external bucket and put a TYPE_EXTERNAL marker (with + // the external RID) in the main record's content. The main record stays small and traversal-only reads never + // hit the external bucket. See LocalDocumentType.getExternalBucketIdFor. + final RID identity = record.getIdentity(); + if (identity == null) + throw new SerializationException( + "Cannot serialize EXTERNAL property '" + propertyName + "' on type '" + documentType.getName() + + "': record has no target bucket. The bucket layer must set a provisional identity before serialize."); + final int primaryBucketId = identity.getBucketId(); + // Look up the external bucket via the type that ACTUALLY owns the primary bucket. This may differ from + // record.getType(): polymorphic scans (scanType POLYMORPHIC, MATCH, etc.) tag every record with the queried + // parent type even when the record physically lives in a subtype's bucket. Trusting documentType in that + // case would miss the subtype's external bucket map. + final LocalDocumentType ownerType = (LocalDocumentType) database.getSchema().getEmbedded().getTypeByBucketId(primaryBucketId); + final Integer extBucketId = (ownerType != null ? ownerType : (LocalDocumentType) documentType) + .getExternalBucketIdFor(primaryBucketId); + if (extBucketId == null) + throw new SerializationException( + "Cannot serialize EXTERNAL property '" + propertyName + "' on type '" + documentType.getName() + + "': no external bucket is paired with primary bucket " + primaryBucketId); + + final RID existingExtRid = existingExternalRids.get(propertyName); + if (consumedExternalProperties != null && existingExtRid != null) + consumedExternalProperties.add(propertyName); + + final ExternalWriteResult written = writeExternalPropertyValue((DatabaseInternal) database, extBucketId, + existingExtRid, type, value, propertyDef.getCompression()); + + // The persisted type byte tells the reader which decoder to use. Bucket id and position are varints, + // mirroring TYPE_COMPRESSED_RID, so each pointer averages 3-7 bytes vs 12 fixed. + content.putByte(written.typeByte); + content.putNumber(written.rid.getBucketId()); + content.putNumber(written.rid.getPosition()); + } + } else { + if (value instanceof String stringValue && type == BinaryTypes.TYPE_STRING) { + final int id = dictionary.getIdByName(stringValue, false); + if (id > -1) { + // WRITE THE COMPRESSED STRING + type = BinaryTypes.TYPE_COMPRESSED_STRING; + value = id; + } } - } - content.putByte(type); - serializeValue(database, content, type, value); + content.putByte(type); + serializeValue(database, content, type, value); + } // WRITE PROPERTY CONTENT POSITION header.putUnsignedNumber(startContentPosition); @@ -887,9 +980,294 @@ public Binary serializeProperties(final Database database, final Document record header.append(content); header.flip(); + + // Orphan cleanup: any existing external RID that was NOT re-used (property no longer EXTERNAL, was renamed, or was + // dropped) must be deleted from the external bucket so we don't leak storage. Same transaction as the primary write. + if (consumedExternalProperties != null) { + for (final Map.Entry entry : existingExternalRids.entrySet()) { + if (consumedExternalProperties.contains(entry.getKey())) + continue; + final RID orphanRid = entry.getValue(); + final LocalBucket externalBucket = database.getSchema().getEmbedded().getBucketById(orphanRid.getBucketId(), false); + if (externalBucket != null) { + externalBucket.deleteRecord(orphanRid); + ((DatabaseInternal) database).getTransaction().updateBucketRecordDelta(externalBucket.getFileId(), -1); + } + } + } + return header; } + /** + * Holder for {@link #writeExternalPropertyValue}: the RID where the value blob lives and the type byte the + * caller should embed in the main record. Field order matches the conceptual weight (the RID is the bulk of + * the pointer; the type byte is the 1-byte discriminator). Package-private along with the writer; tests + * reach it via {@code BinarySerializerTestHelper}. + */ + static final class ExternalWriteResult { + final RID rid; + final byte typeByte; + + ExternalWriteResult(final RID rid, final byte typeByte) { + this.rid = rid; + this.typeByte = typeByte; + } + } + + /** True for any of TYPE_EXTERNAL, TYPE_EXTERNAL_COMPRESSED_FAST, TYPE_EXTERNAL_COMPRESSED_MAX. */ + private static boolean isExternalType(final byte type) { + return type == BinaryTypes.TYPE_EXTERNAL + || type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_FAST + || type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_MAX; + } + + /** + * True only for the compressed external types. LZ4 fast and LZ4 HC share the same byte format, so the read + * path needs only the compressed/raw distinction; both compressed types decode through the same + * {@code LZ4Compression.decompress} call. + */ + private static boolean isExternalCompressedType(final byte type) { + return type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_FAST + || type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_MAX; + } + + /** + * Serialises an EXTERNAL property value per the property's compression policy, writes the resulting blob to + * the paired external bucket, and returns the type byte the caller should put in the main record. + *

+ * Policy values: + *

    + *
  • {@code none} (or null/empty) - store raw, type byte = TYPE_EXTERNAL.
    • + *
  • {@code fast} - LZ4 fast encoder, type byte = TYPE_EXTERNAL_COMPRESSED_FAST.
    • + *
  • {@code max} - LZ4 HC encoder (slower compress, ~10pp smaller), type byte = TYPE_EXTERNAL_COMPRESSED_MAX.
    • + *
  • {@code auto} - try LZ4 fast; keep only when it saves more than 10% of bytes, otherwise store raw.
    • + *
+ * The decision is per-record, so a single property may mix compressed and uncompressed records freely. + * Made package-private; tests reach it through {@code BinarySerializerTestHelper}. + */ + ExternalWriteResult writeExternalPropertyValue(final DatabaseInternal database, final int externalBucketId, + final RID existingExternalRid, final byte valueType, final Object value, final String compressionPolicy) { + if (existingExternalRid != null && existingExternalRid.getBucketId() != externalBucketId) + throw new SerializationException( + "Existing external RID " + existingExternalRid + " does not match the paired external bucket id " + + externalBucketId + " for this record. The schema's external bucket mapping is inconsistent."); + + // The "lz4" legacy alias is normalised to "fast" by LocalProperty.setCompression(), so by the time the + // policy reaches us it's already one of: null, "none", "fast", "max", "auto". No second alias check. + final boolean fastMode = "fast".equalsIgnoreCase(compressionPolicy); + final boolean maxMode = "max".equalsIgnoreCase(compressionPolicy); + final boolean autoMode = "auto".equalsIgnoreCase(compressionPolicy); + final boolean tryCompress = fastMode || maxMode || autoMode; + + final Binary blob = new Binary(); + + if (!tryCompress) { + // Fast path (no compression policy): serialise straight into the blob. Avoids the extra Binary buffer + + // toByteArray() roundtrip the compressed path needs. This is the dense-vector default (compression is a + // loss for embeddings) and the throughput-critical case for write-heavy workloads. + blob.putByte(ExternalValueRecord.RECORD_TYPE); + blob.putByte(valueType); + serializeValue(database, blob, valueType, value); + blob.flip(); + return finalizeExternalWrite(database, externalBucketId, existingExternalRid, blob, BinaryTypes.TYPE_EXTERNAL); + } + + // Compressed path: we need the raw bytes both to compress and to fall back on auto-mode no-win, so this is + // the one case where the intermediate Binary is unavoidable. + // + // AUTO-MODE OVERHEAD (accepted, not optimised). When auto-mode picks the no-win branch the work done is: + // 1. serializeValue once into rawValueBytes (always paid) + // 2. rawValueBytes.toByteArray() byte[] copy (paid even when we end up not keeping the compressed form) + // 3. lz4.compress() call (CPU spent for nothing) + // 4. blob.append(rawValueBytes) byte copy (effectively a third copy of the raw bytes) + // versus the non-compress fast path's single serialise-straight-into-blob. The serialise itself runs once, + // not twice; the cost is the extra byte[] copy + the discarded compress call. For workloads that actually + // tip into the no-win branch routinely (e.g. dense float32 embeddings), use {@code none} explicitly. Auto + // is correct for mixed workloads where most records benefit and the throughput hit on the minority is + // acceptable. We do not memoise the compressor output across calls because it is per-record state. + final Binary rawValueBytes = new Binary(); + serializeValue(database, rawValueBytes, valueType, value); + rawValueBytes.flip(); + + byte typeByte = BinaryTypes.TYPE_EXTERNAL; + byte[] compressedPayload = null; + int uncompressedSize = 0; + + if (rawValueBytes.size() > 0) { + final byte[] raw = rawValueBytes.toByteArray(); + final LZ4Compression lz4 = CompressionFactory.getLZ4(); + final byte[] compressed = maxMode ? lz4.compressMax(raw) : lz4.compress(raw); + // In auto mode skip compression unless it saves >10%. In fast/max mode always keep the compressed form + // (the user explicitly asked to compress, even if a particular record happens not to gain much). + if (!autoMode || compressed.length < raw.length * 0.9) { + typeByte = maxMode ? BinaryTypes.TYPE_EXTERNAL_COMPRESSED_MAX : BinaryTypes.TYPE_EXTERNAL_COMPRESSED_FAST; + compressedPayload = compressed; + uncompressedSize = raw.length; + } + } + + blob.putByte(ExternalValueRecord.RECORD_TYPE); + blob.putByte(valueType); + if (isExternalCompressedType(typeByte)) { + blob.putUnsignedNumber(uncompressedSize); + // Length-prefixed payload (Binary.putBytes writes a varint length followed by the bytes). The earlier + // version used putByteArray and reconstructed the length from buffer.size() - position(), which tied + // the format to "compressed bytes fill the rest of the record". Storing the length explicitly costs + // 1-4 bytes per record and lets future versions append fields (checksum, footer, etc.) without + // requiring a one-shot data migration on every external bucket. + blob.putBytes(compressedPayload); + } else { + blob.append(rawValueBytes); + } + blob.flip(); + return finalizeExternalWrite(database, externalBucketId, existingExternalRid, blob, typeByte); + } + + /** + * Inserts or updates the given blob in the paired external bucket and returns the type byte + RID for the + * caller to embed in the main record. Extracted from {@link #writeExternalPropertyValue} to keep the two + * write paths (raw / compressed) short and to avoid duplicating the bucket-side accounting. + */ + private ExternalWriteResult finalizeExternalWrite(final DatabaseInternal database, final int externalBucketId, + final RID existingExternalRid, final Binary blob, final byte typeByte) { + final LocalBucket externalBucket = database.getSchema().getEmbedded().getBucketById(externalBucketId); + final RID rid; + if (existingExternalRid == null) { + final ExternalValueRecord rec = new ExternalValueRecord(database, null, blob); + rid = externalBucket.createRecord(rec, true); + database.getTransaction().updateBucketRecordDelta(externalBucket.getFileId(), +1); + } else { + // Identity already passed to the constructor (BaseRecord stores it). No need to call setIdentity again. + final ExternalValueRecord rec = new ExternalValueRecord(database, existingExternalRid, blob); + externalBucket.updateRecord(rec, true); + rid = existingExternalRid; + } + return new ExternalWriteResult(rid, typeByte); + } + + public Object readExternalValue(final DatabaseInternal database, final int externalBucketId, final long position, + final EmbeddedModifier embeddedModifier) { + return readExternalValue(database, externalBucketId, position, embeddedModifier, false); + } + + /** + * Reads the value blob at the given external RID. When {@code compressed} is true, the blob's value-bytes are + * LZ4-compressed and prefixed by an uncompressed-size varint. The compression flag is supplied by the caller - + * usually derived from the main record's type byte. LZ4 fast and LZ4 HC share the same byte format, so this + * method does not need to know which encoder produced the bytes. + */ + public Object readExternalValue(final DatabaseInternal database, final int externalBucketId, final long position, + final EmbeddedModifier embeddedModifier, final boolean compressed) { + final LocalBucket externalBucket = database.getSchema().getEmbedded().getBucketById(externalBucketId); + if (externalBucket == null) + // Typical cause: the paired external bucket was created on a tiered path (configured via + // arcadedb.externalPropertyBucketPath) and the database was reopened with that config unset or + // pointing elsewhere, so FileManager's secondary scan never picked the file up. The schema still + // references the old bucket id, but the file isn't loaded -> we'd otherwise NPE on getRecord. + throw new SerializationException( + "Cannot read EXTERNAL property: external bucket id=" + externalBucketId + " is not loaded. " + + "If the bucket was tiered to a secondary path, set 'arcadedb.externalPropertyBucketPath' " + + "to the same value used at creation time and reopen the database."); + final RID rid = RID.create(database, externalBucketId, position); + final Binary buffer = externalBucket.getRecord(rid).copyOfContent(); + buffer.position(Binary.BYTE_SERIALIZED_SIZE); // SKIP RECORD TYPE BYTE + final byte valueType = buffer.getByte(); + if (!compressed) + return deserializeValue(database, buffer, valueType, embeddedModifier); + + final int uncompressedSize = (int) buffer.getUnsignedNumber(); + // Length-prefixed payload (Binary.getBytes reads varint length + the bytes). Trailing space (if any) is + // ignored, leaving the format extensible to a future footer/checksum without a migration. + final byte[] compressedBytes = buffer.getBytes(); + final byte[] decompressed = CompressionFactory.getLZ4().decompress(compressedBytes, uncompressedSize); + return deserializeValue(database, new Binary(decompressed), valueType, embeddedModifier); + } + + /** + * Reused by cascade-delete and the orphan-cleanup-on-update path inside {@link #serializeProperties}: scans + * the OLD buffer for TYPE_EXTERNAL pointers, keyed by property name. + *

+ * Gating check: {@code hasExternalBuckets()}, not {@code hasExternalProperties()}. + * The two are different and the distinction is load-bearing: + *

    + *
  • {@code hasExternalProperties()} reflects the CURRENT schema (count of properties currently flagged + * EXTERNAL). It flips to false the instant {@code setExternal(false)} commits, before any record has + * been re-serialised. Using it as the gate would skip orphan-cleanup during the EXTERNAL→inline + * migration window and leak external blobs.
    • + *
  • {@code hasExternalBuckets()} reflects the lifetime of the paired-bucket mapping + * ({@code externalBucketIdByPrimaryBucketId} non-empty). It stays true through the entire migration + * window: schema flag flipped → REBUILD TYPE re-serialises every record → orphan-cleanup empties the + * paired buckets → {@code reclaimEmptyExternalBuckets()} drops them and clears the map. Only at that + * point does it return false, and by then no record can carry a stale TYPE_EXTERNAL pointer.
    • + *
+ * Types that have never used the EXTERNAL feature have an empty map, so this fast-exit pays for itself on + * the (very common) update hot path of plain-vanilla types. + */ + public Map findExistingExternalRids(final Database database, final Document record) { + final RID identity = record.getIdentity(); + if (identity == null) + return Collections.emptyMap(); + if (!(record instanceof BaseDocument)) + return Collections.emptyMap(); + if (!(record.getType() instanceof LocalDocumentType ldt) || !ldt.hasExternalBuckets()) + return Collections.emptyMap(); + final Binary buf = ((BaseRecord) record).getBuffer(); + if (buf == null) + return Collections.emptyMap(); + + // Scan the record's own buffer in place (no copy). The buffer is the read-side of the record's content, + // which by the time serializeProperties reaches us has finished its own reads (it pulls the property map + // first via propertiesAsMap, which deserialises and caches into a Map, then calls us). Cascade-delete and + // CHECK DATABASE both call us outside any other buffer iteration. We still save/restore the position so + // a future caller mid-iteration would not be disturbed - cheap insurance for a hot update path. + final int savedPosition = buf.position(); + try { + buf.position(((DocumentInternal) record).getPropertiesStartingPosition()); + + final int headerEndOffset = buf.getInt(); + final int properties = (int) buf.getUnsignedNumber(); + if (properties <= 0) + return Collections.emptyMap(); + + final Dictionary dictionary = database.getSchema().getDictionary(); + Map result = null; + + for (int i = 0; i < properties; i++) { + final int nameId = (int) buf.getUnsignedNumber(); + final int contentPosition = (int) buf.getUnsignedNumber(); + final int afterHeader = buf.position(); + + buf.position(headerEndOffset + contentPosition); + final byte type = buf.getByte(); + // All three external type bytes carry the same [bucketIdVarint][positionVarint] RID; only the blob + // decoder differs (raw / LZ4 fast / LZ4 HC). Cascade-delete and orphan cleanup only need the RID. + if (isExternalType(type)) { + final int extBucketId = (int) buf.getNumber(); + final long extPosition = buf.getNumber(); + if (result == null) + result = new HashMap<>(); + result.put(dictionary.getNameById(nameId), RID.create(database, extBucketId, extPosition)); + } + + buf.position(afterHeader); + } + return result == null ? Collections.emptyMap() : result; + } catch (Exception e) { + // Bump the JVM-cumulative counter so CHECK DATABASE surfaces the leak rate without waiting for the + // next paired-bucket scan. The orphan blob (if any) will eventually be caught by the orphan scan, + // but counting failures gives an early signal that something is corrupting record buffers. + externalRidScanFailures.incrementAndGet(); + LogManager.instance().log(this, Level.WARNING, + "Could not parse old buffer to recover external RIDs for record %s: %s. External records linked to this " + + "record may be orphaned in the paired bucket. (cumulative scan failures since process start: %d)", + e, identity, e.getMessage(), externalRidScanFailures.get()); + return Collections.emptyMap(); + } finally { + buf.position(savedPosition); + } + } + public Class getDateImplementation() { return dateImplementation; } diff --git a/engine/src/main/java/com/arcadedb/serializer/BinaryTypes.java b/engine/src/main/java/com/arcadedb/serializer/BinaryTypes.java index 6abb085613..240b823ec2 100644 --- a/engine/src/main/java/com/arcadedb/serializer/BinaryTypes.java +++ b/engine/src/main/java/com/arcadedb/serializer/BinaryTypes.java @@ -65,6 +65,9 @@ public class BinaryTypes { public final static byte TYPE_ARRAY_OF_FLOATS = 26; // @SINCE 23.6.1 public final static byte TYPE_ARRAY_OF_DOUBLES = 27; // @SINCE 23.6.1 public final static byte TYPE_COMPRESSED_GEOMETRY = 28; // @SINCE 26.2.1 - Binary geometry storage (Point, Circle, Rectangle, etc.) + public final static byte TYPE_EXTERNAL = 29; // @SINCE 26.5.1 - Property value stored uncompressed in a paired external bucket. Followed by [bucketIdVarint][positionVarint]. + public final static byte TYPE_EXTERNAL_COMPRESSED_FAST = 30; // @SINCE 26.5.1 - Same as TYPE_EXTERNAL but the value bytes in the external blob are LZ4-fast-compressed; the type byte is the dispatcher (no per-blob algo marker needed). + public final static byte TYPE_EXTERNAL_COMPRESSED_MAX = 31; // @SINCE 26.5.1 - Same as TYPE_EXTERNAL_COMPRESSED_FAST but compressed with LZ4 HC (high compression, ~10pp smaller output, 8-20x slower compress; decompression is identical to FAST since LZ4 HC uses the same format). // Geometry subtypes for TYPE_COMPRESSED_GEOMETRY public final static byte GEOMETRY_SUBTYPE_POINT = 1; // Point: x(double), y(double) diff --git a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyDensitySlowTest.java b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyDensitySlowTest.java new file mode 100644 index 0000000000..57184c104c --- /dev/null +++ b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyDensitySlowTest.java @@ -0,0 +1,76 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.schema; + +import com.arcadedb.TestHelper; +import com.arcadedb.engine.LocalBucket; +import com.arcadedb.graph.MutableVertex; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Verifies that EXTERNAL property storage keeps the primary bucket dense: when a heavy property is flagged EXTERNAL, + * the primary bucket grows in proportion to the inline-only data while the heavy payload accumulates in the paired + * external bucket. This exercises the page-cache density win that motivates the feature. + * + * Tagged @slow because it inserts thousands of records with multi-KB embeddings; not run in the default suite. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +@Tag("slow") +class ExternalPropertyDensitySlowTest extends TestHelper { + + @Test + void primaryStaysSmallWhenHeavyPropertyIsExternal() { + final VertexType type = database.getSchema().createVertexType("V"); + type.createProperty("name", Type.STRING); + type.createProperty("embedding", Type.ARRAY_OF_FLOATS).setExternal(true); + + final int recordCount = 2000; + final int dim = 1024; // 4 KB per embedding + + database.transaction(() -> { + for (int i = 0; i < recordCount; i++) { + final float[] e = new float[dim]; + for (int j = 0; j < dim; j++) + e[j] = (float) (i * 0.001 + j); + final MutableVertex v = database.newVertex("V").set("name", "u" + i).set("embedding", e); + v.save(); + } + }); + + final LocalBucket primary = (LocalBucket) type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + + final long primaryPages = primary.getTotalPages(); + final long externalPages = external.getTotalPages(); + + // External bucket should hold the bulk of bytes (at least ~5x the primary bucket's page count given a 4KB + // payload per record vs. a small name+pointer per record). The exact ratio depends on page packing, so we use + // a conservative lower bound to keep the test stable. + assertThat(externalPages).as("external pages=%d primary pages=%d", externalPages, primaryPages) + .isGreaterThan(primaryPages * 3); + + // Sanity: count of records is recordCount. + assertThat(database.countType("V", false)).isEqualTo(recordCount); + } +} diff --git a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java new file mode 100644 index 0000000000..211c204ff9 --- /dev/null +++ b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java @@ -0,0 +1,1060 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.schema; + +import com.arcadedb.TestHelper; +import com.arcadedb.database.Document; +import com.arcadedb.database.EmbeddedDocument; +import com.arcadedb.database.MutableDocument; +import com.arcadedb.database.MutableEmbeddedDocument; +import com.arcadedb.database.RID; +import com.arcadedb.engine.LocalBucket; +import com.arcadedb.graph.MutableVertex; +import com.arcadedb.query.sql.executor.ResultSet; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Verifies the EXTERNAL property storage feature: when a property is flagged EXTERNAL, its value lives in a paired + * external bucket keyed by the same record's RID, while the main record only carries a TYPE_EXTERNAL pointer. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +class ExternalPropertyTest extends TestHelper { + + @Test + void flagPersistsAcrossReopen() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("name", Type.STRING); + type.createProperty("blob", Type.STRING).setExternal(true); + + assertThat(type.getProperty("blob").isExternal()).isTrue(); + assertThat(type.getProperty("name").isExternal()).isFalse(); + + database.close(); + database = factory.open(); + + final DocumentType reloaded = database.getSchema().getType("Doc"); + assertThat(reloaded.getProperty("blob").isExternal()).isTrue(); + assertThat(reloaded.getProperty("name").isExternal()).isFalse(); + } + + @Test + void valueRoundTripDocument() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("name", Type.STRING); + type.createProperty("blob", Type.STRING).setExternal(true); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Doc") + .set("name", "alice") + .set("blob", "the quick brown fox jumps over the lazy dog"); + d.save(); + saved[0] = d.getIdentity(); + }); + + // Re-read from disk to be sure we exercise the deserialization path, not an in-memory cache hit. + database.close(); + database = factory.open(); + + final MutableDocument loaded = (MutableDocument) database.lookupByRID(saved[0], true).asDocument().modify(); + assertThat(loaded.getString("name")).isEqualTo("alice"); + assertThat(loaded.getString("blob")).isEqualTo("the quick brown fox jumps over the lazy dog"); + } + + @Test + void valueRoundTripVertexLargeArray() { + final VertexType type = database.getSchema().createVertexType("V"); + type.createProperty("name", Type.STRING); + type.createProperty("embedding", Type.ARRAY_OF_FLOATS).setExternal(true); + + final float[] embedding = new float[4096]; + for (int i = 0; i < embedding.length; i++) + embedding[i] = (float) Math.sin(i); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableVertex v = database.newVertex("V") + .set("name", "v1") + .set("embedding", embedding); + v.save(); + saved[0] = v.getIdentity(); + }); + + database.close(); + database = factory.open(); + + final var loaded = database.lookupByRID(saved[0], true).asVertex(); + assertThat(loaded.getString("name")).isEqualTo("v1"); + final Object readBack = loaded.get("embedding"); + assertThat(readBack).isInstanceOf(float[].class); + final float[] readBackArr = (float[]) readBack; + assertThat(readBackArr).hasSize(embedding.length); + for (int i = 0; i < embedding.length; i++) + assertThat(readBackArr[i]).as("position %d", i).isEqualTo(embedding[i]); + } + + @Test + void pairedExternalBucketIsCreatedAndMarkedSystem() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + // For each primary bucket of the type there must be a paired external bucket marked EXTERNAL_PROPERTY. + boolean foundAtLeastOne = false; + for (final var primaryBucket : type.getBuckets(false)) { + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primaryBucket.getFileId()); + assertThat(extId).as("external bucket id for primary %d", primaryBucket.getFileId()).isNotNull(); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + assertThat(external.getPurpose()).isEqualTo(LocalBucket.Purpose.EXTERNAL_PROPERTY); + assertThat(external.getName()).isEqualTo(primaryBucket.getName() + "_ext"); + foundAtLeastOne = true; + } + assertThat(foundAtLeastOne).as("type should have at least one primary bucket").isTrue(); + + // Type's regular buckets() list must NOT include the external buckets. + for (final var b : type.getBuckets(false)) + assertThat(((LocalBucket) b).getPurpose()).isEqualTo(LocalBucket.Purpose.PRIMARY); + } + + @Test + void inheritancePropagatesPairedExternalBucketsToSubtype() { + final DocumentType parent = database.getSchema().createDocumentType("Parent"); + parent.createProperty("blob", Type.STRING).setExternal(true); + + final DocumentType child = database.getSchema().createDocumentType("Child"); + child.addSuperType("Parent"); + + // Each of Child's primary buckets must have its own paired external bucket, even though the EXTERNAL property + // is inherited (not declared on Child directly). Records of Child live in Child's primary buckets. + for (final var primaryBucket : child.getBuckets(false)) { + final Integer extId = ((LocalDocumentType) child).getExternalBucketIdFor(primaryBucket.getFileId()); + assertThat(extId).as("external bucket id for child primary %d", primaryBucket.getFileId()).isNotNull(); + } + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Child").set("blob", "child blob"); + d.save(); + saved[0] = d.getIdentity(); + }); + + database.close(); + database = factory.open(); + + final MutableDocument loaded = (MutableDocument) database.lookupByRID(saved[0], true).asDocument().modify(); + assertThat(loaded.getString("blob")).isEqualTo("child blob"); + } + + @Test + void updateOfExternalProperty() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Doc").set("blob", "v1"); + d.save(); + saved[0] = d.getIdentity(); + }); + + database.transaction(() -> { + final MutableDocument d = (MutableDocument) database.lookupByRID(saved[0], true).asDocument().modify(); + d.set("blob", "v2-the-second-revision"); + d.save(); + }); + + database.close(); + database = factory.open(); + + final MutableDocument loaded = (MutableDocument) database.lookupByRID(saved[0], true).asDocument().modify(); + assertThat(loaded.getString("blob")).isEqualTo("v2-the-second-revision"); + } + + @Test + void deleteCascadesToExternalRecord() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Doc").set("blob", "to-be-deleted"); + d.save(); + saved[0] = d.getIdentity(); + }); + + // Verify the external bucket has at least one record before delete. + final Integer extBucketId = ((LocalDocumentType) type).getExternalBucketIdFor(saved[0].getBucketId()); + final LocalBucket externalBucket = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extBucketId); + final long extCountBefore = externalBucket.count(); + assertThat(extCountBefore).isGreaterThanOrEqualTo(1L); + + database.transaction(() -> { + database.lookupByRID(saved[0], true).asDocument().delete(); + }); + + final long extCountAfter = externalBucket.count(); + assertThat(extCountAfter).as("external record should be deleted by cascade").isEqualTo(extCountBefore - 1L); + } + + @Test + void directWriteToExternalBucketIsRejected() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + final var primaryBucket = type.getBuckets(false).getFirst(); + final Integer extBucketId = ((LocalDocumentType) type).getExternalBucketIdFor(primaryBucket.getFileId()); + final LocalBucket externalBucket = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extBucketId); + + // The Java path that resolves a bucket by name must reject an external bucket. Build a fresh document and try + // to route it to the external bucket via Database.createRecord(record, bucketName). + final MutableDocument fresh = database.newDocument("Doc").set("blob", "x"); + org.assertj.core.api.Assertions.assertThatThrownBy(() -> + database.transaction(() -> + ((com.arcadedb.database.DatabaseInternal) database).createRecord(fresh, externalBucket.getName()))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("internal"); + + // SQL INSERT INTO bucket: is also rejected (by the SQL planner, since the bucket has no associated + // type). Different error message but functionally equivalent: the user cannot target the bucket. + org.assertj.core.api.Assertions.assertThatThrownBy(() -> + database.transaction(() -> + database.command("sql", "INSERT INTO bucket:" + externalBucket.getName() + " SET x = 1"))) + .isInstanceOf(Exception.class); + } + + @Test + void sqlDdlCreateAndAlterExternal() { + database.transaction(() -> { + database.command("sql", "CREATE DOCUMENT TYPE Doc"); + database.command("sql", "CREATE PROPERTY Doc.blob STRING (EXTERNAL true)"); + }); + assertThat(database.getSchema().getType("Doc").getProperty("blob").isExternal()).isTrue(); + + database.transaction(() -> { + database.command("sql", "ALTER PROPERTY Doc.blob EXTERNAL false"); + }); + assertThat(database.getSchema().getType("Doc").getProperty("blob").isExternal()).isFalse(); + } + + @Test + void alterToExternalRelocatesOnNextWrite() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING); // inline initially + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Doc").set("blob", "starts-inline"); + d.save(); + saved[0] = d.getIdentity(); + }); + + // Flip to EXTERNAL. Existing record's bytes are not rewritten yet. + type.getProperty("blob").setExternal(true); + + // Read still returns the inline value: deserializer doesn't see TYPE_EXTERNAL in the OLD bytes. + var loaded = database.lookupByRID(saved[0], true).asDocument(); + assertThat(loaded.getString("blob")).isEqualTo("starts-inline"); + + // Update the property. Re-serialize must route through the external bucket now. + database.transaction(() -> { + final MutableDocument m = (MutableDocument) database.lookupByRID(saved[0], true).asDocument().modify(); + m.set("blob", "now-external"); + m.save(); + }); + + database.close(); + database = factory.open(); + + final var reloaded = database.lookupByRID(saved[0], true).asDocument(); + assertThat(reloaded.getString("blob")).isEqualTo("now-external"); + + // The external bucket should hold the new value. + final Integer extBucketId = ((LocalDocumentType) database.getSchema().getType("Doc")) + .getExternalBucketIdFor(saved[0].getBucketId()); + assertThat(extBucketId).isNotNull(); + final LocalBucket externalBucket = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extBucketId); + assertThat(externalBucket.count()).isGreaterThanOrEqualTo(1L); + } + + @Test + void indexLookupOnExternalProperty() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("name", Type.STRING).setExternal(true); + type.createTypeIndex(Schema.INDEX_TYPE.LSM_TREE, true, "name"); + + database.transaction(() -> { + for (int i = 0; i < 50; i++) + database.newDocument("Doc").set("name", "user-" + i).save(); + }); + + final ResultSet rs = database.query("sql", "SELECT FROM Doc WHERE name = 'user-37'"); + assertThat(rs.hasNext()).isTrue(); + assertThat((String) rs.next().getProperty("name")).isEqualTo("user-37"); + assertThat(rs.hasNext()).isFalse(); + } + + @Test + void schemaBucketsViewExposesPurposeColumn() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + final var primary = type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + + final ResultSet rs = database.query("sql", "SELECT name, purpose FROM schema:buckets"); + boolean foundPrimary = false; + boolean foundExternal = false; + while (rs.hasNext()) { + final var row = rs.next(); + final String name = row.getProperty("name"); + if (name.equals(primary.getName())) { + foundPrimary = true; + assertThat((String) row.getProperty("purpose")).isEqualTo("PRIMARY"); + } else if (name.equals(external.getName())) { + foundExternal = true; + assertThat((String) row.getProperty("purpose")).isEqualTo("EXTERNAL_PROPERTY"); + } + } + assertThat(foundPrimary).as("schema:buckets should list the primary bucket").isTrue(); + assertThat(foundExternal).as("schema:buckets should list the external bucket").isTrue(); + + // The Studio buckets tab uses this exact WHERE filter to hide internal buckets. Verify it works on + // schema:buckets and excludes the EXTERNAL_PROPERTY bucket but still includes the primary one. + final ResultSet filtered = database.query("sql", + "SELECT name, purpose FROM schema:buckets WHERE purpose = 'PRIMARY' OR purpose IS NULL"); + final java.util.Set names = new java.util.HashSet<>(); + while (filtered.hasNext()) + names.add(filtered.next().getProperty("name")); + assertThat(names).contains(primary.getName()); + assertThat(names).doesNotContain(external.getName()); + } + + @Test + void rebuildTypeMovesInlineToExternal() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING); // inline initially + + final int n = 25; + database.transaction(() -> { + for (int i = 0; i < n; i++) + database.newDocument("Doc").set("blob", "payload-" + i).save(); + }); + + // Flip the flag, rebuild. + type.getProperty("blob").setExternal(true); + + database.transaction(() -> { + final ResultSet rs = database.command("sql", "REBUILD TYPE Doc"); + assertThat(rs.hasNext()).isTrue(); + final var row = rs.next(); + assertThat((Long) row.getProperty("recordsRebuilt")).isEqualTo((long) n); + }); + + // After rebuild, the external bucket should hold one record per Doc record. + final var primary = type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + assertThat(external.count()).isEqualTo((long) n); + + // Values still readable. + final ResultSet rs = database.query("sql", "SELECT blob FROM Doc ORDER BY blob"); + int counted = 0; + while (rs.hasNext()) { + final String val = rs.next().getProperty("blob"); + assertThat(val).startsWith("payload-"); + counted++; + } + assertThat(counted).isEqualTo(n); + } + + @Test + void rebuildTypeReversesExternalToInlineAndCleansOrphans() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + final int n = 15; + database.transaction(() -> { + for (int i = 0; i < n; i++) + database.newDocument("Doc").set("blob", "ext-" + i).save(); + }); + + final var primary = type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + final String externalBucketName = external.getName(); + assertThat(external.count()).isEqualTo((long) n); + + // Flip OFF and rebuild. REBUILD manages its own transaction; wrapping it in a caller transaction would + // defer the deferred-update flush past the in-statement reclaim check. + type.getProperty("blob").setExternal(false); + database.command("sql", "REBUILD TYPE Doc"); + + // Values must still be readable inline. + final ResultSet rs = database.query("sql", "SELECT blob FROM Doc"); + int counted = 0; + while (rs.hasNext()) { + assertThat((String) rs.next().getProperty("blob")).startsWith("ext-"); + counted++; + } + assertThat(counted).isEqualTo(n); + + // Bucket reclaim: the now-empty paired external bucket is dropped (no accumulation across toggle cycles) + // and the type's external-bucket map is cleared so schema.json no longer carries an externalBuckets key. + assertThat(((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId())) + .as("paired external bucket id should be cleared from the type after REBUILD") + .isNull(); + assertThat(((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId, false)) + .as("paired external bucket file should be dropped").isNull(); + assertThat(database.getSchema().existsBucket(externalBucketName)) + .as("schema should no longer expose the dropped external bucket").isFalse(); + } + + /** + * REBUILD TYPE inside a caller-supplied transaction cannot reclaim now-empty paired buckets (the queued + * record updates haven't flushed). The result row must surface a {@code warning} so the operator knows to + * re-run REBUILD outside a transaction. + */ + @Test + void rebuildTypeWarnsWhenInsideCallerTransactionAndExternalBucketsRemain() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + database.transaction(() -> { + for (int i = 0; i < 5; i++) + database.newDocument("Doc").set("blob", "v-" + i).save(); + }); + type.getProperty("blob").setExternal(false); + + database.begin(); + final ResultSet rs = database.command("sql", "REBUILD TYPE Doc"); + assertThat(rs.hasNext()).isTrue(); + final var row = rs.next(); + assertThat((String) row.getProperty("warning")) + .as("warning must be set so the operator knows reclaim was deferred") + .contains("paired external buckets were NOT reclaimed"); + database.commit(); + + // After committing the caller tx, re-running REBUILD outside any transaction must complete the reclaim. + final ResultSet rs2 = database.command("sql", "REBUILD TYPE Doc"); + assertThat(rs2.hasNext()).isTrue(); + final var row2 = rs2.next(); + assertThat((String) row2.getProperty("warning")).as("second run owns the tx and reclaims; no warning").isNull(); + assertThat(((LocalDocumentType) database.getSchema().getType("Doc")).hasExternalBuckets()) + .as("paired external buckets should be dropped after the second REBUILD").isFalse(); + } + + @Test + void rebuildTypePolymorphicWalksSubtypes() { + final DocumentType parent = database.getSchema().createDocumentType("Parent"); + parent.createProperty("blob", Type.STRING); + final DocumentType child = database.getSchema().createDocumentType("Child"); + child.addSuperType("Parent"); + + database.transaction(() -> { + database.newDocument("Parent").set("blob", "p1").save(); + database.newDocument("Child").set("blob", "c1").save(); + database.newDocument("Child").set("blob", "c2").save(); + }); + + // Toggle EXTERNAL on the inherited property and rebuild polymorphically. + parent.getProperty("blob").setExternal(true); + database.transaction(() -> { + final ResultSet rs = database.command("sql", "REBUILD TYPE Parent POLYMORPHIC"); + assertThat(rs.hasNext()).isTrue(); + assertThat((Long) rs.next().getProperty("recordsRebuilt")).isEqualTo(3L); + }); + + // Both Parent and Child external buckets should now hold their respective records. + final var parentBucket = parent.getBuckets(false).getFirst(); + final Integer parentExtId = ((LocalDocumentType) parent).getExternalBucketIdFor(parentBucket.getFileId()); + final var childBucket = child.getBuckets(false).getFirst(); + final Integer childExtId = ((LocalDocumentType) child).getExternalBucketIdFor(childBucket.getFileId()); + final var localSchema = (LocalSchema) database.getSchema().getEmbedded(); + assertThat(localSchema.getBucketById(parentExtId).count()).isEqualTo(1L); + assertThat(localSchema.getBucketById(childExtId).count()).isEqualTo(2L); + } + + @Test + void schemaTypesViewExposesExternalFlagAndPairing() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("name", Type.STRING); + type.createProperty("blob", Type.STRING).setExternal(true); + + final ResultSet rs = database.query("sql", "SELECT FROM schema:types WHERE name = 'Doc'"); + assertThat(rs.hasNext()).isTrue(); + final var row = rs.next(); + + // Per-property external flag (only emitted when true). + final var properties = (java.util.List) row.getProperty("properties"); + assertThat(properties).isNotNull(); + boolean foundBlobAsExternal = false; + boolean foundNameWithoutFlag = false; + for (final Object propObj : properties) { + final var prop = (com.arcadedb.query.sql.executor.Result) propObj; + final String name = prop.getProperty("name"); + if ("blob".equals(name)) { + assertThat((Boolean) prop.getProperty("external")).isTrue(); + foundBlobAsExternal = true; + } else if ("name".equals(name)) { + assertThat((Boolean) prop.getProperty("external")).isNull(); + foundNameWithoutFlag = true; + } + } + assertThat(foundBlobAsExternal).isTrue(); + assertThat(foundNameWithoutFlag).isTrue(); + + // Type-level externalBuckets mapping (primaryBucketName -> externalBucketName). + @SuppressWarnings("unchecked") + final java.util.Map extMap = (java.util.Map) row.getProperty("externalBuckets"); + assertThat(extMap).isNotNull().isNotEmpty(); + final String primaryName = type.getBuckets(false).getFirst().getName(); + assertThat(extMap).containsKey(primaryName); + assertThat(extMap.get(primaryName)).endsWith("_ext"); + } + + @Test + void externalBucketUsesLargerDefaultPageSize() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + // Primary bucket uses the standard 64KB page; external bucket uses the heavier 256KB page so multi-KB + // payloads (vectors, big strings) fit in a single page rather than overflowing into the chunk-chain path. + final LocalBucket primary = (LocalBucket) type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + + assertThat(primary.getPageSize()).isEqualTo(65_536); + assertThat(external.getPageSize()).isEqualTo(262_144); + assertThat(external.getPageSize()).isGreaterThan(primary.getPageSize()); + + // External buckets carry few but heavy records, so the slot table is sized down (256 vs 2048): saves about + // 7.5KB of header overhead per page (file-format version EXTERNAL_BUCKET_VERSION encodes this). + assertThat(primary.getMaxRecordsInPage()).isEqualTo(2048); + assertThat(external.getMaxRecordsInPage()).isEqualTo(256); + } + + @Test + void externalBucketPathOverridePlacesFileOnSecondaryDirectory() throws java.io.IOException { + // Persist the override on the database itself via ALTER DATABASE. This writes the value into the + // database's configuration.json so it survives close+reopen without polluting the JVM-wide + // GlobalConfiguration (which would leak into concurrent tests). + final java.nio.file.Path overrideDir = java.nio.file.Files.createTempDirectory("arcadedb-ext-tier-"); + try { + database.command("sql", + "alter database `arcadedb.externalPropertyBucketPath` '" + overrideDir.toString() + "'"); + + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Doc").set("blob", "tiered-payload"); + d.save(); + saved[0] = d.getIdentity(); + }); + + // External bucket file lives in the override directory, not the database directory. Use prefix filters + // (not hardcoded suffixes) so the assertions stay valid if the bucket file-naming convention evolves. + final var primary = type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + + // Per-database subdir is appended to the override path (so multiple DBs sharing the path can't collide). + final java.io.File tieredDbDir = new java.io.File(overrideDir.toFile(), database.getName()); + final java.io.File[] dbDirFiles = new java.io.File(database.getDatabasePath()).listFiles( + (dir, name) -> name.startsWith(external.getName() + ".")); + assertThat(dbDirFiles).as("external bucket should NOT be in the database directory") + .satisfiesAnyOf(arr -> assertThat(arr).isNull(), arr -> assertThat(arr).isEmpty()); + + final java.io.File[] tieredFiles = tieredDbDir.listFiles((dir, name) -> name.startsWith(external.getName() + ".")); + assertThat(tieredFiles).as("external bucket should be in //").isNotNull().isNotEmpty(); + + // Reopen: LocalDatabase.open() reloads configuration.json which contains our ALTER, so FileManager + // rediscovers the tiered file via the secondary scan path. + database.close(); + database = factory.open(); + + final var loaded = database.lookupByRID(saved[0], true).asDocument(); + assertThat(loaded.getString("blob")).isEqualTo("tiered-payload"); + } finally { + com.arcadedb.utility.FileUtils.deleteRecursively(overrideDir.toFile()); + } + } + + @Test + void checkDatabaseDetectsAndFixesOrphanedExternalRecords() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + database.transaction(() -> database.newDocument("Doc").set("blob", "referenced").save()); + + final var primary = type.getBuckets(false).getFirst(); + final Integer extBucketId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket externalBucket = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extBucketId); + final long extCountBefore = externalBucket.count(); + + // Inject an orphan: write a value blob directly to the external bucket, bypassing the property write path + // so no primary record references it. compression="none" keeps the blob raw so cleanup just sees a normal + // unreferenced record. The serializer write path is package-private; the test reaches it via the + // BinarySerializerTestHelper which lives in the same package under src/test/java. + database.transaction(() -> com.arcadedb.serializer.BinarySerializerTestHelper.injectOrphanExternalRecord( + ((com.arcadedb.database.DatabaseInternal) database).getSerializer(), + (com.arcadedb.database.DatabaseInternal) database, extBucketId, + com.arcadedb.serializer.BinaryTypes.TYPE_STRING, "orphan-payload", "none")); + + assertThat(externalBucket.count()).isEqualTo(extCountBefore + 1); + + // CHECK DATABASE (no FIX): reports the orphan but does not delete it. + final ResultSet rs = database.command("sql", "CHECK DATABASE"); + assertThat(rs.hasNext()).isTrue(); + final var row = rs.next(); + assertThat((Long) row.getProperty("orphanedExternalRecords")).isGreaterThanOrEqualTo(1L); + assertThat((Long) row.getProperty("orphanedExternalRecordsFixed")).isEqualTo(0L); + assertThat(externalBucket.count()).isEqualTo(extCountBefore + 1); + + // CHECK DATABASE FIX: removes the orphan. + final ResultSet rsFix = database.command("sql", "CHECK DATABASE FIX"); + assertThat(rsFix.hasNext()).isTrue(); + assertThat((Long) rsFix.next().getProperty("orphanedExternalRecordsFixed")).isGreaterThanOrEqualTo(1L); + assertThat(externalBucket.count()).isEqualTo(extCountBefore); + + // Re-running reports zero orphans. + final ResultSet rsAfter = database.command("sql", "CHECK DATABASE"); + assertThat((Long) rsAfter.next().getProperty("orphanedExternalRecords")).isEqualTo(0L); + } + + @Test + void compressedExternalPropertyRoundTripsLargeText() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("text", Type.STRING).setExternal(true).setCompression("lz4"); + + // Highly redundant text compresses to a fraction of its raw size. + final StringBuilder sb = new StringBuilder(); + final String fragment = "the quick brown fox jumps over the lazy dog "; + for (int i = 0; i < 200; i++) + sb.append(fragment); + final String value = sb.toString(); + final int rawSize = value.length(); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Doc").set("text", value); + d.save(); + saved[0] = d.getIdentity(); + }); + + database.close(); + database = factory.open(); + + // Round-trip: the compressed bytes deserialise back to the original text. + final var loaded = database.lookupByRID(saved[0], true).asDocument(); + assertThat(loaded.getString("text")).isEqualTo(value); + + // External bucket should hold less than half the raw text bytes (typical LZ4 ratio on repeated prose). + final var primary = type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) database.getSchema().getType("Doc")).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket externalBucket = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + assertThat(externalBucket.getTotalPages()).as("only one page expected for one ~9KB text record").isEqualTo(1); + // We can't easily inspect just the record bytes, but if we shipped uncompressed the record itself would be + // ~9KB; with LZ4 it will land far below. + } + + @Test + void compressionAutoSkipsWhenIncompressible() { + final VertexType type = database.getSchema().createVertexType("V"); + type.createProperty("embedding", Type.ARRAY_OF_FLOATS).setExternal(true).setCompression("auto"); + + // High-entropy float bits do not compress; auto-mode should fall back to TYPE_EXTERNAL (raw). + final float[] embedding = new float[1024]; + final java.util.Random rnd = new java.util.Random(42); + for (int i = 0; i < embedding.length; i++) + embedding[i] = rnd.nextFloat() * 1000f; + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableVertex v = database.newVertex("V").set("embedding", embedding); + v.save(); + saved[0] = v.getIdentity(); + }); + + database.close(); + database = factory.open(); + + // Reads back identical bytes whichever path was chosen. + final var loaded = database.lookupByRID(saved[0], true).asVertex(); + final float[] readBack = (float[]) loaded.get("embedding"); + assertThat(readBack).isEqualTo(embedding); + } + + @Test + void compressionFlagPersistsAcrossReopen() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("body", Type.STRING).setExternal(true).setCompression("auto"); + + assertThat(type.getProperty("body").getCompression()).isEqualToIgnoringCase("auto"); + + database.close(); + database = factory.open(); + + final DocumentType reloaded = database.getSchema().getType("Doc"); + assertThat(reloaded.getProperty("body").isExternal()).isTrue(); + assertThat(reloaded.getProperty("body").getCompression()).isEqualToIgnoringCase("auto"); + } + + @Test + void sqlDdlExternalCompression() { + database.transaction(() -> { + database.command("sql", "CREATE DOCUMENT TYPE Doc"); + database.command("sql", "CREATE PROPERTY Doc.body STRING (EXTERNAL true, COMPRESSION 'auto')"); + }); + assertThat(database.getSchema().getType("Doc").getProperty("body").getCompression()).isEqualToIgnoringCase("auto"); + + // Legacy alias: "lz4" must still be accepted and is normalised to the new "fast" tier name. + database.transaction(() -> database.command("sql", "ALTER PROPERTY Doc.body COMPRESSION 'lz4'")); + assertThat(database.getSchema().getType("Doc").getProperty("body").getCompression()).isEqualToIgnoringCase("fast"); + + database.transaction(() -> database.command("sql", "ALTER PROPERTY Doc.body COMPRESSION 'max'")); + assertThat(database.getSchema().getType("Doc").getProperty("body").getCompression()).isEqualToIgnoringCase("max"); + + database.transaction(() -> database.command("sql", "ALTER PROPERTY Doc.body COMPRESSION 'none'")); + assertThat(database.getSchema().getType("Doc").getProperty("body").getCompression()).isEqualToIgnoringCase("none"); + } + + @Test + void rollbackDiscardsBothPrimaryAndExternal() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + final var primary = type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + + final long primaryCountBefore = database.countType("Doc", false); + final long externalCountBefore = external.count(); + + database.begin(); + final MutableDocument d = database.newDocument("Doc").set("blob", "rolled-back"); + d.save(); + database.rollback(); + + // Both halves of the WAL group must be reverted: primary record AND its paired external blob. + assertThat(database.countType("Doc", false)).isEqualTo(primaryCountBefore); + assertThat(external.count()).as("external bucket count must also be unchanged after rollback") + .isEqualTo(externalCountBefore); + } + + /** + * EXTERNAL must work for top-level Type.LIST: writeExternalPropertyValue routes through serializeValue, which + * already handles TYPE_LIST. The whole list lands in the paired bucket as one blob; reads materialise it back. + */ + @Test + void valueRoundTripListProperty() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("name", Type.STRING); + type.createProperty("tags", Type.LIST).setExternal(true); + + final List tags = new ArrayList<>(); + tags.add("alpha"); + tags.add(42); + tags.add(3.14); + tags.add("very-long-string-".repeat(50)); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Doc").set("name", "doc-with-list").set("tags", tags); + d.save(); + saved[0] = d.getIdentity(); + }); + + // Reopen so we exercise the deserializer, not an in-memory buffer. + database.close(); + database = factory.open(); + + final Document loaded = database.lookupByRID(saved[0], true).asDocument(); + assertThat(loaded.getString("name")).isEqualTo("doc-with-list"); + final List readBack = loaded.getList("tags"); + assertThat(readBack).containsExactlyElementsOf(tags); + + // Update the list in-place: only the external blob changes; the main record's pointer stays valid. + final List updated = new ArrayList<>(tags); + updated.add("appended"); + database.transaction(() -> { + final MutableDocument m = database.lookupByRID(saved[0], true).asDocument().modify(); + m.set("tags", updated); + m.save(); + }); + + database.close(); + database = factory.open(); + + final List readBackUpdated = database.lookupByRID(saved[0], true).asDocument().getList("tags"); + assertThat(readBackUpdated).containsExactlyElementsOf(updated); + } + + /** + * EXTERNAL must work for top-level Type.MAP. Same deal: the map serialises through TYPE_MAP into the external + * blob. Use a LinkedHashMap so iteration order is deterministic for the assertion. + */ + @Test + void valueRoundTripMapProperty() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("name", Type.STRING); + type.createProperty("attrs", Type.MAP).setExternal(true); + + final Map attrs = new LinkedHashMap<>(); + attrs.put("city", "Rome"); + attrs.put("zip", 100); + attrs.put("score", 9.81); + attrs.put("notes", "lorem ipsum ".repeat(80)); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Doc").set("name", "doc-with-map").set("attrs", attrs); + d.save(); + saved[0] = d.getIdentity(); + }); + + database.close(); + database = factory.open(); + + final Document loaded = database.lookupByRID(saved[0], true).asDocument(); + assertThat(loaded.getString("name")).isEqualTo("doc-with-map"); + final Map readBack = loaded.getMap("attrs"); + // Use containsAllEntriesOf for order-insensitive equality on the materialised map. + assertThat(readBack).containsAllEntriesOf(attrs); + assertThat(readBack.size()).isEqualTo(attrs.size()); + + // Mutate one entry; verify the external blob is rewritten in place and the change survives reopen. + database.transaction(() -> { + final MutableDocument m = database.lookupByRID(saved[0], true).asDocument().modify(); + final Map mutated = new LinkedHashMap<>(m.getMap("attrs")); + mutated.put("zip", 200); + m.set("attrs", mutated); + m.save(); + }); + + database.close(); + database = factory.open(); + + final Map readBackUpdated = database.lookupByRID(saved[0], true).asDocument().getMap("attrs"); + assertThat(readBackUpdated.get("zip")).isEqualTo(200); + assertThat(readBackUpdated.get("city")).isEqualTo("Rome"); + } + + /** + * EXTERNAL must work for top-level Type.EMBEDDED. The embedded document's own header lives in the external blob; + * the deserializer wires the EmbeddedModifier (parent + property) on read so the embedded knows its owner. + */ + @Test + void valueRoundTripEmbeddedProperty() { + database.getSchema().createDocumentType("Address"); + final DocumentType person = database.getSchema().createDocumentType("Person"); + person.createProperty("name", Type.STRING); + person.createProperty("address", Type.EMBEDDED).setExternal(true); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument p = database.newDocument("Person").set("name", "alice"); + final MutableEmbeddedDocument addr = p.newEmbeddedDocument("Address", "address"); + addr.set("street", "Via Roma"); + addr.set("number", 7); + addr.set("city", "Rome"); + p.save(); + saved[0] = p.getIdentity(); + }); + + database.close(); + database = factory.open(); + + final Document loaded = database.lookupByRID(saved[0], true).asDocument(); + assertThat(loaded.getString("name")).isEqualTo("alice"); + final EmbeddedDocument readBack = loaded.getEmbedded("address"); + assertThat(readBack).isNotNull(); + assertThat(readBack.getString("street")).isEqualTo("Via Roma"); + assertThat(readBack.getInteger("number")).isEqualTo(7); + assertThat(readBack.getString("city")).isEqualTo("Rome"); + + // Mutate the embedded by replacing it with a fresh MutableEmbeddedDocument; the rewrite hits the external bucket. + database.transaction(() -> { + final MutableDocument m = database.lookupByRID(saved[0], true).asDocument().modify(); + final MutableEmbeddedDocument addr = m.newEmbeddedDocument("Address", "address"); + addr.set("street", "Piazza Navona"); + addr.set("number", 99); + addr.set("city", "Rome"); + m.save(); + }); + + database.close(); + database = factory.open(); + + final EmbeddedDocument readBackUpdated = database.lookupByRID(saved[0], true).asDocument().getEmbedded("address"); + assertThat(readBackUpdated.getString("street")).isEqualTo("Piazza Navona"); + assertThat(readBackUpdated.getInteger("number")).isEqualTo(99); + } + + /** + * On an EXTERNAL property, BOTH {@code set("field", null)} and {@code remove("field")} release the paired + * external blob (the bucket count drops). Rationale: charging external-bucket space for a null marker is + * surprising; users expect null to mean "no payload". The serializer special-cases null on the EXTERNAL + * write path: it writes an inline TYPE_NULL byte in the main record's content (not a TYPE_EXTERNAL pointer) + * and lets the orphan-cleanup pass delete any pre-existing external blob. + *

+ * Read-back semantics still differ: set-null leaves the property present (returns null), remove() makes the + * property absent (also reads as null on get(), but the property-header slot is gone). + */ + @Test + void externalPropertyNullAndRemoveBothReleasePairedBlob() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + final var primary = type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + + // Two records, each with an EXTERNAL value. + final RID[] saved = new RID[2]; + database.transaction(() -> { + final MutableDocument a = database.newDocument("Doc").set("blob", "value-a"); + a.save(); + saved[0] = a.getIdentity(); + final MutableDocument b = database.newDocument("Doc").set("blob", "value-b"); + b.save(); + saved[1] = b.getIdentity(); + }); + assertThat(external.count()).as("two external records expected").isEqualTo(2L); + + // set-null: orphan-cleanup releases the paired bucket entry. + database.transaction(() -> { + final MutableDocument m = database.lookupByRID(saved[0], true).asDocument().modify(); + m.set("blob", (Object) null); + m.save(); + }); + assertThat(external.count()).as("set-null releases the paired slot, count decreases").isEqualTo(1L); + assertThat(database.lookupByRID(saved[0], true).asDocument().get("blob")) + .as("set-null reads back as null").isNull(); + + // remove(): drops the property entirely, orphan-cleanup releases the bucket entry. + database.transaction(() -> { + final MutableDocument m = database.lookupByRID(saved[1], true).asDocument().modify(); + m.remove("blob"); + m.save(); + }); + assertThat(external.count()).as("remove() releases the paired slot, count decreases to zero").isEqualTo(0L); + assertThat(database.lookupByRID(saved[1], true).asDocument().get("blob")) + .as("remove() makes the property absent (and absence reads as null)").isNull(); + + // Reopen and confirm the on-disk state is consistent with what the in-memory bucket showed. + database.close(); + database = factory.open(); + final var primary2 = database.getSchema().getType("Doc").getBuckets(false).getFirst(); + final Integer extId2 = ((LocalDocumentType) database.getSchema().getType("Doc")).getExternalBucketIdFor( + primary2.getFileId()); + final LocalBucket external2 = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId2); + assertThat(external2.count()).as("post-reopen: both slots dropped (one by set-null, one by remove)") + .isEqualTo(0L); + // set-null record still reads back as null after reopen; the inline TYPE_NULL byte is the canonical marker. + assertThat(database.lookupByRID(saved[0], true).asDocument().get("blob")) + .as("set-null record reads as null even after reopen").isNull(); + } + + /** + * REBUILD TYPE on a database opened READ_ONLY must surface a clean DatabaseIsReadOnlyException (the + * implicit {@code db.begin()} cannot start a write transaction). The error must wrap to a + * CommandExecutionException that includes the read-only signal so an operator inspecting Studio sees it. + */ + @Test + void rebuildTypeOnReadOnlyDatabaseFailsCleanly() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + database.transaction(() -> database.newDocument("Doc").set("blob", "v").save()); + + database.close(); + database = factory.open(com.arcadedb.engine.ComponentFile.MODE.READ_ONLY); + try { + assertThatThrownBy(() -> database.command("sql", "REBUILD TYPE Doc")) + .isInstanceOfAny(com.arcadedb.exception.DatabaseIsReadOnlyException.class, + com.arcadedb.exception.CommandExecutionException.class) + .satisfies(t -> { + final String msg = (t.getMessage() == null ? "" : t.getMessage()) + + (t.getCause() == null || t.getCause().getMessage() == null ? "" : " " + t.getCause().getMessage()); + assertThat(msg.toLowerCase(java.util.Locale.ENGLISH)) + .as("error must mention read-only state so the operator sees the actual cause") + .containsAnyOf("read-only", "read only", "readonly"); + }); + } finally { + // TestHelper.afterTest runs CHECK DATABASE which requires write access; reopen RW so the harness can + // tear down cleanly without inheriting our READ_ONLY mode. + database.close(); + database = factory.open(); + } + } + + /** + * Defends the DML write guard against a corrupted/old schema.json where the {@code externalBuckets} key was + * lost. The on-disk '_ext' bucket file is still there, but the JSON map is empty. On reopen, the + * name-based heuristic in {@code restoreExternalBuckets} must re-tag the '_ext' bucket as + * {@code EXTERNAL_PROPERTY} so an INSERT INTO bucket:Doc_0_ext stays rejected. + */ + @Test + void heuristicRecoveryAdoptsOrphanExtBucketOnSchemaJsonMissingEntry() throws java.io.IOException { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + database.transaction(() -> database.newDocument("Doc").set("blob", "v").save()); + + final var primary = type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket extBucket = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + final String extBucketName = extBucket.getName(); + + database.close(); + + // Strip the externalBuckets key from schema.json to simulate an older snapshot or partial corruption. + final java.io.File schemaJson = new java.io.File(database.getDatabasePath(), "schema.json"); + String content = java.nio.file.Files.readString(schemaJson.toPath()); + final com.arcadedb.serializer.json.JSONObject schema = new com.arcadedb.serializer.json.JSONObject(content); + final com.arcadedb.serializer.json.JSONObject docType = schema.getJSONObject("types").getJSONObject("Doc"); + docType.remove("externalBuckets"); + java.nio.file.Files.writeString(schemaJson.toPath(), schema.toString()); + + database = factory.open(); + + // Heuristic recovery should have re-tagged the _ext bucket; the user-bucket DML guard now rejects writes. + // Use the Java path (more deterministic error: IllegalArgumentException with "internal" in the message). + final MutableDocument fresh = database.newDocument("Doc").set("blob", "x"); + assertThatThrownBy(() -> database.transaction(() -> + ((com.arcadedb.database.DatabaseInternal) database).createRecord(fresh, extBucketName))) + .as("the heuristic must re-tag the bucket so user DML is still refused after schema.json loss") + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("internal"); + + // The mapping is back in memory; record reads on the type still resolve correctly. + final ResultSet rs = database.query("sql", "SELECT blob FROM Doc"); + assertThat(rs.hasNext()).isTrue(); + assertThat((String) rs.next().getProperty("blob")).isEqualTo("v"); + } +} diff --git a/engine/src/test/java/com/arcadedb/serializer/BinarySerializerTestHelper.java b/engine/src/test/java/com/arcadedb/serializer/BinarySerializerTestHelper.java new file mode 100644 index 0000000000..72ba1c00f0 --- /dev/null +++ b/engine/src/test/java/com/arcadedb/serializer/BinarySerializerTestHelper.java @@ -0,0 +1,44 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.serializer; + +import com.arcadedb.database.DatabaseInternal; +import com.arcadedb.database.RID; + +/** + * Test-only entry point for serializer internals. Lives in {@code src/test/java} under the + * {@code com.arcadedb.serializer} package so it can reach the package-private write path + * {@link BinarySerializer#writeExternalPropertyValue} without that method having to be public on the + * production API surface. Use only from regression tests. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public final class BinarySerializerTestHelper { + private BinarySerializerTestHelper() { + } + + /** + * Writes a value blob directly into a paired external bucket, bypassing the property write path. Used by + * orphan-cleanup tests to plant a record that no primary record references. + */ + public static RID injectOrphanExternalRecord(final BinarySerializer serializer, final DatabaseInternal database, + final int externalBucketId, final byte valueType, final Object value, final String compression) { + return serializer.writeExternalPropertyValue(database, externalBucketId, null, valueType, value, compression).rid; + } +} diff --git a/ha-raft/src/test/java/com/arcadedb/server/ha/raft/RaftLeaderCrashWithExternalPropertyIT.java b/ha-raft/src/test/java/com/arcadedb/server/ha/raft/RaftLeaderCrashWithExternalPropertyIT.java new file mode 100644 index 0000000000..9be41b2343 --- /dev/null +++ b/ha-raft/src/test/java/com/arcadedb/server/ha/raft/RaftLeaderCrashWithExternalPropertyIT.java @@ -0,0 +1,244 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.server.ha.raft; + +import com.arcadedb.ContextConfiguration; +import com.arcadedb.GlobalConfiguration; +import com.arcadedb.database.Database; +import com.arcadedb.database.RID; +import com.arcadedb.engine.LocalBucket; +import com.arcadedb.graph.MutableVertex; +import com.arcadedb.graph.Vertex; +import com.arcadedb.log.LogManager; +import com.arcadedb.schema.LocalDocumentType; +import com.arcadedb.schema.LocalSchema; +import com.arcadedb.schema.Type; +import com.arcadedb.schema.VertexType; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Verifies primary + paired external bucket pages are replicated atomically when the leader crashes mid-commit. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +@Tag("slow") +class RaftLeaderCrashWithExternalPropertyIT extends BaseRaftHATest { + + private static final String VERTEX_TYPE = "ExtCrash"; + private static final String EXT_PROPERTY = "blob"; + private static final String INJECTED_PAYLOAD = "injected-payload-the-quick-brown-fox-jumps-over-the-lazy-dog"; + + @Override + protected void onServerConfiguration(final ContextConfiguration config) { + super.onServerConfiguration(config); + config.setValue(GlobalConfiguration.HA_QUORUM, "majority"); + } + + @Override + protected int getServerCount() { + return 3; + } + + @Override + protected boolean persistentRaftStorage() { + return true; + } + + @AfterEach + void clearPostReplicationHook() { + RaftReplicatedDatabase.TEST_POST_REPLICATION_HOOK = null; + } + + @Test + void externalPropertyRecoversAtomicallyAfterLeaderCrashBetweenReplicationAndPhase2() throws Exception { + final int leaderIndex = findLeaderIndex(); + assertThat(leaderIndex).as("A Raft leader must be elected").isGreaterThanOrEqualTo(0); + + final Database leaderDb = getServerDatabase(leaderIndex, getDatabaseName()); + + // Phase 1: schema + baseline writes with no fault injection. Each save touches BOTH the primary bucket + // and the paired external bucket in a single transaction. + leaderDb.transaction(() -> { + if (!leaderDb.getSchema().existsType(VERTEX_TYPE)) { + final VertexType type = leaderDb.getSchema().createVertexType(VERTEX_TYPE); + type.createProperty(EXT_PROPERTY, Type.STRING).setExternal(true); + } + }); + final int baseline = 50; + leaderDb.transaction(() -> { + for (int i = 0; i < baseline; i++) { + final MutableVertex v = leaderDb.newVertex(VERTEX_TYPE); + v.set("name", "baseline-" + i); + v.set(EXT_PROPERTY, "baseline-blob-" + i); + v.save(); + } + }); + assertClusterConsistency(); + assertExternalBucketsAlignWithPrimary(baseline); + + // Phase 2: arm the fault-injection hook. Single-shot: fires on the next successful Raft replication and + // (a) stops the leader on a separate thread (stopping inline would deadlock the Ratis gRPC channel), + // (b) throws so commit2ndPhase() never runs locally on the crashed leader. + final AtomicBoolean hookFired = new AtomicBoolean(false); + final CountDownLatch leaderStopped = new CountDownLatch(1); + RaftReplicatedDatabase.TEST_POST_REPLICATION_HOOK = dbName -> { + if (!hookFired.compareAndSet(false, true)) + return; + LogManager.instance().log(RaftLeaderCrashWithExternalPropertyIT.class, Level.INFO, + "TEST: fault-injection hook firing for db=%s, stopping leader %d asynchronously", + dbName, leaderIndex); + final Thread stopper = new Thread(() -> { + try { + getServer(leaderIndex).stop(); + } catch (final Throwable t) { + LogManager.instance().log(RaftLeaderCrashWithExternalPropertyIT.class, Level.WARNING, + "TEST: async leader stop failed: %s", t.getMessage()); + } finally { + leaderStopped.countDown(); + } + }, "TEST-fault-injection-stop"); + stopper.setDaemon(true); + stopper.start(); + throw new RuntimeException( + "TEST fault injection: simulated leader crash between Raft commit and commit2ndPhase"); + }; + + // Phase 3: write a record whose EXTERNAL property has a recognisable payload. The transaction commits + // BOTH primary and external bucket pages; the fault fires AFTER Raft has replicated both to followers. + final RID[] injectedRid = new RID[1]; + try { + leaderDb.begin(); + final MutableVertex v = leaderDb.newVertex(VERTEX_TYPE); + v.set("name", "injected-0"); + v.set(EXT_PROPERTY, INJECTED_PAYLOAD); + v.save(); + injectedRid[0] = v.getIdentity(); + leaderDb.commit(); + } catch (final Exception expected) { + LogManager.instance().log(this, Level.INFO, + "TEST: leader commit failed as expected: %s", expected.getMessage()); + } + + assertThat(hookFired.get()).as("Fault-injection hook must have fired").isTrue(); + assertThat(leaderStopped.await(30, TimeUnit.SECONDS)) + .as("Async leader stop must complete within 30s").isTrue(); + + // Phase 4: a new leader must emerge from the 2 survivors. + final int newLeaderIndex = waitForNewLeader(leaderIndex); + assertThat(newLeaderIndex).as("A new leader must be elected").isGreaterThanOrEqualTo(0); + assertThat(newLeaderIndex).as("New leader must differ from crashed leader").isNotEqualTo(leaderIndex); + LogManager.instance().log(this, Level.INFO, "TEST: new leader elected: server %d", newLeaderIndex); + + // Phase 5: the injected record AND its EXTERNAL value must be visible on the new leader. This proves the + // primary AND external bucket pages were replicated atomically before the leader crashed (and that + // Raft followers applied them as a unit). + waitForReplicationIsCompleted(newLeaderIndex); + final Database newLeaderDb = getServerDatabase(newLeaderIndex, getDatabaseName()); + assertThat(newLeaderDb.countType(VERTEX_TYPE, true)) + .as("New leader must have baseline + injected record") + .isEqualTo(baseline + 1L); + final Vertex injectedOnNewLeader = newLeaderDb.lookupByRID(injectedRid[0], true).asVertex(); + assertThat(injectedOnNewLeader.getString(EXT_PROPERTY)) + .as("EXTERNAL value must be readable on the new leader (atomic primary+external replication)") + .isEqualTo(INJECTED_PAYLOAD); + + // Phase 6: restart the crashed leader. Its Raft log has the committed entry (primary+external pages) but + // commit2ndPhase() never ran, so neither half is on disk. Ratis replays the entry through the state + // machine follower path, applying both halves in lock-step. + Thread.sleep(2_000); + LogManager.instance().log(this, Level.INFO, "TEST: restarting old leader %d", leaderIndex); + getServer(leaderIndex).start(); + + waitForReplicationIsCompleted(leaderIndex); + + // Phase 7: the recovered old leader must hold both halves. Specifically: + // - countType reflects the injected record (primary bucket recovered) + // - reading the EXTERNAL property returns the original payload (external bucket recovered) + // - the external bucket count matches the primary bucket count (no half-records, no orphans) + final Database oldLeaderDb = getServerDatabase(leaderIndex, getDatabaseName()); + assertThat(oldLeaderDb.countType(VERTEX_TYPE, true)) + .as("Recovered old leader must have baseline + injected record") + .isEqualTo(baseline + 1L); + final Vertex injectedOnOldLeader = oldLeaderDb.lookupByRID(injectedRid[0], true).asVertex(); + assertThat(injectedOnOldLeader.getString(EXT_PROPERTY)) + .as("EXTERNAL value must be readable on recovered old leader (atomic WAL replay of both buckets)") + .isEqualTo(INJECTED_PAYLOAD); + assertExternalBucketsAlignWithPrimary(baseline + 1); + + // Phase 8: cross-node page-level convergence (already counts external buckets - they are regular components). + assertClusterConsistency(); + } + + /** Catches half-record corruption: each primary bucket and its paired ext bucket must hold the same row count. */ + private void assertExternalBucketsAlignWithPrimary(final int expectedRecordCount) { + for (int i = 0; i < getServerCount(); i++) { + if (getServer(i) == null || !getServer(i).isStarted()) + continue; + final Database db = getServerDatabase(i, getDatabaseName()); + final LocalDocumentType type = (LocalDocumentType) db.getSchema().getType(VERTEX_TYPE); + final LocalSchema schema = (LocalSchema) db.getSchema().getEmbedded(); + long primaryTotal = 0; + long externalTotal = 0; + for (final var b : type.getBuckets(false)) { + primaryTotal += b.count(); + final Integer extId = type.getExternalBucketIdFor(b.getFileId()); + assertThat(extId).as("type '%s' on server %d primary bucket '%s' must have a paired external bucket", + VERTEX_TYPE, i, b.getName()).isNotNull(); + final LocalBucket extBucket = schema.getBucketById(extId); + externalTotal += extBucket.count(); + } + assertThat(primaryTotal) + .as("server %d: primary bucket count for '%s'", i, VERTEX_TYPE) + .isEqualTo(expectedRecordCount); + assertThat(externalTotal) + .as("server %d: external bucket count must equal primary bucket count for '%s' " + + "(half-record indicates WAL replay applied only one half of the transaction)", i, VERTEX_TYPE) + .isEqualTo(expectedRecordCount); + } + } + + private int waitForNewLeader(final int excludeIndex) { + final long deadline = System.currentTimeMillis() + 30_000; + while (System.currentTimeMillis() < deadline) { + for (int i = 0; i < getServerCount(); i++) { + if (i == excludeIndex) + continue; + final RaftHAPlugin plugin = getRaftPlugin(i); + if (plugin != null && plugin.isLeader()) + return i; + } + try { + Thread.sleep(500); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + return -1; + } + } + return -1; + } +} diff --git a/network/src/main/java/com/arcadedb/remote/RemoteProperty.java b/network/src/main/java/com/arcadedb/remote/RemoteProperty.java index 040e7fb8e6..ec40d8a913 100644 --- a/network/src/main/java/com/arcadedb/remote/RemoteProperty.java +++ b/network/src/main/java/com/arcadedb/remote/RemoteProperty.java @@ -71,6 +71,16 @@ public Property setHidden(boolean hidden) { throw new UnsupportedOperationException(); } + @Override + public Property setExternal(boolean external) { + throw new UnsupportedOperationException(); + } + + @Override + public Property setCompression(String compression) { + throw new UnsupportedOperationException(); + } + @Override public Property setMax(String max) { throw new UnsupportedOperationException(); @@ -106,6 +116,10 @@ void reload(final Map entry) { max = (String) entry.get("max"); if (entry.containsKey("hidden")) hidden = (Boolean) entry.get("hidden"); + if (entry.containsKey("external")) + external = (Boolean) entry.get("external"); + if (entry.containsKey("compression")) + compression = (String) entry.get("compression"); if (entry.containsKey("default")) defaultValue = entry.get("default"); if (entry.containsKey("regexp")) diff --git a/studio/src/main/resources/static/js/studio-database.js b/studio/src/main/resources/static/js/studio-database.js index 85b522363f..98afbc9a77 100644 --- a/studio/src/main/resources/static/js/studio-database.js +++ b/studio/src/main/resources/static/js/studio-database.js @@ -1065,8 +1065,20 @@ function createProperty(typeName) { html += "

"; html += "
"; html += "
"; + html += "
"; html += "
"; html += ""; + // Compression policy for EXTERNAL properties. Hidden by default; revealed when External is ticked. + html += ""; globalPrompt("Add Property to " + escapeHtml(typeName), html, "Create", function () { let name = $("#inputCreatePropName").val().trim(); @@ -1085,6 +1097,8 @@ function createProperty(typeName) { let notNull = $("#inputCreatePropNotNull").prop("checked"); let hidden = $("#inputCreatePropHidden").prop("checked"); let readOnly = $("#inputCreatePropReadOnly").prop("checked"); + let external = $("#inputCreatePropExternal").prop("checked"); + let compression = $("#inputCreatePropCompression").val(); let ifNotExists = $("#inputCreatePropIfNotExists").prop("checked"); let command = "CREATE PROPERTY `" + typeName + "`.`" + name + "`"; @@ -1097,6 +1111,8 @@ function createProperty(typeName) { if (notNull) constraints.push("NOTNULL true"); if (hidden) constraints.push("HIDDEN true"); if (readOnly) constraints.push("READONLY true"); + if (external) constraints.push("EXTERNAL true"); + if (external && compression && compression !== "none") constraints.push("COMPRESSION '" + compression + "'"); if (defaultVal != "") constraints.push("DEFAULT " + defaultVal); if (min != "") constraints.push("MIN " + min); if (max != "") constraints.push("MAX " + max); @@ -1128,8 +1144,17 @@ function createProperty(typeName) { else $("#createPropRegexpRow").hide(); } + function updateCompressionVisibility() { + // Compression policy is meaningful only for EXTERNAL properties. + if ($("#inputCreatePropExternal").prop("checked")) + $("#createPropCompressionRow").show(); + else + $("#createPropCompressionRow").hide(); + } $("#inputCreatePropType").on("change", updatePropVisibility); + $("#inputCreatePropExternal").on("change", updateCompressionVisibility); updatePropVisibility(); + updateCompressionVisibility(); }, 100); } @@ -2669,48 +2694,20 @@ function browseType(typeName) { let limit = parseInt($("#inputLimit").val()) || 100; let query = "select from `" + typeName + "`"; - // If a graph already exists, append results to it - if (globalCy != null && globalResultset != null) { - $("#inputLanguage").val("sql"); - editor.setValue(query); - globalActivateTab("tab-query"); - globalActivateTab("tab-graph"); + $("#inputLanguage").val("sql"); + editor.setValue(query); + globalActivateTab("tab-query"); - $("#executeSpinner").show(); + let activeTab = $("#tabs-command .active").attr("id"); + let onGraph = (activeTab == "tab-graph-sel"); - jQuery.ajax({ - type: "POST", - url: "api/v1/command/" + database, - data: JSON.stringify({ language: "sql", command: escapeHtml(query), limit: limit, serializer: "studio" }), - beforeSend: function(xhr) { xhr.setRequestHeader("Authorization", globalCredentials); } - }).done(function(data) { - $("#executeSpinner").hide(); - appendToGraph(data.result); - $("#result-num").html(globalResultset.vertices.length + " vertices, " + globalTotalEdges + " edges"); - $("#resultJson").val(JSON.stringify({ result: globalResultset }, null, 2)); - }).fail(function(jqXHR) { - $("#executeSpinner").hide(); - globalNotifyError(jqXHR.responseText); - }); + // Honor the user's currently active result tab. Only on the graph tab + // do we keep the append-to-existing-graph behavior. + if (!onGraph) { + executeCommand("sql", query); return; } - // No existing graph - use normal executeCommand flow - executeCommand("sql", query); -} - -function browseType(typeName) { - let database = getCurrentDatabase(); - if (!database) return; - - let limit = parseInt($("#inputLimit").val()) || 100; - let query = "select from `" + typeName + "`"; - - $("#inputLanguage").val("sql"); - editor.setValue(query); - globalActivateTab("tab-query"); - globalActivateTab("tab-graph"); - $("#executeSpinner").show(); jQuery.ajax({ @@ -2723,11 +2720,9 @@ function browseType(typeName) { $("#resultJson").val(JSON.stringify(data, null, 2)); if (globalCy != null && globalResultset != null && data.result.vertices.length > 0) { - // Append to existing graph appendToGraph(data.result); $("#result-num").html(globalResultset.vertices.length + " vertices, " + globalTotalEdges + " edges"); } else { - // First click or no vertices - create fresh graph globalResultset = data.result; globalCy = null; $("#result-num").html(data.result.records.length); @@ -3113,7 +3108,7 @@ function showTypeDetail(typeName) { } else { html += "
"; html += ""; - html += ""; + html += ""; html += "" + propHtml + "
NameDefined InTypeMandatoryNot NullHiddenRead OnlyDefaultMinMaxRegexpIndexesActions
NameDefined InTypeStorageMandatoryNot NullHiddenRead OnlyDefaultMinMaxRegexpIndexesActions
"; } html += ""; @@ -3320,6 +3315,28 @@ function renderProperties(row, results) { let property = row.properties[k]; panelHtml += "" + property.name + "" + row.name + "" + property.type + ""; + // Storage cell: External properties show a purple badge with the paired bucket(s) in the tooltip so the user + // immediately sees that the value lives outside the primary record and where to find it on disk. + if (property.external) { + let pairs = []; + if (row.externalBuckets) { + for (let primary in row.externalBuckets) + pairs.push(escapeHtml(primary) + " \u2192 " + escapeHtml(row.externalBuckets[primary])); + } + let tooltip = pairs.length > 0 + ? "Value stored in paired external bucket(s): " + pairs.join(", ") + : "Value stored in a paired external bucket"; + // Compression policy lives on the property; only emitted by schema:types when not 'none'. + let cmpLabel = ""; + if (property.compression) { + tooltip += ". Compression: " + property.compression; + cmpLabel = " (" + escapeHtml(property.compression) + ")"; + } + panelHtml += " External" + cmpLabel + ""; + } else { + panelHtml += "Inline"; + } + panelHtml += "" + (property.mandatory ? true : false) + ""; panelHtml += "" + (property.notNull ? true : false) + ""; panelHtml += "" + (property.hidden ? true : false) + ""; @@ -3346,7 +3363,7 @@ function renderProperties(row, results) { if (property.custom != null && Object.keys(property.custom).length > 0) { panelHtml += ""; - panelHtml += "Custom Properties
"; + panelHtml += "Custom Properties
"; panelHtml += "
"; panelHtml += ""; for (c in property.custom) panelHtml += ""; @@ -4566,7 +4583,7 @@ function showMaterializedViewDetail(viewName) { else { html += "
"; html += "
" + c + "" + property.custom[c] + "
"; - html += ""; + html += ""; html += "" + propHtml + "
NameDefined InTypeMandatoryNot NullHiddenRead OnlyDefaultMinMaxRegexpIndexesActions
NameDefined InTypeStorageMandatoryNot NullHiddenRead OnlyDefaultMinMaxRegexpIndexesActions
"; } html += ""; @@ -5198,7 +5215,9 @@ function loadStorageBuckets() { jQuery.ajax({ type: "POST", url: "api/v1/query/" + database, - data: JSON.stringify({ language: "sql", command: "SELECT FROM schema:buckets" }), + // Hide non-PRIMARY buckets (e.g. paired EXTERNAL_PROPERTY buckets that hold externalised property values). + // Power users can still see them by running SELECT FROM schema:buckets directly in the Query tab. + data: JSON.stringify({ language: "sql", command: "SELECT FROM schema:buckets WHERE purpose = 'PRIMARY' OR purpose IS NULL" }), beforeSend: function (xhr) { xhr.setRequestHeader("Authorization", globalCredentials); }, }).done(function (data) { if (!data.result || data.result.length === 0) {