From 47b429aa8743dafbd7f0f6f9ad871b0f854540d2 Mon Sep 17 00:00:00 2001 From: Luca Garulli Date: Tue, 28 Apr 2026 23:13:41 -0400 Subject: [PATCH 01/12] feat: supported external properties phase 0 Issue #4027 --- .../arcadedb/query/sql/grammar/SQLParser.g4 | 13 + .../com/arcadedb/database/BaseDocument.java | 10 + .../database/ExternalValueRecord.java | 66 +++ .../com/arcadedb/database/LocalDatabase.java | 36 +- .../arcadedb/database/MutableDocument.java | 9 + .../java/com/arcadedb/engine/LocalBucket.java | 24 + .../query/sql/antlr/SQLASTBuilder.java | 13 + .../FetchFromSchemaBucketDetailStep.java | 1 + .../executor/FetchFromSchemaBucketsStep.java | 5 + .../sql/parser/AlterPropertyStatement.java | 3 + .../CreatePropertyAttributeStatement.java | 2 + .../sql/parser/RebuildTypeStatement.java | 133 +++++ .../com/arcadedb/schema/AbstractProperty.java | 13 + .../com/arcadedb/schema/DocumentType.java | 2 + .../arcadedb/schema/LocalDocumentType.java | 97 ++++ .../com/arcadedb/schema/LocalProperty.java | 14 + .../java/com/arcadedb/schema/LocalSchema.java | 10 + .../java/com/arcadedb/schema/Property.java | 4 + .../arcadedb/serializer/BinarySerializer.java | 194 +++++++- .../com/arcadedb/serializer/BinaryTypes.java | 1 + .../ExternalPropertyDensitySlowTest.java | 76 +++ .../arcadedb/schema/ExternalPropertyTest.java | 455 ++++++++++++++++++ .../com/arcadedb/remote/RemoteProperty.java | 7 + 23 files changed, 1176 insertions(+), 12 deletions(-) create mode 100644 engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java create mode 100644 engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java create mode 100644 engine/src/test/java/com/arcadedb/schema/ExternalPropertyDensitySlowTest.java create mode 100644 engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java diff --git a/engine/src/main/antlr4/com/arcadedb/query/sql/grammar/SQLParser.g4 b/engine/src/main/antlr4/com/arcadedb/query/sql/grammar/SQLParser.g4 index 2d1df1db03..005e348534 100644 --- a/engine/src/main/antlr4/com/arcadedb/query/sql/grammar/SQLParser.g4 +++ b/engine/src/main/antlr4/com/arcadedb/query/sql/grammar/SQLParser.g4 @@ -136,6 +136,9 @@ statement // Index Management | rebuildIndexStatement # rebuildIndexStmt + // Type Re-serialisation (e.g. relocate property values after toggling EXTERNAL flag) + | REBUILD TYPE rebuildTypeBody # rebuildTypeStmt + // Transaction Statements | beginStatement # beginStmt | commitStatement # commitStmt @@ -868,6 +871,16 @@ rebuildIndexStatement : REBUILD INDEX (identifier | STAR) (WITH identifier EQ expression (COMMA identifier EQ expression)*)? ; +/** + * REBUILD TYPE statement + * Re-serialises every record of a type so that schema changes (e.g. toggling a property's EXTERNAL flag) are applied + * to existing records on disk. POLYMORPHIC additionally walks subtypes. + * Syntax: REBUILD TYPE typeName [POLYMORPHIC] + */ +rebuildTypeBody + : identifier POLYMORPHIC? + ; + // ============================================================================ // CONTROL FLOW STATEMENTS // ============================================================================ diff --git a/engine/src/main/java/com/arcadedb/database/BaseDocument.java b/engine/src/main/java/com/arcadedb/database/BaseDocument.java index 2a869266d9..ad51c6c5db 100644 --- a/engine/src/main/java/com/arcadedb/database/BaseDocument.java +++ b/engine/src/main/java/com/arcadedb/database/BaseDocument.java @@ -157,6 +157,16 @@ public DocumentType getType() { return type; } + /** + * Returns the byte offset in the record buffer where the property header begins (i.e. immediately after the record-type + * byte and any record-kind prefix such as the 24 bytes of in/out edge pointers on a vertex). Used by the serializer to + * recover the previous serialized bytes when re-serializing an updated record (e.g. to reuse the existing RID of an + * EXTERNAL property's paired record). + */ + public int getPropertiesStartingPosition() { + return propertiesStartingPosition; + } + public String getTypeName() { return type.getName(); } diff --git a/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java b/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java new file mode 100644 index 0000000000..c32f0b93fa --- /dev/null +++ b/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java @@ -0,0 +1,66 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.database; + +import com.arcadedb.serializer.json.JSONObject; + +/** + * Lightweight record used by the serializer to store the value of a property flagged EXTERNAL in a paired bucket. + * Holds an opaque pre-serialized buffer of the form `[RECORD_TYPE_EXTERNAL][value type byte][value bytes]`. The + * leading record-type byte follows the convention of edge segments: the buffer is written verbatim by the bucket and + * we do the framing ourselves. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public class ExternalValueRecord extends BaseRecord implements RecordInternal { + public static final byte RECORD_TYPE = 5; + + public ExternalValueRecord(final Database database, final RID rid, final Binary buffer) { + super(database, rid, buffer); + } + + @Override + public byte getRecordType() { + return RECORD_TYPE; + } + + @Override + public void setIdentity(final RID rid) { + this.rid = upgradeRID(rid); + } + + @Override + public void unsetDirty() { + // NO-OP: BUFFER IS BUILT FRESH ON EACH SERIALIZE + } + + /** + * Returns the buffer with the RECORD_TYPE_EXTERNAL marker at byte 0 and the value blob ([type][value bytes]) following. + */ + public Binary getContent() { + return buffer; + } + + @Override + public JSONObject toJSON(final boolean includeMetadata) { + // Internal infrastructure record - no user-visible JSON form. Returning empty keeps callers like generic record + // dumpers safe even though they should never reach an EXTERNAL value record directly. + return new JSONObject(); + } +} diff --git a/engine/src/main/java/com/arcadedb/database/LocalDatabase.java b/engine/src/main/java/com/arcadedb/database/LocalDatabase.java index bd4e59cb39..9173b1f5c4 100644 --- a/engine/src/main/java/com/arcadedb/database/LocalDatabase.java +++ b/engine/src/main/java/com/arcadedb/database/LocalDatabase.java @@ -962,8 +962,15 @@ public void createRecordNoLock(final Record record, final String bucketName, fin if (bucketName == null && record instanceof Document doc) bucket = (LocalBucket) doc.getType().getBucketIdByRecord(doc, DatabaseContext.INSTANCE.getContext(databasePath).asyncMode); - else + else { bucket = (LocalBucket) schema.getBucketByName(bucketName); + // Reject direct writes to internal buckets (e.g. paired external-property buckets). They are infrastructure + // for the serializer, not user data containers; allowing user DML to write here would corrupt the schema's + // accounting of which records are real records vs. payload blobs. + if (bucket.getPurpose() != LocalBucket.Purpose.PRIMARY) + throw new IllegalArgumentException( + "Bucket '" + bucketName + "' is internal (purpose=" + bucket.getPurpose() + ") and cannot be written to directly"); + } ((RecordInternal) record).setIdentity(bucket.createRecord(record, discardRecordAfter)); @@ -1124,8 +1131,12 @@ public void deleteRecordNoLock(final Record record) { try { final LocalBucket bucket = schema.getBucketById(record.getIdentity().getBucketId()); - if (record instanceof Document document) + if (record instanceof Document document) { indexer.deleteDocument(document); + // Cascade-delete EXTERNAL property values living in paired external buckets. This must run BEFORE the primary + // record is deleted, so the buffer is still readable. Both deletes ride the same transaction. + cascadeDeleteExternalValues(document); + } if (record instanceof Edge edge) { graphEngine.deleteEdge(edge); @@ -1155,6 +1166,27 @@ public void deleteRecordNoLock(final Record record) { } } + /** + * Deletes all external-bucket records referenced by the given document's TYPE_EXTERNAL property pointers, in the same + * transaction as the primary delete. No-op if the type has no EXTERNAL properties or the document was not loaded with + * a buffer. + */ + private void cascadeDeleteExternalValues(final Document document) { + if (!(document.getType() instanceof LocalDocumentType localType)) + return; + if (!localType.hasExternalProperties()) + return; + final Map externalRids = serializer.findExistingExternalRids(this, document); + for (final RID extRid : externalRids.values()) { + final LocalBucket externalBucket = schema.getBucketById(extRid.getBucketId(), false); + if (externalBucket != null) { + externalBucket.deleteRecord(extRid); + // Keep the external bucket's count consistent (mirrors the +1 in BinarySerializer.writeExternalValue). + getTransaction().updateBucketRecordDelta(externalBucket.getFileId(), -1); + } + } + } + @Override public boolean isTransactionActive() { final Transaction tx = getTransactionIfExists(); diff --git a/engine/src/main/java/com/arcadedb/database/MutableDocument.java b/engine/src/main/java/com/arcadedb/database/MutableDocument.java index 8c20ac01cd..c10b18d00d 100644 --- a/engine/src/main/java/com/arcadedb/database/MutableDocument.java +++ b/engine/src/main/java/com/arcadedb/database/MutableDocument.java @@ -70,6 +70,15 @@ public boolean isDirty() { return dirty; } + /** + * Forces the next save to re-serialize this record even if no property has changed. Used by maintenance commands + * like `REBUILD TYPE` to apply schema layout changes (e.g. EXTERNAL flag toggles) to existing records on disk. + */ + public MutableDocument markDirty() { + dirty = true; + return this; + } + @Override public void setBuffer(final Binary buffer) { super.setBuffer(buffer); diff --git a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java index ea8bbe268a..49fdeed4ac 100644 --- a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java +++ b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java @@ -89,6 +89,17 @@ public class LocalBucket extends PaginatedComponent implements Bucket { protected final int contentHeaderSize; private final int maxRecordsInPage = DEF_MAX_RECORDS_IN_PAGE; private final AtomicLong cachedRecordCount = new AtomicLong(-1); + // Buckets are PRIMARY by default (they hold the primary records of a type and are user-targetable via DML). + // Internal kinds (e.g. EXTERNAL_PROPERTY) hold serializer infrastructure that user-facing DML must not target. + // The purpose is persisted in schema.json (per-type) and restored at load time, see LocalDocumentType. + private Purpose purpose = Purpose.PRIMARY; + + public enum Purpose { + /** Bucket holding the primary records of a type (vertex/edge/document). Targetable by user DML. */ + PRIMARY, + /** Paired infrastructure bucket holding externalised property values. NOT targetable by user DML. */ + EXTERNAL_PROPERTY + } // pageId → free-space-bytes. TreeMap ordering is unused (verified by grep), so a primitive // open-addressing map saves memory and avoids Integer boxing on every read/write/remove on // the page-allocation hot path. Bounded by MAX_PAGES_GATHER_STATS (100). Single-threaded @@ -157,9 +168,22 @@ public int getMaxRecordsInPage() { return maxRecordsInPage; } + public Purpose getPurpose() { + return purpose; + } + + public void setPurpose(final Purpose purpose) { + this.purpose = purpose; + } + @Override public RID createRecord(final Record record, final boolean discardRecordAfter) { database.checkPermissionsOnFile(fileId, SecurityDatabaseUser.ACCESS.CREATE_RECORD); + // Set a provisional identity so the serializer can resolve the target primary bucket id (used by EXTERNAL + // property handling to look up the paired external bucket). The actual position is filled in by + // createRecordInternal and overwrites this placeholder when the caller stores the returned RID. + if (record.getIdentity() == null && record instanceof RecordInternal ri) + ri.setIdentity(RID.create(database, fileId, -1L)); return createRecordInternal(record, false, discardRecordAfter); } diff --git a/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java b/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java index 7d676b9d35..c7fa1dc0d5 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java +++ b/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java @@ -5727,6 +5727,19 @@ public RebuildIndexStatement visitRebuildIndexStatement(final SQLParser.RebuildI return stmt; } + /** + * Visit REBUILD TYPE statement. + * Grammar: REBUILD TYPE typeName [POLYMORPHIC] + */ + @Override + public RebuildTypeStatement visitRebuildTypeStmt(final SQLParser.RebuildTypeStmtContext ctx) { + final RebuildTypeStatement stmt = new RebuildTypeStatement(-1); + final SQLParser.RebuildTypeBodyContext bodyCtx = ctx.rebuildTypeBody(); + stmt.typeName = (Identifier) visit(bodyCtx.identifier()); + stmt.polymorphic = bodyCtx.POLYMORPHIC() != null; + return stmt; + } + // DDL STATEMENTS - ALTER /** diff --git a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaBucketDetailStep.java b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaBucketDetailStep.java index 02945fb486..3f196290d0 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaBucketDetailStep.java +++ b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaBucketDetailStep.java @@ -57,6 +57,7 @@ public ResultSet syncPull(final CommandContext context, final int nRecords) thro r.setProperty("fileId", bucket.getFileId()); r.setProperty("pageSize", bucket.getPageSize()); r.setProperty("totalPages", bucket.getTotalPages()); + r.setProperty("purpose", bucket.getPurpose().name()); final Map checkResult = bucket.check(0, false); for (final Map.Entry entry : checkResult.entrySet()) diff --git a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaBucketsStep.java b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaBucketsStep.java index 2510bc3cd0..415963209e 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaBucketsStep.java +++ b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaBucketsStep.java @@ -19,6 +19,7 @@ package com.arcadedb.query.sql.executor; import com.arcadedb.engine.Bucket; +import com.arcadedb.engine.LocalBucket; import com.arcadedb.exception.TimeoutException; import com.arcadedb.schema.Schema; @@ -60,6 +61,10 @@ public ResultSet syncPull(final CommandContext context, final int nRecords) thro r.setProperty("name", bucket.getName()); r.setProperty("fileId", bucket.getFileId()); r.setProperty("records", context.getDatabase().countBucket(bucketName)); + // The bucket's purpose lets tooling (Studio etc.) hide or label internal buckets like paired + // external-property buckets. Filter via `WHERE purpose = 'PRIMARY'` to see only user-targetable ones. + if (bucket instanceof LocalBucket lb) + r.setProperty("purpose", lb.getPurpose().name()); context.setVariable("current", r); } diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java b/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java index ae3902bf30..6c2af1023f 100755 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java @@ -88,6 +88,9 @@ public ResultSet executeDDL(final CommandContext context) { } else if (setting.equalsIgnoreCase("hidden")) { oldValue = property.isHidden(); property.setHidden((boolean) finalValue); + } else if (setting.equalsIgnoreCase("external")) { + oldValue = property.isExternal(); + property.setExternal((boolean) finalValue); } else if (setting.equalsIgnoreCase("max")) { oldValue = property.getMax(); property.setMax("" + finalValue); diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java b/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java index 61b35f6a7b..384ab2c2cc 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java @@ -71,6 +71,8 @@ public Object setOnProperty(final Property internalProp, final CommandContext co internalProp.setNotNull((boolean) attrValue); } else if (attrName.equalsIgnoreCase("hidden")) { internalProp.setHidden((boolean) attrValue); + } else if (attrName.equalsIgnoreCase("external")) { + internalProp.setExternal((boolean) attrValue); } else if (attrName.equalsIgnoreCase("max")) { internalProp.setMax("" + attrValue); } else if (attrName.equalsIgnoreCase("min")) { diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java b/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java new file mode 100644 index 0000000000..89cd614e9b --- /dev/null +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java @@ -0,0 +1,133 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.query.sql.parser; + +import com.arcadedb.database.Database; +import com.arcadedb.database.MutableDocument; +import com.arcadedb.exception.CommandExecutionException; +import com.arcadedb.query.sql.executor.CommandContext; +import com.arcadedb.query.sql.executor.InternalResultSet; +import com.arcadedb.query.sql.executor.ResultInternal; +import com.arcadedb.query.sql.executor.ResultSet; +import com.arcadedb.schema.DocumentType; +import com.arcadedb.schema.Schema; + +import java.util.*; + +/** + * REBUILD TYPE typeName [POLYMORPHIC] + * + * Re-serialises every record of the named type (and optionally its subtypes) so that storage-layout schema changes + * are applied to records on disk. The primary use case is relocating values after toggling a property's EXTERNAL + * flag: after `ALTER PROPERTY T.p EXTERNAL true`, existing records still carry the value inline; running + * `REBUILD TYPE T` moves those values to the paired external bucket. The reverse case (EXTERNAL true -> false) is + * also handled, with orphan external records cleaned up automatically by the serializer. + * + * Commits in batches to keep memory bounded on large types. + */ +public class RebuildTypeStatement extends DDLStatement { + private static final int BATCH_SIZE = 10_000; + + public Identifier typeName; + public boolean polymorphic = false; + + public RebuildTypeStatement(final int id) { + super(id); + } + + @Override + public ResultSet executeDDL(final CommandContext context) { + final Database db = context.getDatabase(); + final Schema schema = db.getSchema(); + final DocumentType type = schema.getType(typeName.getStringValue()); + if (type == null) + throw new CommandExecutionException("Type not found: " + typeName.getStringValue()); + + final long[] count = { 0L }; + final boolean implicitTx = !db.isTransactionActive(); + if (implicitTx) + db.begin(); + + try { + db.scanType(typeName.getStringValue(), polymorphic, rec -> { + // Re-save forces re-serialization which routes property values according to the current schema (e.g. moves + // values to/from the external bucket per the current EXTERNAL flag) and triggers orphan cleanup in the + // serializer for any external pointers that no longer apply. + final MutableDocument m = (MutableDocument) rec.modify(); + m.markDirty(); + m.save(); + count[0]++; + if (count[0] % BATCH_SIZE == 0) { + db.commit(); + db.begin(); + } + return true; + }); + + if (implicitTx) + db.commit(); + } catch (Exception e) { + if (implicitTx && db.isTransactionActive()) + db.rollback(); + throw new CommandExecutionException("Error on rebuilding type '" + typeName.getStringValue() + "'", e); + } + + final ResultInternal result = new ResultInternal(db); + result.setProperty("operation", "rebuild type"); + result.setProperty("typeName", typeName.getStringValue()); + result.setProperty("polymorphic", polymorphic); + result.setProperty("recordsRebuilt", count[0]); + final InternalResultSet rs = new InternalResultSet(); + rs.add(result); + return rs; + } + + @Override + public void toString(final Map params, final StringBuilder builder) { + builder.append("REBUILD TYPE "); + typeName.toString(params, builder); + if (polymorphic) + builder.append(" POLYMORPHIC"); + } + + @Override + public RebuildTypeStatement copy() { + final RebuildTypeStatement result = new RebuildTypeStatement(-1); + result.typeName = typeName == null ? null : typeName.copy(); + result.polymorphic = polymorphic; + return result; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final RebuildTypeStatement that = (RebuildTypeStatement) o; + return polymorphic == that.polymorphic && Objects.equals(typeName, that.typeName); + } + + @Override + public int hashCode() { + int result = typeName != null ? typeName.hashCode() : 0; + result = 31 * result + (polymorphic ? 1 : 0); + return result; + } +} diff --git a/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java b/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java index 21bf2348a6..d45affb020 100644 --- a/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java +++ b/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java @@ -39,6 +39,7 @@ public abstract class AbstractProperty implements Property { protected boolean mandatory = false; protected boolean notNull = false; protected boolean hidden = false; + protected boolean external = false; protected String max = null; protected String min = null; protected String regexp = null; @@ -142,6 +143,16 @@ public boolean isHidden() { return hidden; } + /** + * Returns true if the property value is stored in a separate paired bucket (the external bucket of the type) instead of inline + * in the record. Useful for large payloads (vector embeddings, big strings, embedded JSON) so the primary bucket stays dense and + * page-cache friendly for traversal-heavy workloads. + */ + @Override + public boolean isExternal() { + return external; + } + @Override public String getMax() { return max; @@ -188,6 +199,8 @@ public JSONObject toJSON() { json.put("notNull", notNull); if (hidden) json.put("hidden", hidden); + if (external) + json.put("external", external); if (max != null) json.put("max", max); if (min != null) diff --git a/engine/src/main/java/com/arcadedb/schema/DocumentType.java b/engine/src/main/java/com/arcadedb/schema/DocumentType.java index 361dbd2350..7358231113 100644 --- a/engine/src/main/java/com/arcadedb/schema/DocumentType.java +++ b/engine/src/main/java/com/arcadedb/schema/DocumentType.java @@ -100,6 +100,8 @@ default Property createProperty(String propName, JSONObject prop) { p.setNotNull(prop.getBoolean("notNull")); if (prop.has("hidden")) p.setHidden(prop.getBoolean("hidden")); + if (prop.has("external")) + p.setExternal(prop.getBoolean("external")); if (prop.has("max")) p.setMax(prop.getString("max")); if (prop.has("min")) diff --git a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java index 4eabd66908..7226421601 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java @@ -63,6 +63,9 @@ public class LocalDocumentType implements DocumentType { protected List cachedPolymorphicBucketIds = new ArrayList<>(); // PRE COMPILED LIST TO SPEED UP RUN-TIME OPERATIONS protected BucketSelectionStrategy bucketSelectionStrategy = new RoundRobinBucketSelectionStrategy(); protected Set propertiesWithDefaultDefined = Collections.emptySet(); + // Map: primary bucket id -> external bucket id. Populated lazily when the first EXTERNAL property is + // set on the type, and persisted in schema.json under the per-type "externalBuckets" key. + protected final Map externalBucketIdByPrimaryBucketId = new ConcurrentHashMap<>(); public LocalDocumentType(final LocalSchema schema, final String name) { this.schema = schema; @@ -962,6 +965,82 @@ protected void addBucketInternal(final Bucket bucket) { } }); } + + // IF THE TYPE ALREADY HAS EXTERNAL PROPERTIES, ENSURE A PAIRED EXTERNAL BUCKET FOR THIS NEW PRIMARY BUCKET + if (hasExternalProperties()) + ensureExternalBucketFor((LocalBucket) bucket); + } + + /** + * Returns true if this type or any of its supertypes has at least one property flagged EXTERNAL. Polymorphic properties + * count: if A has an EXTERNAL property and B extends A, B is considered to have external properties. + */ + public boolean hasExternalProperties() { + for (final Property p : getPolymorphicProperties()) + if (p.isExternal()) + return true; + return false; + } + + /** + * Returns the external bucket id paired with the given primary bucket id, or null if no external bucket has been + * created for that primary bucket. + */ + public Integer getExternalBucketIdFor(final int primaryBucketId) { + return externalBucketIdByPrimaryBucketId.get(primaryBucketId); + } + + /** + * Idempotently ensures that every primary bucket of this type has a paired external bucket. Called when the first + * property transitions to EXTERNAL=true and when a new primary bucket is added to a type that already has external + * properties. + */ + public void ensureExternalBuckets() { + for (final Bucket b : buckets) + ensureExternalBucketFor((LocalBucket) b); + } + + /** + * Like {@link #ensureExternalBuckets()} but also recurses into all subtypes. Used when an EXTERNAL property is set on + * a supertype: every concrete subtype must own paired external buckets for its own primary buckets, because records + * of that subtype live in the subtype's primary buckets, not the supertype's. + */ + public void ensureExternalBucketsRecursive() { + ensureExternalBuckets(); + for (final LocalDocumentType sub : subTypes) + sub.ensureExternalBucketsRecursive(); + } + + private void ensureExternalBucketFor(final LocalBucket primary) { + if (externalBucketIdByPrimaryBucketId.containsKey(primary.getFileId())) + return; + final String extName = primary.getName() + "_ext"; + final LocalBucket external = schema.bucketMap.containsKey(extName) ? + schema.bucketMap.get(extName) : + schema.createBucket(extName); + external.setPurpose(LocalBucket.Purpose.EXTERNAL_PROPERTY); + externalBucketIdByPrimaryBucketId.put(primary.getFileId(), external.getFileId()); + } + + /** + * Internal hook called by LocalSchema after loading the type's external bucket map from JSON. Restores the + * primaryBucketId -> externalBucketId entries and stamps each external bucket with the EXTERNAL_PROPERTY purpose + * (which is transient on LocalBucket and must be re-applied on every load). + */ + void restoreExternalBuckets(final Map primaryNameToExternalName) { + externalBucketIdByPrimaryBucketId.clear(); + for (final Map.Entry entry : primaryNameToExternalName.entrySet()) { + final LocalBucket primary = schema.bucketMap.get(entry.getKey()); + final LocalBucket external = schema.bucketMap.get(entry.getValue()); + if (primary == null || external == null) { + LogManager.instance() + .log(this, Level.WARNING, "Cannot restore external bucket mapping '%s' -> '%s' for type '%s'", null, + entry.getKey(), entry.getValue(), name); + continue; + } + external.setPurpose(LocalBucket.Purpose.EXTERNAL_PROPERTY); + externalBucketIdByPrimaryBucketId.put(primary.getFileId(), external.getFileId()); + } } protected void removeBucketInternal(final Bucket bucket) { @@ -1078,6 +1157,19 @@ else if (this instanceof LocalEdgeType edgeType) { type.put("buckets", buckets); + if (!externalBucketIdByPrimaryBucketId.isEmpty()) { + // PRIMARY BUCKET NAME -> EXTERNAL BUCKET NAME. NAMES (NOT IDS) ARE PERSISTED FOR HUMAN READABILITY AND + // BECAUSE FILE IDS CAN BE REMAPPED ON FILE MIGRATION (LocalSchema.migratedFileIds). + final JSONObject extBuckets = new JSONObject(); + for (final Map.Entry e : externalBucketIdByPrimaryBucketId.entrySet()) { + final LocalBucket primary = schema.getBucketById(e.getKey(), false); + final LocalBucket external = schema.getBucketById(e.getValue(), false); + if (primary != null && external != null) + extBuckets.put(primary.getName(), external.getName()); + } + type.put("externalBuckets", extBuckets); + } + type.put("aliases", aliases); final JSONObject properties = new JSONObject(); @@ -1149,6 +1241,11 @@ DocumentType addSuperType(final DocumentType superType, final boolean createInde // UPDATE THE LIST OF POLYMORPHIC BUCKETS TREE embeddedSuperType.updatePolymorphicBucketsCache(true, cachedPolymorphicBuckets, cachedPolymorphicBucketIds); + // IF THE NEWLY-LINKED SUPERTYPE HAS ANY EXTERNAL PROPERTY (OWN OR INHERITED), THIS SUBTYPE MUST OWN PAIRED + // EXTERNAL BUCKETS FOR ITS OWN PRIMARY BUCKETS, BECAUSE RECORDS OF THIS SUBTYPE LIVE IN THIS SUBTYPE'S BUCKETS. + if (embeddedSuperType.hasExternalProperties()) + ensureExternalBucketsRecursive(); + // CREATE INDEXES AUTOMATICALLY ON PROPERTIES DEFINED IN SUPER TYPES final Collection indexes = new ArrayList<>(getAllIndexes(true)); indexes.removeAll(indexesByProperties.values()); diff --git a/engine/src/main/java/com/arcadedb/schema/LocalProperty.java b/engine/src/main/java/com/arcadedb/schema/LocalProperty.java index a017cfabdc..7d5cc93c93 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalProperty.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalProperty.java @@ -122,6 +122,20 @@ public Property setHidden(final boolean hidden) { return this; } + @Override + public Property setExternal(final boolean external) { + final boolean changed = !Objects.equals(this.external, external); + if (changed) { + this.external = external; + if (external) + // ENSURE PAIRED EXTERNAL BUCKETS EXIST FOR EVERY PRIMARY BUCKET OF THIS TYPE AND ALL SUBTYPES (records of a + // subtype live in subtype primary buckets, so each subtype needs its own paired external buckets too). + ((LocalDocumentType) owner).ensureExternalBucketsRecursive(); + owner.getSchema().getEmbedded().saveConfiguration(); + } + return this; + } + @Override public Property setMax(final String max) { final boolean changed = !Objects.equals(this.max, max); diff --git a/engine/src/main/java/com/arcadedb/schema/LocalSchema.java b/engine/src/main/java/com/arcadedb/schema/LocalSchema.java index dcc2a53adb..64a797ab8d 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalSchema.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalSchema.java @@ -1478,6 +1478,16 @@ protected synchronized void readConfiguration() { } } + // RESTORE THE primaryBucket -> externalBucket MAP BEFORE PROPERTIES ARE LOADED, SO THAT setExternal(true) ON A + // PROPERTY DOES NOT TRY TO LAZY-CREATE BUCKETS THAT ALREADY EXIST. + if (schemaType.has("externalBuckets")) { + final JSONObject extBuckets = schemaType.getJSONObject("externalBuckets"); + final Map primaryToExternal = new HashMap<>(); + for (final String primaryName : extBuckets.keySet()) + primaryToExternal.put(primaryName, extBuckets.getString(primaryName)); + type.restoreExternalBuckets(primaryToExternal); + } + type.custom.clear(); if (schemaType.has("custom")) type.custom.putAll(schemaType.getJSONObject("custom").toMap()); diff --git a/engine/src/main/java/com/arcadedb/schema/Property.java b/engine/src/main/java/com/arcadedb/schema/Property.java index f9791bce15..a887ad0413 100644 --- a/engine/src/main/java/com/arcadedb/schema/Property.java +++ b/engine/src/main/java/com/arcadedb/schema/Property.java @@ -76,6 +76,10 @@ public interface Property { boolean isHidden(); + Property setExternal(boolean external); + + boolean isExternal(); + Property setMax(String max); String getMax(); diff --git a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java index dcde91ad2a..d155819d90 100644 --- a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java +++ b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java @@ -30,11 +30,14 @@ import com.arcadedb.database.EmbeddedDocument; import com.arcadedb.database.EmbeddedModifier; import com.arcadedb.database.EmbeddedModifierProperty; +import com.arcadedb.database.ExternalValueRecord; import com.arcadedb.database.Identifiable; import com.arcadedb.database.MutableDocument; import com.arcadedb.database.RID; import com.arcadedb.database.Record; +import com.arcadedb.engine.Bucket; import com.arcadedb.engine.Dictionary; +import com.arcadedb.engine.LocalBucket; import com.arcadedb.exception.SerializationException; import com.arcadedb.graph.Edge; import com.arcadedb.graph.EdgeSegment; @@ -45,7 +48,9 @@ import com.arcadedb.log.LogManager; import com.arcadedb.query.sql.executor.Result; import com.arcadedb.function.sql.geo.GeoUtils; +import com.arcadedb.database.BaseDocument; import com.arcadedb.schema.DocumentType; +import com.arcadedb.schema.LocalDocumentType; import com.arcadedb.schema.Property; import com.arcadedb.serializer.json.JSONObject; import com.arcadedb.utility.DateUtils; @@ -92,6 +97,7 @@ public Binary serialize(final DatabaseInternal database, final Record record) { case Vertex.RECORD_TYPE -> serializeVertex(database, (MutableVertex) record); case Edge.RECORD_TYPE -> serializeEdge(database, (MutableEdge) record); case EdgeSegment.RECORD_TYPE -> serializeEdgeContainer((EdgeSegment) record); + case ExternalValueRecord.RECORD_TYPE -> ((ExternalValueRecord) record).getContent(); default -> throw new IllegalArgumentException("Cannot serialize a record of type=" + record.getRecordType()); }; } @@ -267,7 +273,14 @@ else if (properties == 0) final EmbeddedModifierProperty propertyModifier = embeddedModifier != null ? new EmbeddedModifierProperty(embeddedModifier.getOwner(), propertyName) : null; - final Object propertyValue = deserializeValue(database, buffer, type, propertyModifier); + final Object propertyValue; + if (type == BinaryTypes.TYPE_EXTERNAL) { + final int extBucketId = buffer.getInt(); + final long extPosition = buffer.getLong(); + propertyValue = readExternalValue((DatabaseInternal) database, extBucketId, extPosition, propertyModifier); + } else { + propertyValue = deserializeValue(database, buffer, type, propertyModifier); + } values.put(propertyName, propertyValue); } catch (Exception e) { @@ -341,6 +354,13 @@ else if (properties == 0) final EmbeddedModifierProperty propertyModifier = embeddedModifier != null ? new EmbeddedModifierProperty(embeddedModifier.getOwner(), fieldName) : null; + if (type == BinaryTypes.TYPE_EXTERNAL) { + // VALUE LIVES IN A PAIRED EXTERNAL BUCKET. FOLLOW THE RID. + final int extBucketId = buffer.getInt(); + final long extPosition = buffer.getLong(); + return readExternalValue((DatabaseInternal) database, extBucketId, extPosition, propertyModifier); + } + return deserializeValue(database, buffer, type, propertyModifier); } } catch (Exception e) { @@ -822,6 +842,12 @@ public Binary serializeProperties(final Database database, final Document record final Map properties = record.propertiesAsMap(); final Dictionary dictionary = database.getSchema().getDictionary(); final DocumentType documentType = record.getType(); + // For records being UPDATED, look up existing external RIDs from the old buffer so we can update the external bucket + // record in place rather than allocating a new one. New records (no identity yet) get an empty map. + final Map existingExternalRids = findExistingExternalRids(database, record); + // Track which existing external RIDs we re-used (kept) so we can delete the rest as orphans below. An entry is + // orphaned when the property is no longer EXTERNAL (toggled off via ALTER), was renamed, or was dropped entirely. + final Set consumedExternalProperties = existingExternalRids.isEmpty() ? null : new HashSet<>(); // Pre-resolve types so the property count matches what is actually written. // Skipping an invalid property after writing its nameId would desync the header on read. @@ -862,17 +888,51 @@ public Binary serializeProperties(final Database database, final Document record final int startContentPosition = content.position(); - if (value instanceof String stringValue && type == BinaryTypes.TYPE_STRING) { - final int id = dictionary.getIdByName(stringValue, false); - if (id > -1) { - // WRITE THE COMPRESSED STRING - type = BinaryTypes.TYPE_COMPRESSED_STRING; - value = id; + final Property propertyDef = documentType.getPropertyIfExists(propertyName); + if (propertyDef != null && propertyDef.isExternal()) { + // Externalised property: write the value to the paired external bucket and put a TYPE_EXTERNAL marker (with + // the external RID) in the main record's content. The main record stays small and traversal-only reads never + // hit the external bucket. See LocalDocumentType.getExternalBucketIdFor. + final RID identity = record.getIdentity(); + if (identity == null) + throw new SerializationException( + "Cannot serialize EXTERNAL property '" + propertyName + "' on type '" + documentType.getName() + + "': record has no target bucket. The bucket layer must set a provisional identity before serialize."); + final int primaryBucketId = identity.getBucketId(); + // Look up the external bucket via the type that ACTUALLY owns the primary bucket. This may differ from + // record.getType(): polymorphic scans (scanType POLYMORPHIC, MATCH, etc.) tag every record with the queried + // parent type even when the record physically lives in a subtype's bucket. Trusting documentType in that + // case would miss the subtype's external bucket map. + final LocalDocumentType ownerType = (LocalDocumentType) database.getSchema().getEmbedded().getTypeByBucketId(primaryBucketId); + final Integer extBucketId = (ownerType != null ? ownerType : (LocalDocumentType) documentType) + .getExternalBucketIdFor(primaryBucketId); + if (extBucketId == null) + throw new SerializationException( + "Cannot serialize EXTERNAL property '" + propertyName + "' on type '" + documentType.getName() + + "': no external bucket is paired with primary bucket " + primaryBucketId); + + final RID existingExtRid = existingExternalRids.get(propertyName); + if (consumedExternalProperties != null && existingExtRid != null) + consumedExternalProperties.add(propertyName); + + final RID newExtRid = writeExternalValue((DatabaseInternal) database, extBucketId, existingExtRid, type, value); + + content.putByte(BinaryTypes.TYPE_EXTERNAL); + content.putInt(newExtRid.getBucketId()); + content.putLong(newExtRid.getPosition()); + } else { + if (value instanceof String stringValue && type == BinaryTypes.TYPE_STRING) { + final int id = dictionary.getIdByName(stringValue, false); + if (id > -1) { + // WRITE THE COMPRESSED STRING + type = BinaryTypes.TYPE_COMPRESSED_STRING; + value = id; + } } - } - content.putByte(type); - serializeValue(database, content, type, value); + content.putByte(type); + serializeValue(database, content, type, value); + } // WRITE PROPERTY CONTENT POSITION header.putUnsignedNumber(startContentPosition); @@ -887,9 +947,123 @@ public Binary serializeProperties(final Database database, final Document record header.append(content); header.flip(); + + // Orphan cleanup: any existing external RID that was NOT re-used (property no longer EXTERNAL, was renamed, or was + // dropped) must be deleted from the external bucket so we don't leak storage. Same transaction as the primary write. + if (consumedExternalProperties != null) { + for (final Map.Entry entry : existingExternalRids.entrySet()) { + if (consumedExternalProperties.contains(entry.getKey())) + continue; + final RID orphanRid = entry.getValue(); + final LocalBucket externalBucket = database.getSchema().getEmbedded().getBucketById(orphanRid.getBucketId(), false); + if (externalBucket != null) { + externalBucket.deleteRecord(orphanRid); + ((DatabaseInternal) database).getTransaction().updateBucketRecordDelta(externalBucket.getFileId(), -1); + } + } + } + return header; } + /** + * Builds a value-only blob and writes it to the given external bucket. If existingExternalRid is null, appends a new + * record (insert). Otherwise updates the record at that RID in place (update). The blob format is: + * 
+   * [ExternalValueRecord.RECORD_TYPE : 1B][value type byte : 1B][value bytes : ...]
+   *
+ * Returns the RID where the blob was written. + */ + public RID writeExternalValue(final DatabaseInternal database, final int externalBucketId, final RID existingExternalRid, + final byte type, final Object value) { + final Binary blob = new Binary(); + blob.putByte(ExternalValueRecord.RECORD_TYPE); + blob.putByte(type); + serializeValue(database, blob, type, value); + blob.flip(); + + final LocalBucket externalBucket = database.getSchema().getEmbedded().getBucketById(externalBucketId); + if (existingExternalRid == null) { + final ExternalValueRecord rec = new ExternalValueRecord(database, null, blob); + final RID newRid = externalBucket.createRecord(rec, true); + // Mirror LocalDatabase.createRecord's accounting: keep the bucket's record-count cache consistent across the + // transaction, since this path goes through the bucket directly and bypasses LocalDatabase. + database.getTransaction().updateBucketRecordDelta(externalBucket.getFileId(), +1); + return newRid; + } + + final ExternalValueRecord rec = new ExternalValueRecord(database, existingExternalRid, blob); + rec.setIdentity(existingExternalRid); + externalBucket.updateRecord(rec, true); + return existingExternalRid; + } + + /** + * Reads the value blob at the given external RID and returns the deserialised value. The blob format must match + * {@link #writeExternalValue}. + */ + public Object readExternalValue(final DatabaseInternal database, final int externalBucketId, final long position, + final EmbeddedModifier embeddedModifier) { + final LocalBucket externalBucket = database.getSchema().getEmbedded().getBucketById(externalBucketId); + final RID rid = RID.create(database, externalBucketId, position); + final Binary buffer = externalBucket.getRecord(rid).copyOfContent(); + buffer.position(Binary.BYTE_SERIALIZED_SIZE); // SKIP RECORD TYPE BYTE + final byte valueType = buffer.getByte(); + return deserializeValue(database, buffer, valueType, embeddedModifier); + } + + /** + * Walks the OLD buffer of a Document being updated and collects the existing external RID for each EXTERNAL property, + * keyed by property name. Returns an empty map for new records (no identity, no buffer). + * Public so the database delete path can reuse it to cascade-delete external records. + */ + public Map findExistingExternalRids(final Database database, final Document record) { + final RID identity = record.getIdentity(); + if (identity == null) + return Collections.emptyMap(); + if (!(record instanceof BaseDocument)) + return Collections.emptyMap(); + final Binary oldBuffer = ((BaseRecord) record).getBuffer(); + if (oldBuffer == null) + return Collections.emptyMap(); + + try { + final Binary buf = oldBuffer.copyOfContent(); + buf.position(((BaseDocument) record).getPropertiesStartingPosition()); + + final int headerEndOffset = buf.getInt(); + final int properties = (int) buf.getUnsignedNumber(); + if (properties <= 0) + return Collections.emptyMap(); + + final Dictionary dictionary = database.getSchema().getDictionary(); + Map result = null; + + for (int i = 0; i < properties; i++) { + final int nameId = (int) buf.getUnsignedNumber(); + final int contentPosition = (int) buf.getUnsignedNumber(); + final int afterHeader = buf.position(); + + buf.position(headerEndOffset + contentPosition); + final byte type = buf.getByte(); + if (type == BinaryTypes.TYPE_EXTERNAL) { + final int extBucketId = buf.getInt(); + final long extPosition = buf.getLong(); + if (result == null) + result = new HashMap<>(); + result.put(dictionary.getNameById(nameId), RID.create(database, extBucketId, extPosition)); + } + + buf.position(afterHeader); + } + return result == null ? Collections.emptyMap() : result; + } catch (Exception e) { + LogManager.instance().log(this, Level.WARNING, + "Could not parse old buffer to recover external RIDs for record %s: %s", identity, e.getMessage()); + return Collections.emptyMap(); + } + } + public Class getDateImplementation() { return dateImplementation; } diff --git a/engine/src/main/java/com/arcadedb/serializer/BinaryTypes.java b/engine/src/main/java/com/arcadedb/serializer/BinaryTypes.java index 6abb085613..36d826b73c 100644 --- a/engine/src/main/java/com/arcadedb/serializer/BinaryTypes.java +++ b/engine/src/main/java/com/arcadedb/serializer/BinaryTypes.java @@ -65,6 +65,7 @@ public class BinaryTypes { public final static byte TYPE_ARRAY_OF_FLOATS = 26; // @SINCE 23.6.1 public final static byte TYPE_ARRAY_OF_DOUBLES = 27; // @SINCE 23.6.1 public final static byte TYPE_COMPRESSED_GEOMETRY = 28; // @SINCE 26.2.1 - Binary geometry storage (Point, Circle, Rectangle, etc.) + public final static byte TYPE_EXTERNAL = 29; // @SINCE 26.5.1 - Property value stored in a paired external bucket. Followed by [bucketId:int][position:long]. // Geometry subtypes for TYPE_COMPRESSED_GEOMETRY public final static byte GEOMETRY_SUBTYPE_POINT = 1; // Point: x(double), y(double) diff --git a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyDensitySlowTest.java b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyDensitySlowTest.java new file mode 100644 index 0000000000..57184c104c --- /dev/null +++ b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyDensitySlowTest.java @@ -0,0 +1,76 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.schema; + +import com.arcadedb.TestHelper; +import com.arcadedb.engine.LocalBucket; +import com.arcadedb.graph.MutableVertex; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Verifies that EXTERNAL property storage keeps the primary bucket dense: when a heavy property is flagged EXTERNAL, + * the primary bucket grows in proportion to the inline-only data while the heavy payload accumulates in the paired + * external bucket. This exercises the page-cache density win that motivates the feature. + * + * Tagged @slow because it inserts thousands of records with multi-KB embeddings; not run in the default suite. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +@Tag("slow") +class ExternalPropertyDensitySlowTest extends TestHelper { + + @Test + void primaryStaysSmallWhenHeavyPropertyIsExternal() { + final VertexType type = database.getSchema().createVertexType("V"); + type.createProperty("name", Type.STRING); + type.createProperty("embedding", Type.ARRAY_OF_FLOATS).setExternal(true); + + final int recordCount = 2000; + final int dim = 1024; // 4 KB per embedding + + database.transaction(() -> { + for (int i = 0; i < recordCount; i++) { + final float[] e = new float[dim]; + for (int j = 0; j < dim; j++) + e[j] = (float) (i * 0.001 + j); + final MutableVertex v = database.newVertex("V").set("name", "u" + i).set("embedding", e); + v.save(); + } + }); + + final LocalBucket primary = (LocalBucket) type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + + final long primaryPages = primary.getTotalPages(); + final long externalPages = external.getTotalPages(); + + // External bucket should hold the bulk of bytes (at least ~5x the primary bucket's page count given a 4KB + // payload per record vs. a small name+pointer per record). The exact ratio depends on page packing, so we use + // a conservative lower bound to keep the test stable. + assertThat(externalPages).as("external pages=%d primary pages=%d", externalPages, primaryPages) + .isGreaterThan(primaryPages * 3); + + // Sanity: count of records is recordCount. + assertThat(database.countType("V", false)).isEqualTo(recordCount); + } +} diff --git a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java new file mode 100644 index 0000000000..b0cc71778e --- /dev/null +++ b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java @@ -0,0 +1,455 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.schema; + +import com.arcadedb.TestHelper; +import com.arcadedb.database.MutableDocument; +import com.arcadedb.database.RID; +import com.arcadedb.engine.LocalBucket; +import com.arcadedb.graph.MutableVertex; +import com.arcadedb.query.sql.executor.ResultSet; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Verifies the EXTERNAL property storage feature: when a property is flagged EXTERNAL, its value lives in a paired + * external bucket keyed by the same record's RID, while the main record only carries a TYPE_EXTERNAL pointer. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +class ExternalPropertyTest extends TestHelper { + + @Test + void flagPersistsAcrossReopen() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("name", Type.STRING); + type.createProperty("blob", Type.STRING).setExternal(true); + + assertThat(type.getProperty("blob").isExternal()).isTrue(); + assertThat(type.getProperty("name").isExternal()).isFalse(); + + database.close(); + database = factory.open(); + + final DocumentType reloaded = database.getSchema().getType("Doc"); + assertThat(reloaded.getProperty("blob").isExternal()).isTrue(); + assertThat(reloaded.getProperty("name").isExternal()).isFalse(); + } + + @Test + void valueRoundTripDocument() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("name", Type.STRING); + type.createProperty("blob", Type.STRING).setExternal(true); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Doc") + .set("name", "alice") + .set("blob", "the quick brown fox jumps over the lazy dog"); + d.save(); + saved[0] = d.getIdentity(); + }); + + // Re-read from disk to be sure we exercise the deserialization path, not an in-memory cache hit. + database.close(); + database = factory.open(); + + final MutableDocument loaded = (MutableDocument) database.lookupByRID(saved[0], true).asDocument().modify(); + assertThat(loaded.getString("name")).isEqualTo("alice"); + assertThat(loaded.getString("blob")).isEqualTo("the quick brown fox jumps over the lazy dog"); + } + + @Test + void valueRoundTripVertexLargeArray() { + final VertexType type = database.getSchema().createVertexType("V"); + type.createProperty("name", Type.STRING); + type.createProperty("embedding", Type.ARRAY_OF_FLOATS).setExternal(true); + + final float[] embedding = new float[4096]; + for (int i = 0; i < embedding.length; i++) + embedding[i] = (float) Math.sin(i); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableVertex v = database.newVertex("V") + .set("name", "v1") + .set("embedding", embedding); + v.save(); + saved[0] = v.getIdentity(); + }); + + database.close(); + database = factory.open(); + + final var loaded = database.lookupByRID(saved[0], true).asVertex(); + assertThat(loaded.getString("name")).isEqualTo("v1"); + final Object readBack = loaded.get("embedding"); + assertThat(readBack).isInstanceOf(float[].class); + final float[] readBackArr = (float[]) readBack; + assertThat(readBackArr).hasSize(embedding.length); + for (int i = 0; i < embedding.length; i++) + assertThat(readBackArr[i]).as("position %d", i).isEqualTo(embedding[i]); + } + + @Test + void pairedExternalBucketIsCreatedAndMarkedSystem() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + // For each primary bucket of the type there must be a paired external bucket marked EXTERNAL_PROPERTY. + boolean foundAtLeastOne = false; + for (final var primaryBucket : type.getBuckets(false)) { + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primaryBucket.getFileId()); + assertThat(extId).as("external bucket id for primary %d", primaryBucket.getFileId()).isNotNull(); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + assertThat(external.getPurpose()).isEqualTo(LocalBucket.Purpose.EXTERNAL_PROPERTY); + assertThat(external.getName()).isEqualTo(primaryBucket.getName() + "_ext"); + foundAtLeastOne = true; + } + assertThat(foundAtLeastOne).as("type should have at least one primary bucket").isTrue(); + + // Type's regular buckets() list must NOT include the external buckets. + for (final var b : type.getBuckets(false)) + assertThat(((LocalBucket) b).getPurpose()).isEqualTo(LocalBucket.Purpose.PRIMARY); + } + + @Test + void inheritancePropagatesPairedExternalBucketsToSubtype() { + final DocumentType parent = database.getSchema().createDocumentType("Parent"); + parent.createProperty("blob", Type.STRING).setExternal(true); + + final DocumentType child = database.getSchema().createDocumentType("Child"); + child.addSuperType("Parent"); + + // Each of Child's primary buckets must have its own paired external bucket, even though the EXTERNAL property + // is inherited (not declared on Child directly). Records of Child live in Child's primary buckets. + for (final var primaryBucket : child.getBuckets(false)) { + final Integer extId = ((LocalDocumentType) child).getExternalBucketIdFor(primaryBucket.getFileId()); + assertThat(extId).as("external bucket id for child primary %d", primaryBucket.getFileId()).isNotNull(); + } + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Child").set("blob", "child blob"); + d.save(); + saved[0] = d.getIdentity(); + }); + + database.close(); + database = factory.open(); + + final MutableDocument loaded = (MutableDocument) database.lookupByRID(saved[0], true).asDocument().modify(); + assertThat(loaded.getString("blob")).isEqualTo("child blob"); + } + + @Test + void updateOfExternalProperty() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Doc").set("blob", "v1"); + d.save(); + saved[0] = d.getIdentity(); + }); + + database.transaction(() -> { + final MutableDocument d = (MutableDocument) database.lookupByRID(saved[0], true).asDocument().modify(); + d.set("blob", "v2-the-second-revision"); + d.save(); + }); + + database.close(); + database = factory.open(); + + final MutableDocument loaded = (MutableDocument) database.lookupByRID(saved[0], true).asDocument().modify(); + assertThat(loaded.getString("blob")).isEqualTo("v2-the-second-revision"); + } + + @Test + void deleteCascadesToExternalRecord() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Doc").set("blob", "to-be-deleted"); + d.save(); + saved[0] = d.getIdentity(); + }); + + // Verify the external bucket has at least one record before delete. + final Integer extBucketId = ((LocalDocumentType) type).getExternalBucketIdFor(saved[0].getBucketId()); + final LocalBucket externalBucket = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extBucketId); + final long extCountBefore = externalBucket.count(); + assertThat(extCountBefore).isGreaterThanOrEqualTo(1L); + + database.transaction(() -> { + database.lookupByRID(saved[0], true).asDocument().delete(); + }); + + final long extCountAfter = externalBucket.count(); + assertThat(extCountAfter).as("external record should be deleted by cascade").isEqualTo(extCountBefore - 1L); + } + + @Test + void directWriteToExternalBucketIsRejected() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + final var primaryBucket = type.getBuckets(false).getFirst(); + final Integer extBucketId = ((LocalDocumentType) type).getExternalBucketIdFor(primaryBucket.getFileId()); + final LocalBucket externalBucket = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extBucketId); + + // The Java path that resolves a bucket by name must reject an external bucket. Build a fresh document and try + // to route it to the external bucket via Database.createRecord(record, bucketName). + final MutableDocument fresh = database.newDocument("Doc").set("blob", "x"); + org.assertj.core.api.Assertions.assertThatThrownBy(() -> + database.transaction(() -> + ((com.arcadedb.database.DatabaseInternal) database).createRecord(fresh, externalBucket.getName()))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("internal"); + + // SQL INSERT INTO bucket: is also rejected (by the SQL planner, since the bucket has no associated + // type). Different error message but functionally equivalent: the user cannot target the bucket. + org.assertj.core.api.Assertions.assertThatThrownBy(() -> + database.transaction(() -> + database.command("sql", "INSERT INTO bucket:" + externalBucket.getName() + " SET x = 1"))) + .isInstanceOf(Exception.class); + } + + @Test + void sqlDdlCreateAndAlterExternal() { + database.transaction(() -> { + database.command("sql", "CREATE DOCUMENT TYPE Doc"); + database.command("sql", "CREATE PROPERTY Doc.blob STRING (EXTERNAL true)"); + }); + assertThat(database.getSchema().getType("Doc").getProperty("blob").isExternal()).isTrue(); + + database.transaction(() -> { + database.command("sql", "ALTER PROPERTY Doc.blob EXTERNAL false"); + }); + assertThat(database.getSchema().getType("Doc").getProperty("blob").isExternal()).isFalse(); + } + + @Test + void alterToExternalRelocatesOnNextWrite() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING); // inline initially + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Doc").set("blob", "starts-inline"); + d.save(); + saved[0] = d.getIdentity(); + }); + + // Flip to EXTERNAL. Existing record's bytes are not rewritten yet. + type.getProperty("blob").setExternal(true); + + // Read still returns the inline value: deserializer doesn't see TYPE_EXTERNAL in the OLD bytes. + var loaded = database.lookupByRID(saved[0], true).asDocument(); + assertThat(loaded.getString("blob")).isEqualTo("starts-inline"); + + // Update the property. Re-serialize must route through the external bucket now. + database.transaction(() -> { + final MutableDocument m = (MutableDocument) database.lookupByRID(saved[0], true).asDocument().modify(); + m.set("blob", "now-external"); + m.save(); + }); + + database.close(); + database = factory.open(); + + final var reloaded = database.lookupByRID(saved[0], true).asDocument(); + assertThat(reloaded.getString("blob")).isEqualTo("now-external"); + + // The external bucket should hold the new value. + final Integer extBucketId = ((LocalDocumentType) database.getSchema().getType("Doc")) + .getExternalBucketIdFor(saved[0].getBucketId()); + assertThat(extBucketId).isNotNull(); + final LocalBucket externalBucket = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extBucketId); + assertThat(externalBucket.count()).isGreaterThanOrEqualTo(1L); + } + + @Test + void indexLookupOnExternalProperty() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("name", Type.STRING).setExternal(true); + type.createTypeIndex(Schema.INDEX_TYPE.LSM_TREE, true, "name"); + + database.transaction(() -> { + for (int i = 0; i < 50; i++) + database.newDocument("Doc").set("name", "user-" + i).save(); + }); + + final ResultSet rs = database.query("sql", "SELECT FROM Doc WHERE name = 'user-37'"); + assertThat(rs.hasNext()).isTrue(); + assertThat((String) rs.next().getProperty("name")).isEqualTo("user-37"); + assertThat(rs.hasNext()).isFalse(); + } + + @Test + void schemaBucketsViewExposesPurposeColumn() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + final var primary = type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + + final ResultSet rs = database.query("sql", "SELECT name, purpose FROM schema:buckets"); + boolean foundPrimary = false; + boolean foundExternal = false; + while (rs.hasNext()) { + final var row = rs.next(); + final String name = row.getProperty("name"); + if (name.equals(primary.getName())) { + foundPrimary = true; + assertThat((String) row.getProperty("purpose")).isEqualTo("PRIMARY"); + } else if (name.equals(external.getName())) { + foundExternal = true; + assertThat((String) row.getProperty("purpose")).isEqualTo("EXTERNAL_PROPERTY"); + } + } + assertThat(foundPrimary).as("schema:buckets should list the primary bucket").isTrue(); + assertThat(foundExternal).as("schema:buckets should list the external bucket").isTrue(); + } + + @Test + void rebuildTypeMovesInlineToExternal() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING); // inline initially + + final int n = 25; + database.transaction(() -> { + for (int i = 0; i < n; i++) + database.newDocument("Doc").set("blob", "payload-" + i).save(); + }); + + // Flip the flag, rebuild. + type.getProperty("blob").setExternal(true); + + database.transaction(() -> { + final ResultSet rs = database.command("sql", "REBUILD TYPE Doc"); + assertThat(rs.hasNext()).isTrue(); + final var row = rs.next(); + assertThat((Long) row.getProperty("recordsRebuilt")).isEqualTo((long) n); + }); + + // After rebuild, the external bucket should hold one record per Doc record. + final var primary = type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + assertThat(external.count()).isEqualTo((long) n); + + // Values still readable. + final ResultSet rs = database.query("sql", "SELECT blob FROM Doc ORDER BY blob"); + int counted = 0; + while (rs.hasNext()) { + final String val = rs.next().getProperty("blob"); + assertThat(val).startsWith("payload-"); + counted++; + } + assertThat(counted).isEqualTo(n); + } + + @Test + void rebuildTypeReversesExternalToInlineAndCleansOrphans() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + final int n = 15; + database.transaction(() -> { + for (int i = 0; i < n; i++) + database.newDocument("Doc").set("blob", "ext-" + i).save(); + }); + + final var primary = type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + assertThat(external.count()).isEqualTo((long) n); + + // Flip OFF and rebuild. + type.getProperty("blob").setExternal(false); + database.transaction(() -> database.command("sql", "REBUILD TYPE Doc")); + + // After rebuild every external record should have been deleted (orphan cleanup). + assertThat(external.count()).isEqualTo(0L); + + // Values must still be readable inline. + final ResultSet rs = database.query("sql", "SELECT blob FROM Doc"); + int counted = 0; + while (rs.hasNext()) { + assertThat((String) rs.next().getProperty("blob")).startsWith("ext-"); + counted++; + } + assertThat(counted).isEqualTo(n); + } + + @Test + void rebuildTypePolymorphicWalksSubtypes() { + final DocumentType parent = database.getSchema().createDocumentType("Parent"); + parent.createProperty("blob", Type.STRING); + final DocumentType child = database.getSchema().createDocumentType("Child"); + child.addSuperType("Parent"); + + database.transaction(() -> { + database.newDocument("Parent").set("blob", "p1").save(); + database.newDocument("Child").set("blob", "c1").save(); + database.newDocument("Child").set("blob", "c2").save(); + }); + + // Toggle EXTERNAL on the inherited property and rebuild polymorphically. + parent.getProperty("blob").setExternal(true); + database.transaction(() -> { + final ResultSet rs = database.command("sql", "REBUILD TYPE Parent POLYMORPHIC"); + assertThat(rs.hasNext()).isTrue(); + assertThat((Long) rs.next().getProperty("recordsRebuilt")).isEqualTo(3L); + }); + + // Both Parent and Child external buckets should now hold their respective records. + final var parentBucket = parent.getBuckets(false).getFirst(); + final Integer parentExtId = ((LocalDocumentType) parent).getExternalBucketIdFor(parentBucket.getFileId()); + final var childBucket = child.getBuckets(false).getFirst(); + final Integer childExtId = ((LocalDocumentType) child).getExternalBucketIdFor(childBucket.getFileId()); + final var localSchema = (LocalSchema) database.getSchema().getEmbedded(); + assertThat(localSchema.getBucketById(parentExtId).count()).isEqualTo(1L); + assertThat(localSchema.getBucketById(childExtId).count()).isEqualTo(2L); + } + + @Test + void rollbackDiscardsBothPrimaryAndExternal() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + final long primaryCountBefore = database.countType("Doc", false); + + database.begin(); + final MutableDocument d = database.newDocument("Doc").set("blob", "rolled-back"); + d.save(); + database.rollback(); + + final long primaryCountAfter = database.countType("Doc", false); + assertThat(primaryCountAfter).isEqualTo(primaryCountBefore); + } +} diff --git a/network/src/main/java/com/arcadedb/remote/RemoteProperty.java b/network/src/main/java/com/arcadedb/remote/RemoteProperty.java index 040e7fb8e6..735b5ac698 100644 --- a/network/src/main/java/com/arcadedb/remote/RemoteProperty.java +++ b/network/src/main/java/com/arcadedb/remote/RemoteProperty.java @@ -71,6 +71,11 @@ public Property setHidden(boolean hidden) { throw new UnsupportedOperationException(); } + @Override + public Property setExternal(boolean external) { + throw new UnsupportedOperationException(); + } + @Override public Property setMax(String max) { throw new UnsupportedOperationException(); @@ -106,6 +111,8 @@ void reload(final Map entry) { max = (String) entry.get("max"); if (entry.containsKey("hidden")) hidden = (Boolean) entry.get("hidden"); + if (entry.containsKey("external")) + external = (Boolean) entry.get("external"); if (entry.containsKey("default")) defaultValue = entry.get("default"); if (entry.containsKey("regexp")) From 72c17abd69a86d5b6ed28441e6f92a50c8e95a57 Mon Sep 17 00:00:00 2001 From: Luca Garulli Date: Tue, 28 Apr 2026 23:32:47 -0400 Subject: [PATCH 02/12] fix: added support for studio with external property --- .../executor/FetchFromSchemaTypesStep.java | 18 +++++++ .../arcadedb/schema/ExternalPropertyTest.java | 48 +++++++++++++++++++ .../resources/static/js/studio-database.js | 29 +++++++++-- 3 files changed, 91 insertions(+), 4 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java index 308a55e549..5fdf2891a1 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java +++ b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java @@ -89,6 +89,22 @@ else if (type.getType() == Edge.RECORD_TYPE) r.setProperty("buckets", type.getBuckets(false).stream().map((b) -> b.getName()).collect(Collectors.toList())); r.setProperty("bucketSelectionStrategy", type.getBucketSelectionStrategy().getName()); + // Expose the primary->external bucket mapping for types that have any EXTERNAL property. Lets tooling + // (Studio etc.) tell the user where the externalised values for each primary bucket are stored. + if (type instanceof com.arcadedb.schema.LocalDocumentType ldt) { + final Map extMap = new HashMap<>(); + for (final var b : type.getBuckets(false)) { + final Integer extId = ldt.getExternalBucketIdFor(b.getFileId()); + if (extId != null) { + final var extBucket = context.getDatabase().getSchema().getBucketById(extId); + if (extBucket != null) + extMap.put(b.getName(), extBucket.getName()); + } + } + if (!extMap.isEmpty()) + r.setProperty("externalBuckets", extMap); + } + final List parents = type.getSuperTypes().stream().map(pt -> pt.getName()).collect(Collectors.toList()); r.setProperty("parentTypes", parents); @@ -109,6 +125,8 @@ else if (type.getType() == Edge.RECORD_TYPE) propRes.setProperty("notNull", property.isNotNull()); if (property.isHidden()) propRes.setProperty("hidden", property.isHidden()); + if (property.isExternal()) + propRes.setProperty("external", property.isExternal()); if (property.getMin() != null) propRes.setProperty("min", property.getMin()); if (property.getMax() != null) diff --git a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java index b0cc71778e..6ff6a4088f 100644 --- a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java +++ b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java @@ -333,6 +333,16 @@ void schemaBucketsViewExposesPurposeColumn() { } assertThat(foundPrimary).as("schema:buckets should list the primary bucket").isTrue(); assertThat(foundExternal).as("schema:buckets should list the external bucket").isTrue(); + + // The Studio buckets tab uses this exact WHERE filter to hide internal buckets. Verify it works on + // schema:buckets and excludes the EXTERNAL_PROPERTY bucket but still includes the primary one. + final ResultSet filtered = database.query("sql", + "SELECT name, purpose FROM schema:buckets WHERE purpose = 'PRIMARY' OR purpose IS NULL"); + final java.util.Set names = new java.util.HashSet<>(); + while (filtered.hasNext()) + names.add(filtered.next().getProperty("name")); + assertThat(names).contains(primary.getName()); + assertThat(names).doesNotContain(external.getName()); } @Test @@ -437,6 +447,44 @@ void rebuildTypePolymorphicWalksSubtypes() { assertThat(localSchema.getBucketById(childExtId).count()).isEqualTo(2L); } + @Test + void schemaTypesViewExposesExternalFlagAndPairing() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("name", Type.STRING); + type.createProperty("blob", Type.STRING).setExternal(true); + + final ResultSet rs = database.query("sql", "SELECT FROM schema:types WHERE name = 'Doc'"); + assertThat(rs.hasNext()).isTrue(); + final var row = rs.next(); + + // Per-property external flag (only emitted when true). + final var properties = (java.util.List) row.getProperty("properties"); + assertThat(properties).isNotNull(); + boolean foundBlobAsExternal = false; + boolean foundNameWithoutFlag = false; + for (final Object propObj : properties) { + final var prop = (com.arcadedb.query.sql.executor.Result) propObj; + final String name = prop.getProperty("name"); + if ("blob".equals(name)) { + assertThat((Boolean) prop.getProperty("external")).isTrue(); + foundBlobAsExternal = true; + } else if ("name".equals(name)) { + assertThat((Boolean) prop.getProperty("external")).isNull(); + foundNameWithoutFlag = true; + } + } + assertThat(foundBlobAsExternal).isTrue(); + assertThat(foundNameWithoutFlag).isTrue(); + + // Type-level externalBuckets mapping (primaryBucketName -> externalBucketName). + @SuppressWarnings("unchecked") + final java.util.Map extMap = (java.util.Map) row.getProperty("externalBuckets"); + assertThat(extMap).isNotNull().isNotEmpty(); + final String primaryName = type.getBuckets(false).getFirst().getName(); + assertThat(extMap).containsKey(primaryName); + assertThat(extMap.get(primaryName)).endsWith("_ext"); + } + @Test void rollbackDiscardsBothPrimaryAndExternal() { final DocumentType type = database.getSchema().createDocumentType("Doc"); diff --git a/studio/src/main/resources/static/js/studio-database.js b/studio/src/main/resources/static/js/studio-database.js index 85b522363f..4bbb314e72 100644 --- a/studio/src/main/resources/static/js/studio-database.js +++ b/studio/src/main/resources/static/js/studio-database.js @@ -1065,6 +1065,7 @@ function createProperty(typeName) { html += "
"; html += "
"; html += "
"; + html += "
"; html += "
"; html += ""; @@ -1085,6 +1086,7 @@ function createProperty(typeName) { let notNull = $("#inputCreatePropNotNull").prop("checked"); let hidden = $("#inputCreatePropHidden").prop("checked"); let readOnly = $("#inputCreatePropReadOnly").prop("checked"); + let external = $("#inputCreatePropExternal").prop("checked"); let ifNotExists = $("#inputCreatePropIfNotExists").prop("checked"); let command = "CREATE PROPERTY `" + typeName + "`.`" + name + "`"; @@ -1097,6 +1099,7 @@ function createProperty(typeName) { if (notNull) constraints.push("NOTNULL true"); if (hidden) constraints.push("HIDDEN true"); if (readOnly) constraints.push("READONLY true"); + if (external) constraints.push("EXTERNAL true"); if (defaultVal != "") constraints.push("DEFAULT " + defaultVal); if (min != "") constraints.push("MIN " + min); if (max != "") constraints.push("MAX " + max); @@ -3113,7 +3116,7 @@ function showTypeDetail(typeName) { } else { html += "
"; html += ""; - html += ""; + html += ""; html += "" + propHtml + "
NameDefined InTypeMandatoryNot NullHiddenRead OnlyDefaultMinMaxRegexpIndexesActions
NameDefined InTypeStorageMandatoryNot NullHiddenRead OnlyDefaultMinMaxRegexpIndexesActions
"; } html += ""; @@ -3320,6 +3323,22 @@ function renderProperties(row, results) { let property = row.properties[k]; panelHtml += "" + property.name + "" + row.name + "" + property.type + ""; + // Storage cell: External properties show a purple badge with the paired bucket(s) in the tooltip so the user + // immediately sees that the value lives outside the primary record and where to find it on disk. + if (property.external) { + let pairs = []; + if (row.externalBuckets) { + for (let primary in row.externalBuckets) + pairs.push(escapeHtml(primary) + " \u2192 " + escapeHtml(row.externalBuckets[primary])); + } + let tooltip = pairs.length > 0 + ? "Value stored in paired external bucket(s): " + pairs.join(", ") + : "Value stored in a paired external bucket"; + panelHtml += " External"; + } else { + panelHtml += "Inline"; + } + panelHtml += "" + (property.mandatory ? true : false) + ""; panelHtml += "" + (property.notNull ? true : false) + ""; panelHtml += "" + (property.hidden ? true : false) + ""; @@ -3346,7 +3365,7 @@ function renderProperties(row, results) { if (property.custom != null && Object.keys(property.custom).length > 0) { panelHtml += ""; - panelHtml += "Custom Properties
"; + panelHtml += "Custom Properties
"; panelHtml += "
"; panelHtml += ""; for (c in property.custom) panelHtml += ""; @@ -4566,7 +4585,7 @@ function showMaterializedViewDetail(viewName) { else { html += "
"; html += "
" + c + "" + property.custom[c] + "
"; - html += ""; + html += ""; html += "" + propHtml + "
NameDefined InTypeMandatoryNot NullHiddenRead OnlyDefaultMinMaxRegexpIndexesActions
NameDefined InTypeStorageMandatoryNot NullHiddenRead OnlyDefaultMinMaxRegexpIndexesActions
"; } html += ""; @@ -5198,7 +5217,9 @@ function loadStorageBuckets() { jQuery.ajax({ type: "POST", url: "api/v1/query/" + database, - data: JSON.stringify({ language: "sql", command: "SELECT FROM schema:buckets" }), + // Hide non-PRIMARY buckets (e.g. paired EXTERNAL_PROPERTY buckets that hold externalised property values). + // Power users can still see them by running SELECT FROM schema:buckets directly in the Query tab. + data: JSON.stringify({ language: "sql", command: "SELECT FROM schema:buckets WHERE purpose = 'PRIMARY' OR purpose IS NULL" }), beforeSend: function (xhr) { xhr.setRequestHeader("Authorization", globalCredentials); }, }).done(function (data) { if (!data.result || data.result.length === 0) { From bdf17078108dd77512fcd99f4629cf4b35c6136b Mon Sep 17 00:00:00 2001 From: Luca Garulli Date: Tue, 28 Apr 2026 23:36:46 -0400 Subject: [PATCH 03/12] correctly sized external bucket page size to 256 KB Why 256 KB: - A 4096-float embedding is 16 KB. With a 64 KB page it fits inline, but only ~3 such records per page after slot table + record headers - and any wider JSON payload starts overflowing into the multi-page chunk chain. - A 256 KB page comfortably holds 10-15 such records per page, keeping reads to a single I/O. - Same default the LSM tree and vector indexes use, so it is not a stretch from existing project conventions. --- .../java/com/arcadedb/GlobalConfiguration.java | 4 ++++ .../com/arcadedb/schema/LocalDocumentType.java | 14 +++++++++++--- .../arcadedb/schema/ExternalPropertyTest.java | 16 ++++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/GlobalConfiguration.java b/engine/src/main/java/com/arcadedb/GlobalConfiguration.java index 06c0ef41c4..2d9842b3f2 100644 --- a/engine/src/main/java/com/arcadedb/GlobalConfiguration.java +++ b/engine/src/main/java/com/arcadedb/GlobalConfiguration.java @@ -210,6 +210,10 @@ public Object call(final Object value) { BUCKET_DEFAULT_PAGE_SIZE("arcadedb.bucketDefaultPageSize", SCOPE.DATABASE, "Default page size in bytes for buckets. Default is 64KB", Integer.class, 65_536), + EXTERNAL_PROPERTY_BUCKET_DEFAULT_PAGE_SIZE("arcadedb.externalPropertyBucketDefaultPageSize", SCOPE.DATABASE, + "Default page size in bytes for paired external-property buckets. They hold heavy property payloads (vector embeddings, large strings, embedded JSON) so the default is larger than for primary buckets to reduce multi-page chunking. Matches the LSM-index default (256KB)", + Integer.class, 262_144), + BUCKET_REUSE_SPACE_MODE("arcadedb.bucketReuseSpaceMode", SCOPE.DATABASE, "How to reuse space in pages. 'high' = more space saved, but slower opening and update/delete time. 'medium' to still reuse space without the initial scan at opening time. 'low' for faster performance, but less space reused. Default is 'high'", String.class, "high", Set.of("low", "medium", "high")), diff --git a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java index 7226421601..db452e983b 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java @@ -1015,9 +1015,17 @@ private void ensureExternalBucketFor(final LocalBucket primary) { if (externalBucketIdByPrimaryBucketId.containsKey(primary.getFileId())) return; final String extName = primary.getName() + "_ext"; - final LocalBucket external = schema.bucketMap.containsKey(extName) ? - schema.bucketMap.get(extName) : - schema.createBucket(extName); + final LocalBucket external; + if (schema.bucketMap.containsKey(extName)) + external = schema.bucketMap.get(extName); + else { + // External buckets carry heavy payloads (vectors, large strings, embedded JSON), so they default to a + // larger page size than primary buckets to reduce multi-page chunking. Tunable via + // GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_DEFAULT_PAGE_SIZE. + final int pageSize = schema.getDatabase().getConfiguration() + .getValueAsInteger(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_DEFAULT_PAGE_SIZE); + external = schema.createBucket(extName, pageSize); + } external.setPurpose(LocalBucket.Purpose.EXTERNAL_PROPERTY); externalBucketIdByPrimaryBucketId.put(primary.getFileId(), external.getFileId()); } diff --git a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java index 6ff6a4088f..39865399b7 100644 --- a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java +++ b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java @@ -485,6 +485,22 @@ void schemaTypesViewExposesExternalFlagAndPairing() { assertThat(extMap.get(primaryName)).endsWith("_ext"); } + @Test + void externalBucketUsesLargerDefaultPageSize() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + // Primary bucket uses the standard 64KB page; external bucket uses the heavier 256KB page so multi-KB + // payloads (vectors, big strings) fit in a single page rather than overflowing into the chunk-chain path. + final LocalBucket primary = (LocalBucket) type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + + assertThat(primary.getPageSize()).isEqualTo(65_536); + assertThat(external.getPageSize()).isEqualTo(262_144); + assertThat(external.getPageSize()).isGreaterThan(primary.getPageSize()); + } + @Test void rollbackDiscardsBothPrimaryAndExternal() { final DocumentType type = database.getSchema().createDocumentType("Doc"); From be12d204bbea293ce8b9937bcee58acaa855e764 Mon Sep 17 00:00:00 2001 From: Luca Garulli Date: Tue, 28 Apr 2026 23:53:14 -0400 Subject: [PATCH 04/12] Added new `arcadedb.externalPropertyBucketPath` setting --- .../com/arcadedb/GlobalConfiguration.java | 4 ++ .../com/arcadedb/database/LocalDatabase.java | 5 +- .../java/com/arcadedb/engine/FileManager.java | 46 +++++++++++++++---- .../arcadedb/schema/LocalDocumentType.java | 10 ++-- .../java/com/arcadedb/schema/LocalSchema.java | 17 ++++++- .../arcadedb/schema/ExternalPropertyTest.java | 46 +++++++++++++++++++ 6 files changed, 112 insertions(+), 16 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/GlobalConfiguration.java b/engine/src/main/java/com/arcadedb/GlobalConfiguration.java index 2d9842b3f2..f3808fc445 100644 --- a/engine/src/main/java/com/arcadedb/GlobalConfiguration.java +++ b/engine/src/main/java/com/arcadedb/GlobalConfiguration.java @@ -214,6 +214,10 @@ public Object call(final Object value) { "Default page size in bytes for paired external-property buckets. They hold heavy property payloads (vector embeddings, large strings, embedded JSON) so the default is larger than for primary buckets to reduce multi-page chunking. Matches the LSM-index default (256KB)", Integer.class, 262_144), + EXTERNAL_PROPERTY_BUCKET_PATH("arcadedb.externalPropertyBucketPath", SCOPE.DATABASE, + "Filesystem directory where new paired external-property buckets are created. If empty (default), external buckets sit alongside primary buckets in the database directory. Set to a path on cheaper/slower storage (HDD, network mount) to tier the heavy payloads away from the topology files. The directory must exist and be writable. Existing external buckets are not relocated when this changes.", + String.class, ""), + BUCKET_REUSE_SPACE_MODE("arcadedb.bucketReuseSpaceMode", SCOPE.DATABASE, "How to reuse space in pages. 'high' = more space saved, but slower opening and update/delete time. 'medium' to still reuse space without the initial scan at opening time. 'low' for faster performance, but less space reused. Default is 'high'", String.class, "high", Set.of("low", "medium", "high")), diff --git a/engine/src/main/java/com/arcadedb/database/LocalDatabase.java b/engine/src/main/java/com/arcadedb/database/LocalDatabase.java index 9173b1f5c4..5207145eba 100644 --- a/engine/src/main/java/com/arcadedb/database/LocalDatabase.java +++ b/engine/src/main/java/com/arcadedb/database/LocalDatabase.java @@ -2060,7 +2060,10 @@ private void openInternal() { DatabaseContext.INSTANCE.init(this); setLockingEnabled(configuration.getValueAsBoolean(GlobalConfiguration.BACKUP_ENABLED)); - fileManager = new FileManager(databasePath, mode, SUPPORTED_FILE_EXT); + // Optional second scan directory for paired external-property buckets that were tiered to cheaper storage + // via arcadedb.externalPropertyBucketPath. Empty by default; if set, FileManager opens .bucket files there too. + final String externalBucketPath = configuration.getValueAsString(GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH); + fileManager = new FileManager(databasePath, mode, SUPPORTED_FILE_EXT, externalBucketPath); transactionManager = new TransactionManager(wrappedDatabaseInstance); open = true; diff --git a/engine/src/main/java/com/arcadedb/engine/FileManager.java b/engine/src/main/java/com/arcadedb/engine/FileManager.java index 5dfface981..6213a17956 100644 --- a/engine/src/main/java/com/arcadedb/engine/FileManager.java +++ b/engine/src/main/java/com/arcadedb/engine/FileManager.java @@ -68,6 +68,17 @@ public static class FileManagerStats { } public FileManager(final String path, final ComponentFile.MODE mode, final Set supportedFileExt) { + this(path, mode, supportedFileExt, null); + } + + /** + * @param path primary database directory; created if missing. + * @param extraScanPath optional secondary directory to scan for additional component files (e.g. paired + * external-property buckets that have been tiered to a different disk via + * {@code arcadedb.externalPropertyBucketPath}). May be null/empty. + */ + public FileManager(final String path, final ComponentFile.MODE mode, final Set supportedFileExt, + final String extraScanPath) { this.mode = mode; final File dbDirectory = new File(path); @@ -83,18 +94,33 @@ public FileManager(final String path, final ComponentFile.MODE mode, final Set supportedFileExt) { + final File[] entries = dir.listFiles(); + if (entries == null) + return; + for (final File f : entries) { + final String filePath = f.getAbsolutePath(); + final int lastDot = filePath.lastIndexOf("."); + if (lastDot < 0) + continue; + final String fileExt = filePath.substring(lastDot + 1); + if (!supportedFileExt.contains(fileExt)) + continue; + try { + final ComponentFile file = new PaginatedComponentFile(f.getAbsolutePath(), mode); + registerFile(file); + } catch (final FileNotFoundException e) { + LogManager.instance().log(this, Level.WARNING, "Cannot load file '%s'", null, f); } } } diff --git a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java index db452e983b..c343a8b961 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java @@ -1021,10 +1021,12 @@ private void ensureExternalBucketFor(final LocalBucket primary) { else { // External buckets carry heavy payloads (vectors, large strings, embedded JSON), so they default to a // larger page size than primary buckets to reduce multi-page chunking. Tunable via - // GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_DEFAULT_PAGE_SIZE. - final int pageSize = schema.getDatabase().getConfiguration() - .getValueAsInteger(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_DEFAULT_PAGE_SIZE); - external = schema.createBucket(extName, pageSize); + // EXTERNAL_PROPERTY_BUCKET_DEFAULT_PAGE_SIZE. The file may also be tiered to a different directory via + // EXTERNAL_PROPERTY_BUCKET_PATH for cheaper-storage placement; the FileManager scans that path at startup. + final var config = schema.getDatabase().getConfiguration(); + final int pageSize = config.getValueAsInteger(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_DEFAULT_PAGE_SIZE); + final String overridePath = config.getValueAsString(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH); + external = schema.createBucket(extName, pageSize, overridePath); } external.setPurpose(LocalBucket.Purpose.EXTERNAL_PROPERTY); externalBucketIdByPrimaryBucketId.put(primary.getFileId(), external.getFileId()); diff --git a/engine/src/main/java/com/arcadedb/schema/LocalSchema.java b/engine/src/main/java/com/arcadedb/schema/LocalSchema.java index 64a797ab8d..a95c4d60e2 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalSchema.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalSchema.java @@ -372,14 +372,29 @@ public LocalBucket createBucket(final String bucketName) { } public LocalBucket createBucket(final String bucketName, final int pageSize) { + return createBucket(bucketName, pageSize, databasePath); + } + + /** + * Creates a bucket whose underlying file lives at {@code parentDirectory + File.separator + bucketName} instead of + * the default database directory. Used by paired external-property buckets when + * {@code arcadedb.externalPropertyBucketPath} is configured, so the heavy payload files can sit on cheaper storage. + * Falls back to the database directory when {@code parentDirectory} is null or empty. + */ + public LocalBucket createBucket(final String bucketName, final int pageSize, final String parentDirectory) { database.checkPermissionsOnDatabase(SecurityDatabaseUser.DATABASE_ACCESS.UPDATE_SCHEMA); if (bucketMap.containsKey(bucketName)) throw new SchemaException("Cannot create bucket '" + bucketName + "' because already exists"); + final String dir = (parentDirectory == null || parentDirectory.isEmpty()) ? databasePath : parentDirectory; + return recordFileChanges(() -> { try { - final LocalBucket bucket = new LocalBucket(database, bucketName, databasePath + File.separator + bucketName, + final File parent = new File(dir); + if (!parent.exists() && !parent.mkdirs()) + throw new SchemaException("Cannot create directory '" + dir + "' for bucket '" + bucketName + "'"); + final LocalBucket bucket = new LocalBucket(database, bucketName, dir + File.separator + bucketName, ComponentFile.MODE.READ_WRITE, pageSize, LocalBucket.CURRENT_VERSION); registerFile((Component) bucket); bucketMap.put(bucketName, bucket); diff --git a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java index 39865399b7..e2d34b9152 100644 --- a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java +++ b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java @@ -501,6 +501,52 @@ void externalBucketUsesLargerDefaultPageSize() { assertThat(external.getPageSize()).isGreaterThan(primary.getPageSize()); } + @Test + void externalBucketPathOverridePlacesFileOnSecondaryDirectory() throws java.io.IOException { + // Use a tier directory outside the database path to simulate cheaper-storage placement. + final java.nio.file.Path overrideDir = java.nio.file.Files.createTempDirectory("arcadedb-ext-tier-"); + final Object previous = com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH.getValue(); + try { + // Apply the override on the open database. ensureExternalBucketFor reads it lazily when the first paired + // bucket is allocated, so the new path takes effect immediately for the type we are about to create. + database.getConfiguration().setValue(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH, overrideDir.toString()); + + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Doc").set("blob", "tiered-payload"); + d.save(); + saved[0] = d.getIdentity(); + }); + + // External bucket file lives in the override directory, not the database directory. + final var primary = type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + + final java.io.File extFile = new java.io.File(database.getDatabasePath(), external.getName() + ".0.262144.v0.bucket"); + assertThat(extFile.exists()).as("external bucket should NOT be in the database directory").isFalse(); + + final java.io.File[] tieredFiles = overrideDir.toFile().listFiles((dir, name) -> name.startsWith(external.getName())); + assertThat(tieredFiles).as("external bucket should be in the override directory").isNotNull().isNotEmpty(); + + // Reopen: FileManager must rediscover the tiered file via the secondary scan path so the record stays readable. + // The override is applied at open() via the global config, so set it there too before reopening. + com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH.setValue(overrideDir.toString()); + database.close(); + database = factory.open(); + database.getConfiguration().setValue(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH, overrideDir.toString()); + + final var loaded = database.lookupByRID(saved[0], true).asDocument(); + assertThat(loaded.getString("blob")).isEqualTo("tiered-payload"); + } finally { + com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH.setValue(previous != null ? previous.toString() : ""); + com.arcadedb.utility.FileUtils.deleteRecursively(overrideDir.toFile()); + } + } + @Test void rollbackDiscardsBothPrimaryAndExternal() { final DocumentType type = database.getSchema().createDocumentType("Doc"); From 30a3d48fc500e5397a318bf54e41d8ae3fcb480a Mon Sep 17 00:00:00 2001 From: Luca Garulli Date: Wed, 29 Apr 2026 11:54:03 -0400 Subject: [PATCH 05/12] +check database, +fix, 256 max record in ext buckets (v1) --- .../arcadedb/query/sql/grammar/SQLParser.g4 | 2 +- .../com/arcadedb/database/BaseDocument.java | 6 - .../database/ExternalValueRecord.java | 11 +- .../com/arcadedb/database/LocalDatabase.java | 13 +- .../com/arcadedb/engine/DatabaseChecker.java | 84 ++++++ .../java/com/arcadedb/engine/FileManager.java | 8 +- .../java/com/arcadedb/engine/LocalBucket.java | 23 +- .../query/sql/antlr/SQLASTBuilder.java | 14 +- .../executor/FetchFromSchemaTypesStep.java | 3 +- .../sql/parser/AlterPropertyStatement.java | 3 + .../CreatePropertyAttributeStatement.java | 2 + .../sql/parser/RebuildTypeStatement.java | 36 +-- .../com/arcadedb/schema/AbstractProperty.java | 14 +- .../com/arcadedb/schema/DocumentType.java | 2 + .../arcadedb/schema/LocalDocumentType.java | 72 +++--- .../com/arcadedb/schema/LocalProperty.java | 19 +- .../java/com/arcadedb/schema/LocalSchema.java | 18 +- .../java/com/arcadedb/schema/Property.java | 8 + .../arcadedb/serializer/BinarySerializer.java | 179 ++++++++++--- .../com/arcadedb/serializer/BinaryTypes.java | 3 +- .../arcadedb/schema/ExternalPropertyTest.java | 163 +++++++++++- ...RaftLeaderCrashWithExternalPropertyIT.java | 244 ++++++++++++++++++ .../com/arcadedb/remote/RemoteProperty.java | 7 + 23 files changed, 798 insertions(+), 136 deletions(-) create mode 100644 ha-raft/src/test/java/com/arcadedb/server/ha/raft/RaftLeaderCrashWithExternalPropertyIT.java diff --git a/engine/src/main/antlr4/com/arcadedb/query/sql/grammar/SQLParser.g4 b/engine/src/main/antlr4/com/arcadedb/query/sql/grammar/SQLParser.g4 index 005e348534..f7e11940de 100644 --- a/engine/src/main/antlr4/com/arcadedb/query/sql/grammar/SQLParser.g4 +++ b/engine/src/main/antlr4/com/arcadedb/query/sql/grammar/SQLParser.g4 @@ -878,7 +878,7 @@ rebuildIndexStatement * Syntax: REBUILD TYPE typeName [POLYMORPHIC] */ rebuildTypeBody - : identifier POLYMORPHIC? + : identifier POLYMORPHIC? (WITH identifier EQ expression (COMMA identifier EQ expression)*)? ; // ============================================================================ diff --git a/engine/src/main/java/com/arcadedb/database/BaseDocument.java b/engine/src/main/java/com/arcadedb/database/BaseDocument.java index ad51c6c5db..7ab77d51eb 100644 --- a/engine/src/main/java/com/arcadedb/database/BaseDocument.java +++ b/engine/src/main/java/com/arcadedb/database/BaseDocument.java @@ -157,12 +157,6 @@ public DocumentType getType() { return type; } - /** - * Returns the byte offset in the record buffer where the property header begins (i.e. immediately after the record-type - * byte and any record-kind prefix such as the 24 bytes of in/out edge pointers on a vertex). Used by the serializer to - * recover the previous serialized bytes when re-serializing an updated record (e.g. to reuse the existing RID of an - * EXTERNAL property's paired record). - */ public int getPropertiesStartingPosition() { return propertiesStartingPosition; } diff --git a/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java b/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java index c32f0b93fa..540172663e 100644 --- a/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java +++ b/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java @@ -21,10 +21,7 @@ import com.arcadedb.serializer.json.JSONObject; /** - * Lightweight record used by the serializer to store the value of a property flagged EXTERNAL in a paired bucket. - * Holds an opaque pre-serialized buffer of the form `[RECORD_TYPE_EXTERNAL][value type byte][value bytes]`. The - * leading record-type byte follows the convention of edge segments: the buffer is written verbatim by the bucket and - * we do the framing ourselves. + * Opaque payload record for EXTERNAL property values. Buffer = [RECORD_TYPE][value type][value bytes]. * * @author Luca Garulli (l.garulli@arcadedata.com) */ @@ -50,17 +47,13 @@ public void unsetDirty() { // NO-OP: BUFFER IS BUILT FRESH ON EACH SERIALIZE } - /** - * Returns the buffer with the RECORD_TYPE_EXTERNAL marker at byte 0 and the value blob ([type][value bytes]) following. - */ public Binary getContent() { return buffer; } @Override public JSONObject toJSON(final boolean includeMetadata) { - // Internal infrastructure record - no user-visible JSON form. Returning empty keeps callers like generic record - // dumpers safe even though they should never reach an EXTERNAL value record directly. + // No user-visible JSON form - generic record dumpers should never reach an EXTERNAL value blob directly. return new JSONObject(); } } diff --git a/engine/src/main/java/com/arcadedb/database/LocalDatabase.java b/engine/src/main/java/com/arcadedb/database/LocalDatabase.java index 5207145eba..be31694062 100644 --- a/engine/src/main/java/com/arcadedb/database/LocalDatabase.java +++ b/engine/src/main/java/com/arcadedb/database/LocalDatabase.java @@ -1468,6 +1468,14 @@ public String getName() { return name; } + /** Override root + dbName subdir, or null if not configured. The subdir prevents collisions across databases. */ + public String resolveExternalBucketPath() { + final String configured = configuration.getValueAsString(GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH); + if (configured == null || configured.isEmpty()) + return null; + return configured + File.separator + name; + } + @Override public ComponentFile.MODE getMode() { return mode; @@ -2060,10 +2068,7 @@ private void openInternal() { DatabaseContext.INSTANCE.init(this); setLockingEnabled(configuration.getValueAsBoolean(GlobalConfiguration.BACKUP_ENABLED)); - // Optional second scan directory for paired external-property buckets that were tiered to cheaper storage - // via arcadedb.externalPropertyBucketPath. Empty by default; if set, FileManager opens .bucket files there too. - final String externalBucketPath = configuration.getValueAsString(GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH); - fileManager = new FileManager(databasePath, mode, SUPPORTED_FILE_EXT, externalBucketPath); + fileManager = new FileManager(databasePath, mode, SUPPORTED_FILE_EXT, resolveExternalBucketPath()); transactionManager = new TransactionManager(wrappedDatabaseInstance); open = true; diff --git a/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java b/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java index 67e31ea66b..635c7937ac 100644 --- a/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java +++ b/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java @@ -20,6 +20,7 @@ import com.arcadedb.database.Database; import com.arcadedb.database.DatabaseInternal; +import com.arcadedb.database.Document; import com.arcadedb.database.RID; import com.arcadedb.database.Record; import com.arcadedb.graph.GraphDatabaseChecker; @@ -28,6 +29,7 @@ import com.arcadedb.index.lsm.LSMTreeIndexAbstract; import com.arcadedb.log.LogManager; import com.arcadedb.schema.DocumentType; +import com.arcadedb.schema.LocalDocumentType; import com.arcadedb.schema.LocalEdgeType; import com.arcadedb.schema.LocalVertexType; import com.arcadedb.schema.Schema; @@ -71,6 +73,8 @@ public Map check() { checkBuckets(result); + checkExternalProperties(); + final Set affectedBuckets = new HashSet<>(); for (final RID rid : (Collection) result.get("corruptedRecords")) if (rid != null) @@ -254,6 +258,86 @@ public DatabaseChecker setCompress(final boolean compress) { return this; } + /** Detects (and on FIX deletes) external-property records that are no longer referenced by any primary record. */ + private void checkExternalProperties() { + if (verboseLevel > 0) + LogManager.instance().log(this, Level.INFO, "Checking external-property buckets..."); + + final List warnings = new ArrayList<>(); + final Set orphanedExternalRecords = new LinkedHashSet<>(); + long fixedCount = 0L; + + // For every external bucket, build the set of positions actually referenced from primary records of its + // owning type. Orphan = an external record whose position is NOT in that set. + final Map> referencedByExtBucketId = new HashMap<>(); + final Map extBucketsToCheck = new HashMap<>(); + + for (final DocumentType type : database.getSchema().getTypes()) { + if (!(type instanceof LocalDocumentType ldt) || !ldt.hasExternalProperties()) + continue; + if (types != null && !types.isEmpty() && !types.contains(type.getName())) + continue; + + for (final Bucket primaryBucket : type.getBuckets(false)) { + final Integer extBucketId = ldt.getExternalBucketIdFor(primaryBucket.getFileId()); + if (extBucketId == null) + continue; + final LocalBucket extBucket = (LocalBucket) database.getSchema().getBucketById(extBucketId); + extBucketsToCheck.put(extBucketId, extBucket); + final Set referenced = referencedByExtBucketId.computeIfAbsent(extBucketId, k -> new HashSet<>()); + + primaryBucket.scan((rid, view) -> { + try { + final Document record = (Document) database.getRecordFactory().newImmutableRecord(database, type, rid, view, null); + for (final RID extRid : database.getSerializer().findExistingExternalRids(database, record).values()) + referenced.add(extRid.getPosition()); + } catch (final Exception e) { + warnings.add("primary record " + rid + " could not be parsed for external pointer scan: " + e.getMessage()); + } + return true; + }, null); + } + } + + for (final Map.Entry entry : extBucketsToCheck.entrySet()) { + final LocalBucket extBucket = entry.getValue(); + final Set referenced = referencedByExtBucketId.get(entry.getKey()); + + final List orphans = new ArrayList<>(); + extBucket.scan((rid, view) -> { + if (!referenced.contains(rid.getPosition())) + orphans.add(rid); + return true; + }, null); + + orphanedExternalRecords.addAll(orphans); + + if (fix && !orphans.isEmpty()) { + final boolean startedNewTx = !database.isTransactionActive(); + if (startedNewTx) + database.begin(); + for (final RID orphan : orphans) { + try { + extBucket.deleteRecord(orphan); + // Mirror the accounting in LocalDatabase.cascadeDeleteExternalValues so count() stays consistent. + database.getTransaction().updateBucketRecordDelta(extBucket.getFileId(), -1); + fixedCount++; + } catch (final Exception e) { + warnings.add("could not delete orphan external record " + orphan + ": " + e.getMessage()); + } + } + if (startedNewTx) + database.commit(); + } + } + + result.put("orphanedExternalRecords", (long) orphanedExternalRecords.size()); + result.put("orphanedExternalRecordsFixed", fixedCount); + ((LinkedHashSet) result.get("warnings")).addAll(warnings); + if (fix) + ((LinkedHashSet) result.get("deletedRecordsAfterFix")).addAll(orphanedExternalRecords); + } + private void checkBuckets(final Map result) { if (verboseLevel > 0) LogManager.instance().log(this, Level.INFO, "Checking buckets..."); diff --git a/engine/src/main/java/com/arcadedb/engine/FileManager.java b/engine/src/main/java/com/arcadedb/engine/FileManager.java index 6213a17956..7350e88529 100644 --- a/engine/src/main/java/com/arcadedb/engine/FileManager.java +++ b/engine/src/main/java/com/arcadedb/engine/FileManager.java @@ -109,11 +109,13 @@ private void scanDirectoryForComponentFiles(final File dir, final Set su if (entries == null) return; for (final File f : entries) { - final String filePath = f.getAbsolutePath(); - final int lastDot = filePath.lastIndexOf("."); + // Compute the extension from the file name (not the full path) so a database directory containing dots + // (e.g. /home/u/my.db/bucket1) doesn't accidentally find the dot in the directory name. + final String fileName = f.getName(); + final int lastDot = fileName.lastIndexOf("."); if (lastDot < 0) continue; - final String fileExt = filePath.substring(lastDot + 1); + final String fileExt = fileName.substring(lastDot + 1); if (!supportedFileExt.contains(fileExt)) continue; try { diff --git a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java index 49fdeed4ac..802b77e4f6 100644 --- a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java +++ b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java @@ -21,6 +21,7 @@ import com.arcadedb.GlobalConfiguration; import com.arcadedb.database.Binary; import com.arcadedb.database.DatabaseInternal; +import com.arcadedb.database.Document; import com.arcadedb.database.RID; import com.arcadedb.database.Record; import com.arcadedb.database.RecordEventsRegistry; @@ -71,6 +72,11 @@ public class LocalBucket extends PaginatedComponent implements Bucket { public static final String BUCKET_EXT = "bucket"; public static final int CURRENT_VERSION = 0; + // Bucket file-format version 1 is reserved for paired external-property buckets. They hold heavier records, + // so the page-slot table is sized down to 256 (vs 2048 at v0): a 256KB page loses ~1034 bytes of header + // overhead versus ~8194 bytes at v0, sized to host typical 1-2KB records (esp. with compression enabled). + public static final int EXTERNAL_BUCKET_VERSION = 1; + private static final int DEF_MAX_RECORDS_IN_PAGE_V1 = 256; public static final long RECORD_PLACEHOLDER_POINTER = -1L; // USE -1 AS SIZE TO STORE A PLACEHOLDER (THAT POINTS TO A RECORD ON ANOTHER PAGE) public static final long FIRST_CHUNK = -2L; // USE -2 TO MARK THE FIRST CHUNK OF A BIG RECORD. FOLLOWS THE CHUNK SIZE AND THE POINTER TO THE NEXT CHUNK public static final long NEXT_CHUNK = -3L; // USE -3 TO MARK THE SECOND AND FURTHER CHUNK THAT IS PART OF A BIG RECORD THAT DOES NOT FIT A PAGE. FOLLOWS THE CHUNK SIZE AND THE POINTER TO THE NEXT CHUNK OR 0 IF THE CURRENT CHUNK IS THE LAST (NO FURTHER CHUNKS) @@ -87,7 +93,7 @@ public class LocalBucket extends PaginatedComponent implements Bucket { private static final int GATHER_STATS_MIN_SPACE_PERC = 10; private static final int SPARE_SPACE_FOR_GROWTH = 32; protected final int contentHeaderSize; - private final int maxRecordsInPage = DEF_MAX_RECORDS_IN_PAGE; + private final int maxRecordsInPage; private final AtomicLong cachedRecordCount = new AtomicLong(-1); // Buckets are PRIMARY by default (they hold the primary records of a type and are user-targetable via DML). // Internal kinds (e.g. EXTERNAL_PROPERTY) hold serializer infrastructure that user-facing DML must not target. @@ -141,6 +147,7 @@ public PaginatedComponent createOnLoad(final DatabaseInternal database, final St public LocalBucket(final DatabaseInternal database, final String name, final String filePath, final ComponentFile.MODE mode, final int pageSize, final int version) throws IOException { super(database, name, filePath, BUCKET_EXT, mode, pageSize, version); + this.maxRecordsInPage = maxRecordsInPageForVersion(version); this.contentHeaderSize = PAGE_RECORD_TABLE_OFFSET + (maxRecordsInPage * INT_SERIALIZED_SIZE); this.cachedRecordCount.set(0); this.reuseSpaceMode = REUSE_SPACE_MODE.valueOf(GlobalConfiguration.BUCKET_REUSE_SPACE_MODE.getValueAsString().toUpperCase()); @@ -152,6 +159,7 @@ public LocalBucket(final DatabaseInternal database, final String name, final Str public LocalBucket(final DatabaseInternal database, final String name, final String filePath, final int id, final ComponentFile.MODE mode, final int pageSize, final int version) throws IOException { super(database, name, filePath, id, mode, pageSize, version); + this.maxRecordsInPage = maxRecordsInPageForVersion(version); contentHeaderSize = PAGE_RECORD_TABLE_OFFSET + (maxRecordsInPage * INT_SERIALIZED_SIZE); this.reuseSpaceMode = REUSE_SPACE_MODE.valueOf(GlobalConfiguration.BUCKET_REUSE_SPACE_MODE.getValueAsString().toUpperCase()); if (this.reuseSpaceMode.ordinal() >= REUSE_SPACE_MODE.HIGH.ordinal()) @@ -168,6 +176,11 @@ public int getMaxRecordsInPage() { return maxRecordsInPage; } + /** Slot-table sizing for the bucket file format version. v0=2048 (legacy), v1=128 (paired external buckets). */ + private static int maxRecordsInPageForVersion(final int version) { + return version >= EXTERNAL_BUCKET_VERSION ? DEF_MAX_RECORDS_IN_PAGE_V1 : DEF_MAX_RECORDS_IN_PAGE; + } + public Purpose getPurpose() { return purpose; } @@ -179,10 +192,10 @@ public void setPurpose(final Purpose purpose) { @Override public RID createRecord(final Record record, final boolean discardRecordAfter) { database.checkPermissionsOnFile(fileId, SecurityDatabaseUser.ACCESS.CREATE_RECORD); - // Set a provisional identity so the serializer can resolve the target primary bucket id (used by EXTERNAL - // property handling to look up the paired external bucket). The actual position is filled in by - // createRecordInternal and overwrites this placeholder when the caller stores the returned RID. - if (record.getIdentity() == null && record instanceof RecordInternal ri) + // Provisional identity for Document records (and subtypes: vertex, edge) so the serializer can resolve the + // target primary bucket for EXTERNAL property handling. EdgeSegment, ExternalValueRecord, and other internal + // record types do not need this and must not be touched. + if (record.getIdentity() == null && record instanceof Document && record instanceof RecordInternal ri) ri.setIdentity(RID.create(database, fileId, -1L)); return createRecordInternal(record, false, discardRecordAfter); } diff --git a/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java b/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java index c7fa1dc0d5..944e692be0 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java +++ b/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java @@ -5729,14 +5729,24 @@ public RebuildIndexStatement visitRebuildIndexStatement(final SQLParser.RebuildI /** * Visit REBUILD TYPE statement. - * Grammar: REBUILD TYPE typeName [POLYMORPHIC] + * Grammar: REBUILD TYPE typeName [POLYMORPHIC] [WITH key = expression (, key = expression)*] */ @Override public RebuildTypeStatement visitRebuildTypeStmt(final SQLParser.RebuildTypeStmtContext ctx) { final RebuildTypeStatement stmt = new RebuildTypeStatement(-1); final SQLParser.RebuildTypeBodyContext bodyCtx = ctx.rebuildTypeBody(); - stmt.typeName = (Identifier) visit(bodyCtx.identifier()); + // The type name is the first identifier; any further identifiers are WITH-setting keys. + final List ids = bodyCtx.identifier(); + stmt.typeName = (Identifier) visit(ids.get(0)); stmt.polymorphic = bodyCtx.POLYMORPHIC() != null; + if (bodyCtx.WITH() != null) { + final List values = bodyCtx.expression(); + for (int i = 0; i < values.size(); i++) { + final Expression key = new Expression((Identifier) visit(ids.get(i + 1))); + final Expression value = (Expression) visit(values.get(i)); + stmt.settings.put(key, value); + } + } return stmt; } diff --git a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java index 5fdf2891a1..7a7b806035 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java +++ b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java @@ -28,6 +28,7 @@ import com.arcadedb.graph.Vertex; import com.arcadedb.index.Index; import com.arcadedb.schema.DocumentType; +import com.arcadedb.schema.LocalDocumentType; import com.arcadedb.schema.LocalTimeSeriesType; import com.arcadedb.schema.Schema; @@ -91,7 +92,7 @@ else if (type.getType() == Edge.RECORD_TYPE) // Expose the primary->external bucket mapping for types that have any EXTERNAL property. Lets tooling // (Studio etc.) tell the user where the externalised values for each primary bucket are stored. - if (type instanceof com.arcadedb.schema.LocalDocumentType ldt) { + if (type instanceof LocalDocumentType ldt) { final Map extMap = new HashMap<>(); for (final var b : type.getBuckets(false)) { final Integer extId = ldt.getExternalBucketIdFor(b.getFileId()); diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java b/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java index 6c2af1023f..ef5b5b2333 100755 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java @@ -91,6 +91,9 @@ public ResultSet executeDDL(final CommandContext context) { } else if (setting.equalsIgnoreCase("external")) { oldValue = property.isExternal(); property.setExternal((boolean) finalValue); + } else if (setting.equalsIgnoreCase("compression") || setting.equalsIgnoreCase("external_compression")) { + oldValue = property.getCompression(); + property.setCompression(String.valueOf(finalValue)); } else if (setting.equalsIgnoreCase("max")) { oldValue = property.getMax(); property.setMax("" + finalValue); diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java b/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java index 384ab2c2cc..cc39e1df7f 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java @@ -73,6 +73,8 @@ public Object setOnProperty(final Property internalProp, final CommandContext co internalProp.setHidden((boolean) attrValue); } else if (attrName.equalsIgnoreCase("external")) { internalProp.setExternal((boolean) attrValue); + } else if (attrName.equalsIgnoreCase("compression") || attrName.equalsIgnoreCase("external_compression")) { + internalProp.setCompression(String.valueOf(attrValue)); } else if (attrName.equalsIgnoreCase("max")) { internalProp.setMax("" + attrValue); } else if (attrName.equalsIgnoreCase("min")) { diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java b/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java index 89cd614e9b..aa288ed6c0 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java @@ -21,6 +21,7 @@ import com.arcadedb.database.Database; import com.arcadedb.database.MutableDocument; import com.arcadedb.exception.CommandExecutionException; +import com.arcadedb.exception.CommandSQLParsingException; import com.arcadedb.query.sql.executor.CommandContext; import com.arcadedb.query.sql.executor.InternalResultSet; import com.arcadedb.query.sql.executor.ResultInternal; @@ -31,21 +32,16 @@ import java.util.*; /** - * REBUILD TYPE typeName [POLYMORPHIC] + * REBUILD TYPE typeName [POLYMORPHIC] [WITH batchSize = N] - re-serialises records to apply schema layout changes. * - * Re-serialises every record of the named type (and optionally its subtypes) so that storage-layout schema changes - * are applied to records on disk. The primary use case is relocating values after toggling a property's EXTERNAL - * flag: after `ALTER PROPERTY T.p EXTERNAL true`, existing records still carry the value inline; running - * `REBUILD TYPE T` moves those values to the paired external bucket. The reverse case (EXTERNAL true -> false) is - * also handled, with orphan external records cleaned up automatically by the serializer. - * - * Commits in batches to keep memory bounded on large types. + * @author Luca Garulli (l.garulli@arcadedata.com) */ public class RebuildTypeStatement extends DDLStatement { - private static final int BATCH_SIZE = 10_000; + private static final int DEFAULT_BATCH_SIZE = 10_000; - public Identifier typeName; - public boolean polymorphic = false; + public Identifier typeName; + public boolean polymorphic = false; + public final Map settings = new HashMap<>(); public RebuildTypeStatement(final int id) { super(id); @@ -59,6 +55,17 @@ public ResultSet executeDDL(final CommandContext context) { if (type == null) throw new CommandExecutionException("Type not found: " + typeName.getStringValue()); + int batchSize = DEFAULT_BATCH_SIZE; + for (final Map.Entry e : settings.entrySet()) { + final String key = e.getKey().toString(); + if (key.equalsIgnoreCase("batchSize")) + batchSize = Integer.parseInt(e.getValue().value.toString()); + else + throw new CommandSQLParsingException( + "Unrecognized setting '" + key + "' in REBUILD TYPE statement (supported: batchSize)"); + } + final int finalBatchSize = batchSize; + final long[] count = { 0L }; final boolean implicitTx = !db.isTransactionActive(); if (implicitTx) @@ -66,14 +73,13 @@ public ResultSet executeDDL(final CommandContext context) { try { db.scanType(typeName.getStringValue(), polymorphic, rec -> { - // Re-save forces re-serialization which routes property values according to the current schema (e.g. moves - // values to/from the external bucket per the current EXTERNAL flag) and triggers orphan cleanup in the - // serializer for any external pointers that no longer apply. final MutableDocument m = (MutableDocument) rec.modify(); m.markDirty(); m.save(); count[0]++; - if (count[0] % BATCH_SIZE == 0) { + // Batch only when we own the transaction. Committing inside a caller-supplied TX would leak the user's + // writes prematurely and leave the trailing batch uncommitted. + if (implicitTx && count[0] % finalBatchSize == 0) { db.commit(); db.begin(); } diff --git a/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java b/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java index d45affb020..b76378a8ac 100644 --- a/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java +++ b/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java @@ -40,6 +40,8 @@ public abstract class AbstractProperty implements Property { protected boolean notNull = false; protected boolean hidden = false; protected boolean external = false; + // Compression policy for EXTERNAL property values: "none" | "auto" | "lz4". Null means "none". + protected String compression = null; protected String max = null; protected String min = null; protected String regexp = null; @@ -143,16 +145,16 @@ public boolean isHidden() { return hidden; } - /** - * Returns true if the property value is stored in a separate paired bucket (the external bucket of the type) instead of inline - * in the record. Useful for large payloads (vector embeddings, big strings, embedded JSON) so the primary bucket stays dense and - * page-cache friendly for traversal-heavy workloads. - */ @Override public boolean isExternal() { return external; } + @Override + public String getCompression() { + return compression == null ? "none" : compression; + } + @Override public String getMax() { return max; @@ -201,6 +203,8 @@ public JSONObject toJSON() { json.put("hidden", hidden); if (external) json.put("external", external); + if (compression != null && !"none".equalsIgnoreCase(compression)) + json.put("compression", compression); if (max != null) json.put("max", max); if (min != null) diff --git a/engine/src/main/java/com/arcadedb/schema/DocumentType.java b/engine/src/main/java/com/arcadedb/schema/DocumentType.java index 7358231113..2b3716ac5a 100644 --- a/engine/src/main/java/com/arcadedb/schema/DocumentType.java +++ b/engine/src/main/java/com/arcadedb/schema/DocumentType.java @@ -102,6 +102,8 @@ default Property createProperty(String propName, JSONObject prop) { p.setHidden(prop.getBoolean("hidden")); if (prop.has("external")) p.setExternal(prop.getBoolean("external")); + if (prop.has("compression")) + p.setCompression(prop.getString("compression")); if (prop.has("max")) p.setMax(prop.getString("max")); if (prop.has("min")) diff --git a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java index c343a8b961..9d994ce8df 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java @@ -18,8 +18,10 @@ */ package com.arcadedb.schema; +import com.arcadedb.GlobalConfiguration; import com.arcadedb.database.DatabaseInternal; import com.arcadedb.database.Document; +import com.arcadedb.database.LocalDatabase; import com.arcadedb.database.MutableDocument; import com.arcadedb.database.RecordEvents; import com.arcadedb.database.RecordEventsRegistry; @@ -971,10 +973,7 @@ protected void addBucketInternal(final Bucket bucket) { ensureExternalBucketFor((LocalBucket) bucket); } - /** - * Returns true if this type or any of its supertypes has at least one property flagged EXTERNAL. Polymorphic properties - * count: if A has an EXTERNAL property and B extends A, B is considered to have external properties. - */ + /** Polymorphic: counts inherited EXTERNAL properties too. */ public boolean hasExternalProperties() { for (final Property p : getPolymorphicProperties()) if (p.isExternal()) @@ -982,29 +981,16 @@ public boolean hasExternalProperties() { return false; } - /** - * Returns the external bucket id paired with the given primary bucket id, or null if no external bucket has been - * created for that primary bucket. - */ public Integer getExternalBucketIdFor(final int primaryBucketId) { return externalBucketIdByPrimaryBucketId.get(primaryBucketId); } - /** - * Idempotently ensures that every primary bucket of this type has a paired external bucket. Called when the first - * property transitions to EXTERNAL=true and when a new primary bucket is added to a type that already has external - * properties. - */ public void ensureExternalBuckets() { for (final Bucket b : buckets) ensureExternalBucketFor((LocalBucket) b); } - /** - * Like {@link #ensureExternalBuckets()} but also recurses into all subtypes. Used when an EXTERNAL property is set on - * a supertype: every concrete subtype must own paired external buckets for its own primary buckets, because records - * of that subtype live in the subtype's primary buckets, not the supertype's. - */ + /** Recurses into subtypes: records of a subtype live in subtype primary buckets, so each needs its own paired ext. */ public void ensureExternalBucketsRecursive() { ensureExternalBuckets(); for (final LocalDocumentType sub : subTypes) @@ -1016,36 +1002,50 @@ private void ensureExternalBucketFor(final LocalBucket primary) { return; final String extName = primary.getName() + "_ext"; final LocalBucket external; - if (schema.bucketMap.containsKey(extName)) + if (schema.bucketMap.containsKey(extName)) { external = schema.bucketMap.get(extName); - else { - // External buckets carry heavy payloads (vectors, large strings, embedded JSON), so they default to a - // larger page size than primary buckets to reduce multi-page chunking. Tunable via - // EXTERNAL_PROPERTY_BUCKET_DEFAULT_PAGE_SIZE. The file may also be tiered to a different directory via - // EXTERNAL_PROPERTY_BUCKET_PATH for cheaper-storage placement; the FileManager scans that path at startup. - final var config = schema.getDatabase().getConfiguration(); - final int pageSize = config.getValueAsInteger(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_DEFAULT_PAGE_SIZE); - final String overridePath = config.getValueAsString(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH); - external = schema.createBucket(extName, pageSize, overridePath); + // Refuse to adopt a bucket that is already registered as the primary bucket of some user type. A bucket + // freshly loaded from disk shows purpose=PRIMARY (transient field), so we cannot rely on purpose alone; + // bucketId2TypeMap is the authoritative source of "this bucket is a user type's primary bucket". + if (schema.getTypeByBucketId(external.getFileId()) != null + && external.getPurpose() != LocalBucket.Purpose.EXTERNAL_PROPERTY) + throw new SchemaException( + "Cannot adopt bucket '" + extName + "' as the external-property bucket for type '" + name + + "': it is already a primary bucket of another user type. Rename the conflicting bucket and retry."); + } else { + // External buckets get larger pages (256KB vs 64KB primary), a smaller slot table (128 vs 2048: file-format + // version EXTERNAL_BUCKET_VERSION), and optional placement on cheaper-storage tier via + // resolveExternalBucketPath() which returns /. + final int pageSize = schema.getDatabase().getConfiguration() + .getValueAsInteger(GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_DEFAULT_PAGE_SIZE); + final String overridePath = ((LocalDatabase) schema.getDatabase()).resolveExternalBucketPath(); + external = schema.createBucket(extName, pageSize, overridePath, LocalBucket.EXTERNAL_BUCKET_VERSION); } external.setPurpose(LocalBucket.Purpose.EXTERNAL_PROPERTY); externalBucketIdByPrimaryBucketId.put(primary.getFileId(), external.getFileId()); } - /** - * Internal hook called by LocalSchema after loading the type's external bucket map from JSON. Restores the - * primaryBucketId -> externalBucketId entries and stamps each external bucket with the EXTERNAL_PROPERTY purpose - * (which is transient on LocalBucket and must be re-applied on every load). - */ + /** Re-applies EXTERNAL_PROPERTY purpose (transient on LocalBucket) and rebuilds the map from JSON at load time. */ void restoreExternalBuckets(final Map primaryNameToExternalName) { externalBucketIdByPrimaryBucketId.clear(); for (final Map.Entry entry : primaryNameToExternalName.entrySet()) { final LocalBucket primary = schema.bucketMap.get(entry.getKey()); final LocalBucket external = schema.bucketMap.get(entry.getValue()); - if (primary == null || external == null) { + if (primary == null) { LogManager.instance() - .log(this, Level.WARNING, "Cannot restore external bucket mapping '%s' -> '%s' for type '%s'", null, - entry.getKey(), entry.getValue(), name); + .log(this, Level.WARNING, "Cannot restore external bucket mapping for type '%s': primary bucket '%s' not found", + null, name, entry.getKey()); + continue; + } + if (external == null) { + // Tiered bucket file not found: usually means arcadedb.externalPropertyBucketPath is unset on this restart + // but was set when the bucket was created. Reads of EXTERNAL properties for records in this primary + // bucket would silently fail - so we surface the configuration mismatch loudly and explicitly. + LogManager.instance().log(this, Level.SEVERE, + "Cannot find external bucket '%s' for type '%s' primary bucket '%s'. If the bucket was tiered to a " + + "secondary path, set 'arcadedb.externalPropertyBucketPath' to the same value used at creation " + + "time before reopening the database. EXTERNAL property reads on this type will fail until fixed.", + null, entry.getValue(), name, entry.getKey()); continue; } external.setPurpose(LocalBucket.Purpose.EXTERNAL_PROPERTY); diff --git a/engine/src/main/java/com/arcadedb/schema/LocalProperty.java b/engine/src/main/java/com/arcadedb/schema/LocalProperty.java index 7d5cc93c93..ac764495ea 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalProperty.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalProperty.java @@ -128,14 +128,29 @@ public Property setExternal(final boolean external) { if (changed) { this.external = external; if (external) - // ENSURE PAIRED EXTERNAL BUCKETS EXIST FOR EVERY PRIMARY BUCKET OF THIS TYPE AND ALL SUBTYPES (records of a - // subtype live in subtype primary buckets, so each subtype needs its own paired external buckets too). ((LocalDocumentType) owner).ensureExternalBucketsRecursive(); owner.getSchema().getEmbedded().saveConfiguration(); } return this; } + @Override + public Property setCompression(final String compression) { + final String normalized; + if (compression == null || compression.isEmpty() || "none".equalsIgnoreCase(compression)) + normalized = null; + else if ("auto".equalsIgnoreCase(compression) || "lz4".equalsIgnoreCase(compression)) + normalized = compression.toLowerCase(Locale.ENGLISH); + else + throw new IllegalArgumentException( + "Unsupported compression '" + compression + "' (supported: none, auto, lz4)"); + if (!Objects.equals(this.compression, normalized)) { + this.compression = normalized; + owner.getSchema().getEmbedded().saveConfiguration(); + } + return this; + } + @Override public Property setMax(final String max) { final boolean changed = !Objects.equals(this.max, max); diff --git a/engine/src/main/java/com/arcadedb/schema/LocalSchema.java b/engine/src/main/java/com/arcadedb/schema/LocalSchema.java index a95c4d60e2..01b1cc85e5 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalSchema.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalSchema.java @@ -372,16 +372,20 @@ public LocalBucket createBucket(final String bucketName) { } public LocalBucket createBucket(final String bucketName, final int pageSize) { - return createBucket(bucketName, pageSize, databasePath); + return createBucket(bucketName, pageSize, databasePath, LocalBucket.CURRENT_VERSION); + } + + /** Creates the bucket file under {@code parentDirectory} instead of the database directory; null/empty falls back. */ + public LocalBucket createBucket(final String bucketName, final int pageSize, final String parentDirectory) { + return createBucket(bucketName, pageSize, parentDirectory, LocalBucket.CURRENT_VERSION); } /** - * Creates a bucket whose underlying file lives at {@code parentDirectory + File.separator + bucketName} instead of - * the default database directory. Used by paired external-property buckets when - * {@code arcadedb.externalPropertyBucketPath} is configured, so the heavy payload files can sit on cheaper storage. - * Falls back to the database directory when {@code parentDirectory} is null or empty. + * Full overload: creates a bucket with an explicit file-format version. Paired external-property buckets pass + * {@link LocalBucket#EXTERNAL_BUCKET_VERSION} so they get the smaller (128-slot) page-slot table appropriate for + * heavy payloads; everything else uses {@link LocalBucket#CURRENT_VERSION}. */ - public LocalBucket createBucket(final String bucketName, final int pageSize, final String parentDirectory) { + public LocalBucket createBucket(final String bucketName, final int pageSize, final String parentDirectory, final int version) { database.checkPermissionsOnDatabase(SecurityDatabaseUser.DATABASE_ACCESS.UPDATE_SCHEMA); if (bucketMap.containsKey(bucketName)) @@ -395,7 +399,7 @@ public LocalBucket createBucket(final String bucketName, final int pageSize, fin if (!parent.exists() && !parent.mkdirs()) throw new SchemaException("Cannot create directory '" + dir + "' for bucket '" + bucketName + "'"); final LocalBucket bucket = new LocalBucket(database, bucketName, dir + File.separator + bucketName, - ComponentFile.MODE.READ_WRITE, pageSize, LocalBucket.CURRENT_VERSION); + ComponentFile.MODE.READ_WRITE, pageSize, version); registerFile((Component) bucket); bucketMap.put(bucketName, bucket); diff --git a/engine/src/main/java/com/arcadedb/schema/Property.java b/engine/src/main/java/com/arcadedb/schema/Property.java index a887ad0413..6398d4d57d 100644 --- a/engine/src/main/java/com/arcadedb/schema/Property.java +++ b/engine/src/main/java/com/arcadedb/schema/Property.java @@ -80,6 +80,14 @@ public interface Property { boolean isExternal(); + /** + * Compression policy for an EXTERNAL property's value: "none" (default), "auto" (try LZ4, keep compressed only + * if it saves >10%), or an explicit algorithm like "lz4". Ignored for non-EXTERNAL properties. + */ + Property setCompression(String compression); + + String getCompression(); + Property setMax(String max); String getMax(); diff --git a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java index d155819d90..f96d7957f0 100644 --- a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java +++ b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java @@ -274,10 +274,11 @@ else if (properties == 0) embeddedModifier != null ? new EmbeddedModifierProperty(embeddedModifier.getOwner(), propertyName) : null; final Object propertyValue; - if (type == BinaryTypes.TYPE_EXTERNAL) { - final int extBucketId = buffer.getInt(); - final long extPosition = buffer.getLong(); - propertyValue = readExternalValue((DatabaseInternal) database, extBucketId, extPosition, propertyModifier); + if (type == BinaryTypes.TYPE_EXTERNAL || type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_LZ4) { + final int extBucketId = (int) buffer.getNumber(); + final long extPosition = buffer.getNumber(); + propertyValue = readExternalValue((DatabaseInternal) database, extBucketId, extPosition, propertyModifier, + type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_LZ4); } else { propertyValue = deserializeValue(database, buffer, type, propertyModifier); } @@ -354,11 +355,11 @@ else if (properties == 0) final EmbeddedModifierProperty propertyModifier = embeddedModifier != null ? new EmbeddedModifierProperty(embeddedModifier.getOwner(), fieldName) : null; - if (type == BinaryTypes.TYPE_EXTERNAL) { - // VALUE LIVES IN A PAIRED EXTERNAL BUCKET. FOLLOW THE RID. - final int extBucketId = buffer.getInt(); - final long extPosition = buffer.getLong(); - return readExternalValue((DatabaseInternal) database, extBucketId, extPosition, propertyModifier); + if (type == BinaryTypes.TYPE_EXTERNAL || type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_LZ4) { + final int extBucketId = (int) buffer.getNumber(); + final long extPosition = buffer.getNumber(); + return readExternalValue((DatabaseInternal) database, extBucketId, extPosition, propertyModifier, + type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_LZ4); } return deserializeValue(database, buffer, type, propertyModifier); @@ -915,11 +916,14 @@ public Binary serializeProperties(final Database database, final Document record if (consumedExternalProperties != null && existingExtRid != null) consumedExternalProperties.add(propertyName); - final RID newExtRid = writeExternalValue((DatabaseInternal) database, extBucketId, existingExtRid, type, value); + final ExternalWriteResult written = writeExternalPropertyValue((DatabaseInternal) database, extBucketId, + existingExtRid, type, value, propertyDef.getCompression()); - content.putByte(BinaryTypes.TYPE_EXTERNAL); - content.putInt(newExtRid.getBucketId()); - content.putLong(newExtRid.getPosition()); + // The persisted type byte tells the reader which decoder to use. Bucket id and position are varints, + // mirroring TYPE_COMPRESSED_RID, so each pointer averages 3-7 bytes vs 12 fixed. + content.putByte(written.typeByte); + content.putNumber(written.rid.getBucketId()); + content.putNumber(written.rid.getPosition()); } else { if (value instanceof String stringValue && type == BinaryTypes.TYPE_STRING) { final int id = dictionary.getIdByName(stringValue, false); @@ -966,16 +970,112 @@ public Binary serializeProperties(final Database database, final Document record return header; } + /** Holder for {@link #writeExternalPropertyValue}: the bytes written and the type byte to put in the main record. */ + public static final class ExternalWriteResult { + public final byte typeByte; + public final RID rid; + + public ExternalWriteResult(final byte typeByte, final RID rid) { + this.typeByte = typeByte; + this.rid = rid; + } + } + + // Lazy-init: LZ4Factory.fastestInstance() does JNI/SIMD probing on first call, so we cache the wrapper. + private static volatile com.arcadedb.compression.LZ4Compression lz4Singleton; + + private static com.arcadedb.compression.LZ4Compression lz4() { + com.arcadedb.compression.LZ4Compression local = lz4Singleton; + if (local == null) { + synchronized (BinarySerializer.class) { + local = lz4Singleton; + if (local == null) + lz4Singleton = local = new com.arcadedb.compression.LZ4Compression(); + } + } + return local; + } + /** - * Builds a value-only blob and writes it to the given external bucket. If existingExternalRid is null, appends a new - * record (insert). Otherwise updates the record at that RID in place (update). The blob format is: - * 
-   * [ExternalValueRecord.RECORD_TYPE : 1B][value type byte : 1B][value bytes : ...]
-   *
- * Returns the RID where the blob was written. + * Serialises an EXTERNAL property value, optionally compressing per the property's policy + * ("none"|"auto"|"lz4"), writes the resulting blob to the paired external bucket, and returns the type byte + * the caller should put in the main record (TYPE_EXTERNAL or TYPE_EXTERNAL_COMPRESSED_LZ4). + * In "auto" mode compression is kept only when it saves more than 10% of bytes; otherwise the record is written + * raw. The decision is per-record so a single property can mix compressed and uncompressed records freely. */ + public ExternalWriteResult writeExternalPropertyValue(final DatabaseInternal database, final int externalBucketId, + final RID existingExternalRid, final byte valueType, final Object value, final String compressionPolicy) { + if (existingExternalRid != null && existingExternalRid.getBucketId() != externalBucketId) + throw new SerializationException( + "Existing external RID " + existingExternalRid + " does not match the paired external bucket id " + + externalBucketId + " for this record. The schema's external bucket mapping is inconsistent."); + + // Step 1: serialise the raw value bytes. We do this even when compressing, because we need both the raw size + // (uncompressed-size header) and the option to fall back to raw on auto-mode no-win. + final Binary rawValueBytes = new Binary(); + serializeValue(database, rawValueBytes, valueType, value); + rawValueBytes.flip(); + + final boolean tryLz4 = compressionPolicy != null + && ("auto".equalsIgnoreCase(compressionPolicy) || "lz4".equalsIgnoreCase(compressionPolicy)); + final boolean autoMode = "auto".equalsIgnoreCase(compressionPolicy); + + byte typeByte = BinaryTypes.TYPE_EXTERNAL; + byte[] compressedPayload = null; + int uncompressedSize = 0; + + if (tryLz4 && rawValueBytes.size() > 0) { + final byte[] raw = rawValueBytes.toByteArray(); + final byte[] compressed = lz4().compress(raw); + // In auto mode skip compression unless it saves >10%. Outside auto mode (explicit "lz4") always keep it. + if (!autoMode || compressed.length < raw.length * 0.9) { + typeByte = BinaryTypes.TYPE_EXTERNAL_COMPRESSED_LZ4; + compressedPayload = compressed; + uncompressedSize = raw.length; + } + } + + // Step 2: build the blob the bucket will store. + final Binary blob = new Binary(); + blob.putByte(ExternalValueRecord.RECORD_TYPE); + blob.putByte(valueType); + if (typeByte == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_LZ4) { + blob.putUnsignedNumber(uncompressedSize); + // putByteArray writes the raw bytes without a length prefix; the compressed payload runs to end-of-record. + blob.putByteArray(compressedPayload); + } else { + blob.append(rawValueBytes); + } + blob.flip(); + + // Step 3: insert or update in the paired bucket, with delta accounting consistent with cascade-delete. + final LocalBucket externalBucket = database.getSchema().getEmbedded().getBucketById(externalBucketId); + final RID rid; + if (existingExternalRid == null) { + final ExternalValueRecord rec = new ExternalValueRecord(database, null, blob); + rid = externalBucket.createRecord(rec, true); + database.getTransaction().updateBucketRecordDelta(externalBucket.getFileId(), +1); + } else { + final ExternalValueRecord rec = new ExternalValueRecord(database, existingExternalRid, blob); + rec.setIdentity(existingExternalRid); + externalBucket.updateRecord(rec, true); + rid = existingExternalRid; + } + + return new ExternalWriteResult(typeByte, rid); + } + + /** Insert or in-place update of an EXTERNAL property's value blob. Format: [RECORD_TYPE][type][value bytes]. */ public RID writeExternalValue(final DatabaseInternal database, final int externalBucketId, final RID existingExternalRid, final byte type, final Object value) { + // Refuse to update an existing external record at a different bucket than the type's currently-paired one. + // This would mean the schema's external bucket mapping has shifted under us (e.g. partial migration) and + // overwriting blindly could corrupt unrelated data. + if (existingExternalRid != null && existingExternalRid.getBucketId() != externalBucketId) + throw new SerializationException( + "Existing external RID " + existingExternalRid + " does not match the paired external bucket id " + + externalBucketId + " for this record. The schema's external bucket mapping is inconsistent."); + final Binary blob = new Binary(); blob.putByte(ExternalValueRecord.RECORD_TYPE); blob.putByte(type); @@ -998,25 +1098,35 @@ public RID writeExternalValue(final DatabaseInternal database, final int externa return existingExternalRid; } + public Object readExternalValue(final DatabaseInternal database, final int externalBucketId, final long position, + final EmbeddedModifier embeddedModifier) { + return readExternalValue(database, externalBucketId, position, embeddedModifier, false); + } + /** - * Reads the value blob at the given external RID and returns the deserialised value. The blob format must match - * {@link #writeExternalValue}. + * Reads the value blob at the given external RID. When {@code compressed} is true, the blob's value-bytes are + * LZ4-compressed and prefixed by an uncompressed-size varint. The compression flag is supplied by the caller - + * usually derived from the main record's type byte (TYPE_EXTERNAL vs TYPE_EXTERNAL_COMPRESSED_LZ4). */ public Object readExternalValue(final DatabaseInternal database, final int externalBucketId, final long position, - final EmbeddedModifier embeddedModifier) { + final EmbeddedModifier embeddedModifier, final boolean compressed) { final LocalBucket externalBucket = database.getSchema().getEmbedded().getBucketById(externalBucketId); final RID rid = RID.create(database, externalBucketId, position); final Binary buffer = externalBucket.getRecord(rid).copyOfContent(); buffer.position(Binary.BYTE_SERIALIZED_SIZE); // SKIP RECORD TYPE BYTE final byte valueType = buffer.getByte(); - return deserializeValue(database, buffer, valueType, embeddedModifier); + if (!compressed) + return deserializeValue(database, buffer, valueType, embeddedModifier); + + final int uncompressedSize = (int) buffer.getUnsignedNumber(); + final int compressedLen = buffer.size() - buffer.position(); + final byte[] compressedBytes = new byte[compressedLen]; + System.arraycopy(buffer.getContent(), buffer.position(), compressedBytes, 0, compressedLen); + final byte[] decompressed = lz4().decompress(compressedBytes, uncompressedSize); + return deserializeValue(database, new Binary(decompressed), valueType, embeddedModifier); } - /** - * Walks the OLD buffer of a Document being updated and collects the existing external RID for each EXTERNAL property, - * keyed by property name. Returns an empty map for new records (no identity, no buffer). - * Public so the database delete path can reuse it to cascade-delete external records. - */ + /** Reused by cascade-delete: scans the OLD buffer for TYPE_EXTERNAL pointers, keyed by property name. */ public Map findExistingExternalRids(final Database database, final Document record) { final RID identity = record.getIdentity(); if (identity == null) @@ -1046,9 +1156,12 @@ public Map findExistingExternalRids(final Database database, final buf.position(headerEndOffset + contentPosition); final byte type = buf.getByte(); - if (type == BinaryTypes.TYPE_EXTERNAL) { - final int extBucketId = buf.getInt(); - final long extPosition = buf.getLong(); + // Both raw and LZ4-compressed external pointers carry the same [bucketIdVarint][positionVarint] RID; + // the type byte differs only in how the blob is decoded - it doesn't change cascade-delete or orphan + // cleanup, both of which only need the RID. + if (type == BinaryTypes.TYPE_EXTERNAL || type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_LZ4) { + final int extBucketId = (int) buf.getNumber(); + final long extPosition = buf.getNumber(); if (result == null) result = new HashMap<>(); result.put(dictionary.getNameById(nameId), RID.create(database, extBucketId, extPosition)); @@ -1059,7 +1172,9 @@ public Map findExistingExternalRids(final Database database, final return result == null ? Collections.emptyMap() : result; } catch (Exception e) { LogManager.instance().log(this, Level.WARNING, - "Could not parse old buffer to recover external RIDs for record %s: %s", identity, e.getMessage()); + "Could not parse old buffer to recover external RIDs for record %s: %s. External records linked to this " + + "record may be orphaned in the paired bucket.", + e, identity, e.getMessage()); return Collections.emptyMap(); } } diff --git a/engine/src/main/java/com/arcadedb/serializer/BinaryTypes.java b/engine/src/main/java/com/arcadedb/serializer/BinaryTypes.java index 36d826b73c..ee2984001b 100644 --- a/engine/src/main/java/com/arcadedb/serializer/BinaryTypes.java +++ b/engine/src/main/java/com/arcadedb/serializer/BinaryTypes.java @@ -65,7 +65,8 @@ public class BinaryTypes { public final static byte TYPE_ARRAY_OF_FLOATS = 26; // @SINCE 23.6.1 public final static byte TYPE_ARRAY_OF_DOUBLES = 27; // @SINCE 23.6.1 public final static byte TYPE_COMPRESSED_GEOMETRY = 28; // @SINCE 26.2.1 - Binary geometry storage (Point, Circle, Rectangle, etc.) - public final static byte TYPE_EXTERNAL = 29; // @SINCE 26.5.1 - Property value stored in a paired external bucket. Followed by [bucketId:int][position:long]. + public final static byte TYPE_EXTERNAL = 29; // @SINCE 26.5.1 - Property value stored uncompressed in a paired external bucket. Followed by [bucketIdVarint][positionVarint]. + public final static byte TYPE_EXTERNAL_COMPRESSED_LZ4 = 30; // @SINCE 26.5.1 - Same as TYPE_EXTERNAL but the value bytes in the external blob are LZ4-compressed; the type byte is the dispatcher (no per-blob algo marker needed). // Geometry subtypes for TYPE_COMPRESSED_GEOMETRY public final static byte GEOMETRY_SUBTYPE_POINT = 1; // Point: x(double), y(double) diff --git a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java index e2d34b9152..3e3727d3b4 100644 --- a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java +++ b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java @@ -499,6 +499,11 @@ void externalBucketUsesLargerDefaultPageSize() { assertThat(primary.getPageSize()).isEqualTo(65_536); assertThat(external.getPageSize()).isEqualTo(262_144); assertThat(external.getPageSize()).isGreaterThan(primary.getPageSize()); + + // External buckets carry few but heavy records, so the slot table is sized down (128 vs 2048): saves about + // 7.5KB of header overhead per page (file-format version EXTERNAL_BUCKET_VERSION encodes this). + assertThat(primary.getMaxRecordsInPage()).isEqualTo(2048); + assertThat(external.getMaxRecordsInPage()).isEqualTo(256); } @Test @@ -521,16 +526,21 @@ void externalBucketPathOverridePlacesFileOnSecondaryDirectory() throws java.io.I saved[0] = d.getIdentity(); }); - // External bucket file lives in the override directory, not the database directory. + // External bucket file lives in the override directory, not the database directory. Use prefix filters + // (not hardcoded suffixes) so the assertions stay valid if the bucket file-naming convention evolves. final var primary = type.getBuckets(false).getFirst(); final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); - final java.io.File extFile = new java.io.File(database.getDatabasePath(), external.getName() + ".0.262144.v0.bucket"); - assertThat(extFile.exists()).as("external bucket should NOT be in the database directory").isFalse(); + // Per-database subdir is appended to the override path (so multiple DBs sharing the path can't collide). + final java.io.File tieredDbDir = new java.io.File(overrideDir.toFile(), database.getName()); + final java.io.File[] dbDirFiles = new java.io.File(database.getDatabasePath()).listFiles( + (dir, name) -> name.startsWith(external.getName() + ".")); + assertThat(dbDirFiles).as("external bucket should NOT be in the database directory") + .satisfiesAnyOf(arr -> assertThat(arr).isNull(), arr -> assertThat(arr).isEmpty()); - final java.io.File[] tieredFiles = overrideDir.toFile().listFiles((dir, name) -> name.startsWith(external.getName())); - assertThat(tieredFiles).as("external bucket should be in the override directory").isNotNull().isNotEmpty(); + final java.io.File[] tieredFiles = tieredDbDir.listFiles((dir, name) -> name.startsWith(external.getName() + ".")); + assertThat(tieredFiles).as("external bucket should be in //").isNotNull().isNotEmpty(); // Reopen: FileManager must rediscover the tiered file via the secondary scan path so the record stays readable. // The override is applied at open() via the global config, so set it there too before reopening. @@ -547,19 +557,158 @@ void externalBucketPathOverridePlacesFileOnSecondaryDirectory() throws java.io.I } } + @Test + void checkDatabaseDetectsAndFixesOrphanedExternalRecords() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + database.transaction(() -> database.newDocument("Doc").set("blob", "referenced").save()); + + final var primary = type.getBuckets(false).getFirst(); + final Integer extBucketId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket externalBucket = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extBucketId); + final long extCountBefore = externalBucket.count(); + + // Inject an orphan: write a value blob directly to the external bucket, bypassing the property write path + // so no primary record references it. + database.transaction(() -> ((com.arcadedb.database.DatabaseInternal) database).getSerializer() + .writeExternalValue((com.arcadedb.database.DatabaseInternal) database, extBucketId, null, + com.arcadedb.serializer.BinaryTypes.TYPE_STRING, "orphan-payload")); + + assertThat(externalBucket.count()).isEqualTo(extCountBefore + 1); + + // CHECK DATABASE (no FIX): reports the orphan but does not delete it. + final ResultSet rs = database.command("sql", "CHECK DATABASE"); + assertThat(rs.hasNext()).isTrue(); + final var row = rs.next(); + assertThat((Long) row.getProperty("orphanedExternalRecords")).isGreaterThanOrEqualTo(1L); + assertThat((Long) row.getProperty("orphanedExternalRecordsFixed")).isEqualTo(0L); + assertThat(externalBucket.count()).isEqualTo(extCountBefore + 1); + + // CHECK DATABASE FIX: removes the orphan. + final ResultSet rsFix = database.command("sql", "CHECK DATABASE FIX"); + assertThat(rsFix.hasNext()).isTrue(); + assertThat((Long) rsFix.next().getProperty("orphanedExternalRecordsFixed")).isGreaterThanOrEqualTo(1L); + assertThat(externalBucket.count()).isEqualTo(extCountBefore); + + // Re-running reports zero orphans. + final ResultSet rsAfter = database.command("sql", "CHECK DATABASE"); + assertThat((Long) rsAfter.next().getProperty("orphanedExternalRecords")).isEqualTo(0L); + } + + @Test + void compressedExternalPropertyRoundTripsLargeText() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("text", Type.STRING).setExternal(true).setCompression("lz4"); + + // Highly redundant text compresses to a fraction of its raw size. + final StringBuilder sb = new StringBuilder(); + final String fragment = "the quick brown fox jumps over the lazy dog "; + for (int i = 0; i < 200; i++) + sb.append(fragment); + final String value = sb.toString(); + final int rawSize = value.length(); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Doc").set("text", value); + d.save(); + saved[0] = d.getIdentity(); + }); + + database.close(); + database = factory.open(); + + // Round-trip: the compressed bytes deserialise back to the original text. + final var loaded = database.lookupByRID(saved[0], true).asDocument(); + assertThat(loaded.getString("text")).isEqualTo(value); + + // External bucket should hold less than half the raw text bytes (typical LZ4 ratio on repeated prose). + final var primary = type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) database.getSchema().getType("Doc")).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket externalBucket = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + assertThat(externalBucket.getTotalPages()).as("only one page expected for one ~9KB text record").isEqualTo(1); + // We can't easily inspect just the record bytes, but if we shipped uncompressed the record itself would be + // ~9KB; with LZ4 it will land far below. + } + + @Test + void compressionAutoSkipsWhenIncompressible() { + final VertexType type = database.getSchema().createVertexType("V"); + type.createProperty("embedding", Type.ARRAY_OF_FLOATS).setExternal(true).setCompression("auto"); + + // High-entropy float bits do not compress; auto-mode should fall back to TYPE_EXTERNAL (raw). + final float[] embedding = new float[1024]; + final java.util.Random rnd = new java.util.Random(42); + for (int i = 0; i < embedding.length; i++) + embedding[i] = rnd.nextFloat() * 1000f; + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableVertex v = database.newVertex("V").set("embedding", embedding); + v.save(); + saved[0] = v.getIdentity(); + }); + + database.close(); + database = factory.open(); + + // Reads back identical bytes whichever path was chosen. + final var loaded = database.lookupByRID(saved[0], true).asVertex(); + final float[] readBack = (float[]) loaded.get("embedding"); + assertThat(readBack).isEqualTo(embedding); + } + + @Test + void compressionFlagPersistsAcrossReopen() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("body", Type.STRING).setExternal(true).setCompression("auto"); + + assertThat(type.getProperty("body").getCompression()).isEqualToIgnoringCase("auto"); + + database.close(); + database = factory.open(); + + final DocumentType reloaded = database.getSchema().getType("Doc"); + assertThat(reloaded.getProperty("body").isExternal()).isTrue(); + assertThat(reloaded.getProperty("body").getCompression()).isEqualToIgnoringCase("auto"); + } + + @Test + void sqlDdlExternalCompression() { + database.transaction(() -> { + database.command("sql", "CREATE DOCUMENT TYPE Doc"); + database.command("sql", "CREATE PROPERTY Doc.body STRING (EXTERNAL true, COMPRESSION 'auto')"); + }); + assertThat(database.getSchema().getType("Doc").getProperty("body").getCompression()).isEqualToIgnoringCase("auto"); + + database.transaction(() -> database.command("sql", "ALTER PROPERTY Doc.body COMPRESSION 'lz4'")); + assertThat(database.getSchema().getType("Doc").getProperty("body").getCompression()).isEqualToIgnoringCase("lz4"); + + database.transaction(() -> database.command("sql", "ALTER PROPERTY Doc.body COMPRESSION 'none'")); + assertThat(database.getSchema().getType("Doc").getProperty("body").getCompression()).isEqualToIgnoringCase("none"); + } + @Test void rollbackDiscardsBothPrimaryAndExternal() { final DocumentType type = database.getSchema().createDocumentType("Doc"); type.createProperty("blob", Type.STRING).setExternal(true); + final var primary = type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + final long primaryCountBefore = database.countType("Doc", false); + final long externalCountBefore = external.count(); database.begin(); final MutableDocument d = database.newDocument("Doc").set("blob", "rolled-back"); d.save(); database.rollback(); - final long primaryCountAfter = database.countType("Doc", false); - assertThat(primaryCountAfter).isEqualTo(primaryCountBefore); + // Both halves of the WAL group must be reverted: primary record AND its paired external blob. + assertThat(database.countType("Doc", false)).isEqualTo(primaryCountBefore); + assertThat(external.count()).as("external bucket count must also be unchanged after rollback") + .isEqualTo(externalCountBefore); } } diff --git a/ha-raft/src/test/java/com/arcadedb/server/ha/raft/RaftLeaderCrashWithExternalPropertyIT.java b/ha-raft/src/test/java/com/arcadedb/server/ha/raft/RaftLeaderCrashWithExternalPropertyIT.java new file mode 100644 index 0000000000..f4375beef9 --- /dev/null +++ b/ha-raft/src/test/java/com/arcadedb/server/ha/raft/RaftLeaderCrashWithExternalPropertyIT.java @@ -0,0 +1,244 @@ +/* + * Copyright 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.server.ha.raft; + +import com.arcadedb.ContextConfiguration; +import com.arcadedb.GlobalConfiguration; +import com.arcadedb.database.Database; +import com.arcadedb.database.RID; +import com.arcadedb.engine.LocalBucket; +import com.arcadedb.graph.MutableVertex; +import com.arcadedb.graph.Vertex; +import com.arcadedb.log.LogManager; +import com.arcadedb.schema.LocalDocumentType; +import com.arcadedb.schema.LocalSchema; +import com.arcadedb.schema.Type; +import com.arcadedb.schema.VertexType; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Verifies primary + paired external bucket pages are replicated atomically when the leader crashes mid-commit. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +@Tag("slow") +class RaftLeaderCrashWithExternalPropertyIT extends BaseRaftHATest { + + private static final String VERTEX_TYPE = "ExtCrash"; + private static final String EXT_PROPERTY = "blob"; + private static final String INJECTED_PAYLOAD = "injected-payload-the-quick-brown-fox-jumps-over-the-lazy-dog"; + + @Override + protected void onServerConfiguration(final ContextConfiguration config) { + super.onServerConfiguration(config); + config.setValue(GlobalConfiguration.HA_QUORUM, "majority"); + } + + @Override + protected int getServerCount() { + return 3; + } + + @Override + protected boolean persistentRaftStorage() { + return true; + } + + @AfterEach + void clearPostReplicationHook() { + RaftReplicatedDatabase.TEST_POST_REPLICATION_HOOK = null; + } + + @Test + void externalPropertyRecoversAtomicallyAfterLeaderCrashBetweenReplicationAndPhase2() throws Exception { + final int leaderIndex = findLeaderIndex(); + assertThat(leaderIndex).as("A Raft leader must be elected").isGreaterThanOrEqualTo(0); + + final Database leaderDb = getServerDatabase(leaderIndex, getDatabaseName()); + + // Phase 1: schema + baseline writes with no fault injection. Each save touches BOTH the primary bucket + // and the paired external bucket in a single transaction. + leaderDb.transaction(() -> { + if (!leaderDb.getSchema().existsType(VERTEX_TYPE)) { + final VertexType type = leaderDb.getSchema().createVertexType(VERTEX_TYPE); + type.createProperty(EXT_PROPERTY, Type.STRING).setExternal(true); + } + }); + final int baseline = 50; + leaderDb.transaction(() -> { + for (int i = 0; i < baseline; i++) { + final MutableVertex v = leaderDb.newVertex(VERTEX_TYPE); + v.set("name", "baseline-" + i); + v.set(EXT_PROPERTY, "baseline-blob-" + i); + v.save(); + } + }); + assertClusterConsistency(); + assertExternalBucketsAlignWithPrimary(baseline); + + // Phase 2: arm the fault-injection hook. Single-shot: fires on the next successful Raft replication and + // (a) stops the leader on a separate thread (stopping inline would deadlock the Ratis gRPC channel), + // (b) throws so commit2ndPhase() never runs locally on the crashed leader. + final AtomicBoolean hookFired = new AtomicBoolean(false); + final CountDownLatch leaderStopped = new CountDownLatch(1); + RaftReplicatedDatabase.TEST_POST_REPLICATION_HOOK = dbName -> { + if (!hookFired.compareAndSet(false, true)) + return; + LogManager.instance().log(RaftLeaderCrashWithExternalPropertyIT.class, Level.INFO, + "TEST: fault-injection hook firing for db=%s, stopping leader %d asynchronously", + dbName, leaderIndex); + final Thread stopper = new Thread(() -> { + try { + getServer(leaderIndex).stop(); + } catch (final Throwable t) { + LogManager.instance().log(RaftLeaderCrashWithExternalPropertyIT.class, Level.WARNING, + "TEST: async leader stop failed: %s", t.getMessage()); + } finally { + leaderStopped.countDown(); + } + }, "TEST-fault-injection-stop"); + stopper.setDaemon(true); + stopper.start(); + throw new RuntimeException( + "TEST fault injection: simulated leader crash between Raft commit and commit2ndPhase"); + }; + + // Phase 3: write a record whose EXTERNAL property has a recognisable payload. The transaction commits + // BOTH primary and external bucket pages; the fault fires AFTER Raft has replicated both to followers. + final RID[] injectedRid = new RID[1]; + try { + leaderDb.begin(); + final MutableVertex v = leaderDb.newVertex(VERTEX_TYPE); + v.set("name", "injected-0"); + v.set(EXT_PROPERTY, INJECTED_PAYLOAD); + v.save(); + injectedRid[0] = v.getIdentity(); + leaderDb.commit(); + } catch (final Exception expected) { + LogManager.instance().log(this, Level.INFO, + "TEST: leader commit failed as expected: %s", expected.getMessage()); + } + + assertThat(hookFired.get()).as("Fault-injection hook must have fired").isTrue(); + assertThat(leaderStopped.await(30, TimeUnit.SECONDS)) + .as("Async leader stop must complete within 30s").isTrue(); + + // Phase 4: a new leader must emerge from the 2 survivors. + final int newLeaderIndex = waitForNewLeader(leaderIndex); + assertThat(newLeaderIndex).as("A new leader must be elected").isGreaterThanOrEqualTo(0); + assertThat(newLeaderIndex).as("New leader must differ from crashed leader").isNotEqualTo(leaderIndex); + LogManager.instance().log(this, Level.INFO, "TEST: new leader elected: server %d", newLeaderIndex); + + // Phase 5: the injected record AND its EXTERNAL value must be visible on the new leader. This proves the + // primary AND external bucket pages were replicated atomically before the leader crashed (and that + // Raft followers applied them as a unit). + waitForReplicationIsCompleted(newLeaderIndex); + final Database newLeaderDb = getServerDatabase(newLeaderIndex, getDatabaseName()); + assertThat(newLeaderDb.countType(VERTEX_TYPE, true)) + .as("New leader must have baseline + injected record") + .isEqualTo(baseline + 1L); + final Vertex injectedOnNewLeader = newLeaderDb.lookupByRID(injectedRid[0], true).asVertex(); + assertThat(injectedOnNewLeader.getString(EXT_PROPERTY)) + .as("EXTERNAL value must be readable on the new leader (atomic primary+external replication)") + .isEqualTo(INJECTED_PAYLOAD); + + // Phase 6: restart the crashed leader. Its Raft log has the committed entry (primary+external pages) but + // commit2ndPhase() never ran, so neither half is on disk. Ratis replays the entry through the state + // machine follower path, applying both halves in lock-step. + Thread.sleep(2_000); + LogManager.instance().log(this, Level.INFO, "TEST: restarting old leader %d", leaderIndex); + getServer(leaderIndex).start(); + + waitForReplicationIsCompleted(leaderIndex); + + // Phase 7: the recovered old leader must hold both halves. Specifically: + // - countType reflects the injected record (primary bucket recovered) + // - reading the EXTERNAL property returns the original payload (external bucket recovered) + // - the external bucket count matches the primary bucket count (no half-records, no orphans) + final Database oldLeaderDb = getServerDatabase(leaderIndex, getDatabaseName()); + assertThat(oldLeaderDb.countType(VERTEX_TYPE, true)) + .as("Recovered old leader must have baseline + injected record") + .isEqualTo(baseline + 1L); + final Vertex injectedOnOldLeader = oldLeaderDb.lookupByRID(injectedRid[0], true).asVertex(); + assertThat(injectedOnOldLeader.getString(EXT_PROPERTY)) + .as("EXTERNAL value must be readable on recovered old leader (atomic WAL replay of both buckets)") + .isEqualTo(INJECTED_PAYLOAD); + assertExternalBucketsAlignWithPrimary(baseline + 1); + + // Phase 8: cross-node page-level convergence (already counts external buckets - they are regular components). + assertClusterConsistency(); + } + + /** Catches half-record corruption: each primary bucket and its paired ext bucket must hold the same row count. */ + private void assertExternalBucketsAlignWithPrimary(final int expectedRecordCount) { + for (int i = 0; i < getServerCount(); i++) { + if (getServer(i) == null || !getServer(i).isStarted()) + continue; + final Database db = getServerDatabase(i, getDatabaseName()); + final LocalDocumentType type = (LocalDocumentType) db.getSchema().getType(VERTEX_TYPE); + final LocalSchema schema = (LocalSchema) db.getSchema().getEmbedded(); + long primaryTotal = 0; + long externalTotal = 0; + for (final var b : type.getBuckets(false)) { + primaryTotal += b.count(); + final Integer extId = type.getExternalBucketIdFor(b.getFileId()); + assertThat(extId).as("type '%s' on server %d primary bucket '%s' must have a paired external bucket", + VERTEX_TYPE, i, b.getName()).isNotNull(); + final LocalBucket extBucket = schema.getBucketById(extId); + externalTotal += extBucket.count(); + } + assertThat(primaryTotal) + .as("server %d: primary bucket count for '%s'", i, VERTEX_TYPE) + .isEqualTo(expectedRecordCount); + assertThat(externalTotal) + .as("server %d: external bucket count must equal primary bucket count for '%s' " + + "(half-record indicates WAL replay applied only one half of the transaction)", i, VERTEX_TYPE) + .isEqualTo(expectedRecordCount); + } + } + + private int waitForNewLeader(final int excludeIndex) { + final long deadline = System.currentTimeMillis() + 30_000; + while (System.currentTimeMillis() < deadline) { + for (int i = 0; i < getServerCount(); i++) { + if (i == excludeIndex) + continue; + final RaftHAPlugin plugin = getRaftPlugin(i); + if (plugin != null && plugin.isLeader()) + return i; + } + try { + Thread.sleep(500); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + return -1; + } + } + return -1; + } +} diff --git a/network/src/main/java/com/arcadedb/remote/RemoteProperty.java b/network/src/main/java/com/arcadedb/remote/RemoteProperty.java index 735b5ac698..ec40d8a913 100644 --- a/network/src/main/java/com/arcadedb/remote/RemoteProperty.java +++ b/network/src/main/java/com/arcadedb/remote/RemoteProperty.java @@ -76,6 +76,11 @@ public Property setExternal(boolean external) { throw new UnsupportedOperationException(); } + @Override + public Property setCompression(String compression) { + throw new UnsupportedOperationException(); + } + @Override public Property setMax(String max) { throw new UnsupportedOperationException(); @@ -113,6 +118,8 @@ void reload(final Map entry) { hidden = (Boolean) entry.get("hidden"); if (entry.containsKey("external")) external = (Boolean) entry.get("external"); + if (entry.containsKey("compression")) + compression = (String) entry.get("compression"); if (entry.containsKey("default")) defaultValue = entry.get("default"); if (entry.containsKey("regexp")) From 9d96fb629a9ae57b3af92b8d6af9db68a7c9fde7 Mon Sep 17 00:00:00 2001 From: Luca Garulli Date: Wed, 29 Apr 2026 14:26:21 -0400 Subject: [PATCH 06/12] Fixed claude review --- .../com/arcadedb/database/LocalDatabase.java | 8 +- .../java/com/arcadedb/engine/LocalBucket.java | 6 +- .../executor/FetchFromSchemaTypesStep.java | 5 + .../sql/parser/RebuildTypeStatement.java | 45 ++++++++- .../com/arcadedb/schema/AbstractProperty.java | 6 +- .../arcadedb/schema/LocalDocumentType.java | 92 +++++++++++++------ .../com/arcadedb/schema/LocalProperty.java | 9 +- .../java/com/arcadedb/schema/LocalSchema.java | 2 +- .../arcadedb/serializer/BinarySerializer.java | 36 +------- .../arcadedb/schema/ExternalPropertyTest.java | 54 +++++++---- .../resources/static/js/studio-database.js | 29 +++++- 11 files changed, 200 insertions(+), 92 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/database/LocalDatabase.java b/engine/src/main/java/com/arcadedb/database/LocalDatabase.java index be31694062..bb3af6c0a2 100644 --- a/engine/src/main/java/com/arcadedb/database/LocalDatabase.java +++ b/engine/src/main/java/com/arcadedb/database/LocalDatabase.java @@ -1181,7 +1181,7 @@ private void cascadeDeleteExternalValues(final Document document) { final LocalBucket externalBucket = schema.getBucketById(extRid.getBucketId(), false); if (externalBucket != null) { externalBucket.deleteRecord(extRid); - // Keep the external bucket's count consistent (mirrors the +1 in BinarySerializer.writeExternalValue). + // Keep the external bucket's count consistent (mirrors the +1 in BinarySerializer.writeExternalPropertyValue). getTransaction().updateBucketRecordDelta(externalBucket.getFileId(), -1); } } @@ -2068,7 +2068,11 @@ private void openInternal() { DatabaseContext.INSTANCE.init(this); setLockingEnabled(configuration.getValueAsBoolean(GlobalConfiguration.BACKUP_ENABLED)); - fileManager = new FileManager(databasePath, mode, SUPPORTED_FILE_EXT, resolveExternalBucketPath()); + final String resolved = resolveExternalBucketPath(); + System.out.println("[DEBUG-FM] open db=" + name + " externalBucketPath=" + resolved + + " configValue=" + configuration.getValueAsString(GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH) + + " configHash=" + System.identityHashCode(configuration)); + fileManager = new FileManager(databasePath, mode, SUPPORTED_FILE_EXT, resolved); transactionManager = new TransactionManager(wrappedDatabaseInstance); open = true; diff --git a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java index 802b77e4f6..2ca2cae62f 100644 --- a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java +++ b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java @@ -73,8 +73,10 @@ public class LocalBucket extends PaginatedComponent implements Bucket { public static final String BUCKET_EXT = "bucket"; public static final int CURRENT_VERSION = 0; // Bucket file-format version 1 is reserved for paired external-property buckets. They hold heavier records, - // so the page-slot table is sized down to 256 (vs 2048 at v0): a 256KB page loses ~1034 bytes of header - // overhead versus ~8194 bytes at v0, sized to host typical 1-2KB records (esp. with compression enabled). + // so the page-slot table is sized down to 256 (vs 2048 at v0). The bucket-page header (computed by + // contentHeaderSize = PAGE_RECORD_TABLE_OFFSET + maxSlots*4) shrinks from 8194 bytes at v0 to 1026 bytes at + // v1 - reclaiming ~7KB of header per page that becomes available for value blobs. Sized to host typical + // 1-2KB records (especially with compression enabled). public static final int EXTERNAL_BUCKET_VERSION = 1; private static final int DEF_MAX_RECORDS_IN_PAGE_V1 = 256; public static final long RECORD_PLACEHOLDER_POINTER = -1L; // USE -1 AS SIZE TO STORE A PLACEHOLDER (THAT POINTS TO A RECORD ON ANOTHER PAGE) diff --git a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java index 7a7b806035..78aa9c8d0d 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java +++ b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java @@ -128,6 +128,11 @@ else if (type.getType() == Edge.RECORD_TYPE) propRes.setProperty("hidden", property.isHidden()); if (property.isExternal()) propRes.setProperty("external", property.isExternal()); + // Only emit the compression policy when explicitly set to a non-default value, mirroring the + // toJSON convention. Studio renders the badge tooltip and a small label off this field. + final String cmp = property.getCompression(); + if (cmp != null && !"none".equalsIgnoreCase(cmp)) + propRes.setProperty("compression", cmp); if (property.getMin() != null) propRes.setProperty("min", property.getMin()); if (property.getMax() != null) diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java b/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java index aa288ed6c0..2e09720212 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java @@ -67,6 +67,9 @@ public ResultSet executeDDL(final CommandContext context) { final int finalBatchSize = batchSize; final long[] count = { 0L }; + // Records committed in earlier batches and therefore NOT rolled back by a later failure: needed so we can + // tell the caller exactly how many rows are already in the new layout when the rebuild aborts mid-stream. + final long[] committedBefore = { 0L }; final boolean implicitTx = !db.isTransactionActive(); if (implicitTx) db.begin(); @@ -81,6 +84,7 @@ public ResultSet executeDDL(final CommandContext context) { // writes prematurely and leave the trailing batch uncommitted. if (implicitTx && count[0] % finalBatchSize == 0) { db.commit(); + committedBefore[0] = count[0]; db.begin(); } return true; @@ -91,9 +95,29 @@ public ResultSet executeDDL(final CommandContext context) { } catch (Exception e) { if (implicitTx && db.isTransactionActive()) db.rollback(); - throw new CommandExecutionException("Error on rebuilding type '" + typeName.getStringValue() + "'", e); + // REBUILD TYPE is NOT atomic across batch boundaries: every db.commit() above already persisted those + // records in the new layout. After rollback, only the in-flight batch is reverted; the type is left + // half-migrated. Surface the boundary clearly so the operator knows to re-run REBUILD TYPE to finish. + final long migratedAndKept = committedBefore[0]; + final long rolledBack = count[0] - committedBefore[0]; + throw new CommandExecutionException( + "Error on rebuilding type '" + typeName.getStringValue() + "' after " + count[0] + " records (" + + migratedAndKept + " committed in earlier batches and remain in the new layout, " + rolledBack + + " rolled back from the in-flight batch). REBUILD TYPE is NOT atomic across batches; re-run the" + + " command to migrate the remaining records once the underlying issue is fixed.", e); } + // If the rebuild was triggered to revert a property from EXTERNAL to inline (i.e. the type no longer has + // any EXTERNAL property), the previously-paired external buckets are now empty (orphan-cleanup in + // serializeProperties deleted every external blob during the re-save). Drop them so they don't accumulate + // across toggle cycles, and persist the cleared mapping. + // Reclaim paired external buckets only if REBUILD owned the transaction. With a caller-supplied tx the + // queued record updates haven't flushed yet (LocalDatabase.updateRecord defers serialization to commit), + // so the orphan cleanup hasn't run and the buckets are still non-empty. The caller can re-run REBUILD + // outside their tx to trigger reclaim, or invoke it explicitly. + if (implicitTx && type instanceof com.arcadedb.schema.LocalDocumentType ldt && !ldt.hasExternalProperties()) + ldt.reclaimEmptyExternalBuckets(); + final ResultInternal result = new ResultInternal(db); result.setProperty("operation", "rebuild type"); result.setProperty("typeName", typeName.getStringValue()); @@ -110,6 +134,18 @@ public void toString(final Map params, final StringBuilder build typeName.toString(params, builder); if (polymorphic) builder.append(" POLYMORPHIC"); + if (!settings.isEmpty()) { + builder.append(" WITH "); + boolean first = true; + for (final Map.Entry e : settings.entrySet()) { + if (!first) + builder.append(", "); + e.getKey().toString(params, builder); + builder.append(" = "); + e.getValue().toString(params, builder); + first = false; + } + } } @Override @@ -117,6 +153,8 @@ public RebuildTypeStatement copy() { final RebuildTypeStatement result = new RebuildTypeStatement(-1); result.typeName = typeName == null ? null : typeName.copy(); result.polymorphic = polymorphic; + for (final Map.Entry e : settings.entrySet()) + result.settings.put(e.getKey().copy(), e.getValue().copy()); return result; } @@ -127,13 +165,16 @@ public boolean equals(final Object o) { if (o == null || getClass() != o.getClass()) return false; final RebuildTypeStatement that = (RebuildTypeStatement) o; - return polymorphic == that.polymorphic && Objects.equals(typeName, that.typeName); + return polymorphic == that.polymorphic + && Objects.equals(typeName, that.typeName) + && Objects.equals(settings, that.settings); } @Override public int hashCode() { int result = typeName != null ? typeName.hashCode() : 0; result = 31 * result + (polymorphic ? 1 : 0); + result = 31 * result + settings.hashCode(); return result; } } diff --git a/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java b/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java index b76378a8ac..6cc97417da 100644 --- a/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java +++ b/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java @@ -40,7 +40,11 @@ public abstract class AbstractProperty implements Property { protected boolean notNull = false; protected boolean hidden = false; protected boolean external = false; - // Compression policy for EXTERNAL property values: "none" | "auto" | "lz4". Null means "none". + // Compression policy for EXTERNAL property values: "none" | "auto" | "lz4". + // STORAGE CONVENTION: null means "none" (the default), so toJSON omits the key. Read access MUST go through + // getCompression(), which materialises null as the literal string "none". LocalProperty.setCompression + // normalises "none" / null / "" all to null on write. Direct field reads from outside this class would see + // null when the user wrote "none"; always use the getter. protected String compression = null; protected String max = null; protected String min = null; diff --git a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java index 9d994ce8df..7805e0602f 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java @@ -68,6 +68,10 @@ public class LocalDocumentType implements DocumentType { // Map: primary bucket id -> external bucket id. Populated lazily when the first EXTERNAL property is // set on the type, and persisted in schema.json under the per-type "externalBuckets" key. protected final Map externalBucketIdByPrimaryBucketId = new ConcurrentHashMap<>(); + // Cached count of OWN properties (not inherited) currently flagged EXTERNAL. Avoids O(N) scans of + // getPolymorphicProperties() on hot paths (cascadeDeleteExternalValues, addBucketInternal, addSuperType). + // Maintained by LocalProperty.setExternal, dropProperty, and the schema-load path. + final java.util.concurrent.atomic.AtomicInteger ownExternalPropertyCount = new java.util.concurrent.atomic.AtomicInteger(0); public LocalDocumentType(final LocalSchema schema, final String name) { this.schema = schema; @@ -508,7 +512,11 @@ public Property dropProperty(final String propertyName) { } return recordFileChanges(() -> { - return properties.remove(propertyName); + final Property removed = properties.remove(propertyName); + // Keep the EXTERNAL counter consistent so hasExternalProperties() stays O(1). + if (removed != null && removed.isExternal()) + ownExternalPropertyCount.decrementAndGet(); + return removed; }); } @@ -973,10 +981,12 @@ protected void addBucketInternal(final Bucket bucket) { ensureExternalBucketFor((LocalBucket) bucket); } - /** Polymorphic: counts inherited EXTERNAL properties too. */ + /** Polymorphic: counts inherited EXTERNAL properties too. O(1) on own count + O(depth) on supertype walk. */ public boolean hasExternalProperties() { - for (final Property p : getPolymorphicProperties()) - if (p.isExternal()) + if (ownExternalPropertyCount.get() > 0) + return true; + for (final LocalDocumentType st : superTypes) + if (st.hasExternalProperties()) return true; return false; } @@ -998,31 +1008,57 @@ public void ensureExternalBucketsRecursive() { } private void ensureExternalBucketFor(final LocalBucket primary) { - if (externalBucketIdByPrimaryBucketId.containsKey(primary.getFileId())) - return; - final String extName = primary.getName() + "_ext"; - final LocalBucket external; - if (schema.bucketMap.containsKey(extName)) { - external = schema.bucketMap.get(extName); - // Refuse to adopt a bucket that is already registered as the primary bucket of some user type. A bucket - // freshly loaded from disk shows purpose=PRIMARY (transient field), so we cannot rely on purpose alone; - // bucketId2TypeMap is the authoritative source of "this bucket is a user type's primary bucket". - if (schema.getTypeByBucketId(external.getFileId()) != null - && external.getPurpose() != LocalBucket.Purpose.EXTERNAL_PROPERTY) - throw new SchemaException( - "Cannot adopt bucket '" + extName + "' as the external-property bucket for type '" + name - + "': it is already a primary bucket of another user type. Rename the conflicting bucket and retry."); - } else { - // External buckets get larger pages (256KB vs 64KB primary), a smaller slot table (128 vs 2048: file-format - // version EXTERNAL_BUCKET_VERSION), and optional placement on cheaper-storage tier via - // resolveExternalBucketPath() which returns /. - final int pageSize = schema.getDatabase().getConfiguration() - .getValueAsInteger(GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_DEFAULT_PAGE_SIZE); - final String overridePath = ((LocalDatabase) schema.getDatabase()).resolveExternalBucketPath(); - external = schema.createBucket(extName, pageSize, overridePath, LocalBucket.EXTERNAL_BUCKET_VERSION); + // Atomic check-and-create: two threads racing through ensureExternalBuckets()/addBucketInternal() must not + // both attempt to allocate the paired _ext bucket and trip schema.createBucket's "already exists" guard. + externalBucketIdByPrimaryBucketId.computeIfAbsent(primary.getFileId(), pid -> { + final String extName = primary.getName() + "_ext"; + final LocalBucket external; + if (schema.bucketMap.containsKey(extName)) { + external = schema.bucketMap.get(extName); + // Refuse to adopt a bucket that is already registered as the primary bucket of some user type. A bucket + // freshly loaded from disk shows purpose=PRIMARY (transient field), so we cannot rely on purpose alone; + // bucketId2TypeMap is the authoritative source of "this bucket is a user type's primary bucket". + if (schema.getTypeByBucketId(external.getFileId()) != null + && external.getPurpose() != LocalBucket.Purpose.EXTERNAL_PROPERTY) + throw new SchemaException( + "Cannot adopt bucket '" + extName + "' as the external-property bucket for type '" + name + + "': it is already a primary bucket of another user type. Rename the conflicting bucket and retry."); + } else { + // External buckets get larger pages (256KB vs 64KB primary), a smaller slot table (256 vs 2048: file-format + // version EXTERNAL_BUCKET_VERSION), and optional placement on cheaper-storage tier via + // resolveExternalBucketPath() which returns /. + final int pageSize = schema.getDatabase().getConfiguration() + .getValueAsInteger(GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_DEFAULT_PAGE_SIZE); + final String overridePath = ((LocalDatabase) schema.getDatabase()).resolveExternalBucketPath(); + external = schema.createBucket(extName, pageSize, overridePath, LocalBucket.EXTERNAL_BUCKET_VERSION); + } + external.setPurpose(LocalBucket.Purpose.EXTERNAL_PROPERTY); + return external.getFileId(); + }); + } + + /** + * Drops paired external-property buckets that no longer back any record (typical case: a REBUILD TYPE that + * moved every value back inline because the EXTERNAL flag was just toggled off). Skips buckets that still + * hold records so we never lose data; those point at persistent corruption and need investigation. Caller + * is expected to verify {@code !hasExternalProperties()} before calling. + */ + public void reclaimEmptyExternalBuckets() { + final List toDrop = new ArrayList<>(); + for (final Map.Entry entry : externalBucketIdByPrimaryBucketId.entrySet()) { + final LocalBucket extBucket = schema.getBucketById(entry.getValue(), false); + if (extBucket == null || extBucket.count() == 0L) + toDrop.add(entry.getKey()); } - external.setPurpose(LocalBucket.Purpose.EXTERNAL_PROPERTY); - externalBucketIdByPrimaryBucketId.put(primary.getFileId(), external.getFileId()); + for (final Integer primaryBucketId : toDrop) { + final Integer extBucketId = externalBucketIdByPrimaryBucketId.remove(primaryBucketId); + if (extBucketId == null) + continue; + final LocalBucket extBucket = schema.getBucketById(extBucketId, false); + if (extBucket != null) + schema.dropBucket(extBucket.getName()); + } + schema.saveConfiguration(); } /** Re-applies EXTERNAL_PROPERTY purpose (transient on LocalBucket) and rebuilds the map from JSON at load time. */ diff --git a/engine/src/main/java/com/arcadedb/schema/LocalProperty.java b/engine/src/main/java/com/arcadedb/schema/LocalProperty.java index ac764495ea..71ec130c29 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalProperty.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalProperty.java @@ -127,8 +127,13 @@ public Property setExternal(final boolean external) { final boolean changed = !Objects.equals(this.external, external); if (changed) { this.external = external; - if (external) - ((LocalDocumentType) owner).ensureExternalBucketsRecursive(); + final LocalDocumentType localOwner = (LocalDocumentType) owner; + if (external) { + localOwner.ownExternalPropertyCount.incrementAndGet(); + localOwner.ensureExternalBucketsRecursive(); + } else { + localOwner.ownExternalPropertyCount.decrementAndGet(); + } owner.getSchema().getEmbedded().saveConfiguration(); } return this; diff --git a/engine/src/main/java/com/arcadedb/schema/LocalSchema.java b/engine/src/main/java/com/arcadedb/schema/LocalSchema.java index 01b1cc85e5..bbd9e50d41 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalSchema.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalSchema.java @@ -382,7 +382,7 @@ public LocalBucket createBucket(final String bucketName, final int pageSize, fin /** * Full overload: creates a bucket with an explicit file-format version. Paired external-property buckets pass - * {@link LocalBucket#EXTERNAL_BUCKET_VERSION} so they get the smaller (128-slot) page-slot table appropriate for + * {@link LocalBucket#EXTERNAL_BUCKET_VERSION} so they get the smaller (256-slot) page-slot table appropriate for * heavy payloads; everything else uses {@link LocalBucket#CURRENT_VERSION}. */ public LocalBucket createBucket(final String bucketName, final int pageSize, final String parentDirectory, final int version) { diff --git a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java index f96d7957f0..aa36d4a6bb 100644 --- a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java +++ b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java @@ -118,7 +118,6 @@ public Binary serializeDocument(final DatabaseInternal database, final Document header.position(Binary.BYTE_SERIALIZED_SIZE); serializeProperties = false; } - if (serializeProperties) return serializeProperties(database, document, header, context.getTemporaryBuffer2()); @@ -1056,8 +1055,8 @@ public ExternalWriteResult writeExternalPropertyValue(final DatabaseInternal dat rid = externalBucket.createRecord(rec, true); database.getTransaction().updateBucketRecordDelta(externalBucket.getFileId(), +1); } else { + // Identity already passed to the constructor (BaseRecord stores it). No need to call setIdentity again. final ExternalValueRecord rec = new ExternalValueRecord(database, existingExternalRid, blob); - rec.setIdentity(existingExternalRid); externalBucket.updateRecord(rec, true); rid = existingExternalRid; } @@ -1065,39 +1064,6 @@ public ExternalWriteResult writeExternalPropertyValue(final DatabaseInternal dat return new ExternalWriteResult(typeByte, rid); } - /** Insert or in-place update of an EXTERNAL property's value blob. Format: [RECORD_TYPE][type][value bytes]. */ - public RID writeExternalValue(final DatabaseInternal database, final int externalBucketId, final RID existingExternalRid, - final byte type, final Object value) { - // Refuse to update an existing external record at a different bucket than the type's currently-paired one. - // This would mean the schema's external bucket mapping has shifted under us (e.g. partial migration) and - // overwriting blindly could corrupt unrelated data. - if (existingExternalRid != null && existingExternalRid.getBucketId() != externalBucketId) - throw new SerializationException( - "Existing external RID " + existingExternalRid + " does not match the paired external bucket id " - + externalBucketId + " for this record. The schema's external bucket mapping is inconsistent."); - - final Binary blob = new Binary(); - blob.putByte(ExternalValueRecord.RECORD_TYPE); - blob.putByte(type); - serializeValue(database, blob, type, value); - blob.flip(); - - final LocalBucket externalBucket = database.getSchema().getEmbedded().getBucketById(externalBucketId); - if (existingExternalRid == null) { - final ExternalValueRecord rec = new ExternalValueRecord(database, null, blob); - final RID newRid = externalBucket.createRecord(rec, true); - // Mirror LocalDatabase.createRecord's accounting: keep the bucket's record-count cache consistent across the - // transaction, since this path goes through the bucket directly and bypasses LocalDatabase. - database.getTransaction().updateBucketRecordDelta(externalBucket.getFileId(), +1); - return newRid; - } - - final ExternalValueRecord rec = new ExternalValueRecord(database, existingExternalRid, blob); - rec.setIdentity(existingExternalRid); - externalBucket.updateRecord(rec, true); - return existingExternalRid; - } - public Object readExternalValue(final DatabaseInternal database, final int externalBucketId, final long position, final EmbeddedModifier embeddedModifier) { return readExternalValue(database, externalBucketId, position, embeddedModifier, false); diff --git a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java index 3e3727d3b4..1fe6997108 100644 --- a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java +++ b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java @@ -397,14 +397,13 @@ void rebuildTypeReversesExternalToInlineAndCleansOrphans() { final var primary = type.getBuckets(false).getFirst(); final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + final String externalBucketName = external.getName(); assertThat(external.count()).isEqualTo((long) n); - // Flip OFF and rebuild. + // Flip OFF and rebuild. REBUILD manages its own transaction; wrapping it in a caller transaction would + // defer the deferred-update flush past the in-statement reclaim check. type.getProperty("blob").setExternal(false); - database.transaction(() -> database.command("sql", "REBUILD TYPE Doc")); - - // After rebuild every external record should have been deleted (orphan cleanup). - assertThat(external.count()).isEqualTo(0L); + database.command("sql", "REBUILD TYPE Doc"); // Values must still be readable inline. final ResultSet rs = database.query("sql", "SELECT blob FROM Doc"); @@ -414,6 +413,16 @@ void rebuildTypeReversesExternalToInlineAndCleansOrphans() { counted++; } assertThat(counted).isEqualTo(n); + + // Bucket reclaim: the now-empty paired external bucket is dropped (no accumulation across toggle cycles) + // and the type's external-bucket map is cleared so schema.json no longer carries an externalBuckets key. + assertThat(((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId())) + .as("paired external bucket id should be cleared from the type after REBUILD") + .isNull(); + assertThat(((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId, false)) + .as("paired external bucket file should be dropped").isNull(); + assertThat(database.getSchema().existsBucket(externalBucketName)) + .as("schema should no longer expose the dropped external bucket").isFalse(); } @Test @@ -508,13 +517,19 @@ void externalBucketUsesLargerDefaultPageSize() { @Test void externalBucketPathOverridePlacesFileOnSecondaryDirectory() throws java.io.IOException { - // Use a tier directory outside the database path to simulate cheaper-storage placement. + // Use a tier directory outside the database path to simulate cheaper-storage placement. We set the override + // on the FACTORY's ContextConfiguration (not the global one) so the reopened database inherits it without + // mutating JVM-wide state that could leak into concurrent tests. final java.nio.file.Path overrideDir = java.nio.file.Files.createTempDirectory("arcadedb-ext-tier-"); - final Object previous = com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH.getValue(); try { - // Apply the override on the open database. ensureExternalBucketFor reads it lazily when the first paired - // bucket is allocated, so the new path takes effect immediately for the type we are about to create. - database.getConfiguration().setValue(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH, overrideDir.toString()); + System.out.println("[DEBUG-TEST] factory.cfg=" + System.identityHashCode(factory.getContextConfiguration()) + + " db.cfg=" + System.identityHashCode(database.getConfiguration())); + factory.getContextConfiguration().setValue(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH, + overrideDir.toString()); + database.getConfiguration().setValue(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH, + overrideDir.toString()); + System.out.println("[DEBUG-TEST] after setValue, factory.cfg.val=" + + factory.getContextConfiguration().getValueAsString(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH)); final DocumentType type = database.getSchema().createDocumentType("Doc"); type.createProperty("blob", Type.STRING).setExternal(true); @@ -542,17 +557,19 @@ void externalBucketPathOverridePlacesFileOnSecondaryDirectory() throws java.io.I final java.io.File[] tieredFiles = tieredDbDir.listFiles((dir, name) -> name.startsWith(external.getName() + ".")); assertThat(tieredFiles).as("external bucket should be in //").isNotNull().isNotEmpty(); - // Reopen: FileManager must rediscover the tiered file via the secondary scan path so the record stays readable. - // The override is applied at open() via the global config, so set it there too before reopening. - com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH.setValue(overrideDir.toString()); + // Reopen: the factory's per-instance ContextConfiguration carries the override into the new + // LocalDatabase, so FileManager rediscovers the tiered file via the secondary scan path. No global + // config mutation needed. + System.out.println("[DEBUG-TEST] before close, factory.cfg.val=" + + factory.getContextConfiguration().getValueAsString(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH)); database.close(); + System.out.println("[DEBUG-TEST] after close, factory.cfg.val=" + + factory.getContextConfiguration().getValueAsString(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH)); database = factory.open(); - database.getConfiguration().setValue(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH, overrideDir.toString()); final var loaded = database.lookupByRID(saved[0], true).asDocument(); assertThat(loaded.getString("blob")).isEqualTo("tiered-payload"); } finally { - com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH.setValue(previous != null ? previous.toString() : ""); com.arcadedb.utility.FileUtils.deleteRecursively(overrideDir.toFile()); } } @@ -570,10 +587,11 @@ void checkDatabaseDetectsAndFixesOrphanedExternalRecords() { final long extCountBefore = externalBucket.count(); // Inject an orphan: write a value blob directly to the external bucket, bypassing the property write path - // so no primary record references it. + // so no primary record references it. compression="none" keeps the blob raw so cleanup just sees a normal + // unreferenced record. database.transaction(() -> ((com.arcadedb.database.DatabaseInternal) database).getSerializer() - .writeExternalValue((com.arcadedb.database.DatabaseInternal) database, extBucketId, null, - com.arcadedb.serializer.BinaryTypes.TYPE_STRING, "orphan-payload")); + .writeExternalPropertyValue((com.arcadedb.database.DatabaseInternal) database, extBucketId, null, + com.arcadedb.serializer.BinaryTypes.TYPE_STRING, "orphan-payload", "none")); assertThat(externalBucket.count()).isEqualTo(extCountBefore + 1); diff --git a/studio/src/main/resources/static/js/studio-database.js b/studio/src/main/resources/static/js/studio-database.js index 4bbb314e72..9766eaa4ec 100644 --- a/studio/src/main/resources/static/js/studio-database.js +++ b/studio/src/main/resources/static/js/studio-database.js @@ -1068,6 +1068,16 @@ function createProperty(typeName) { html += "
"; html += "
"; html += ""; + // Compression policy for EXTERNAL properties. Hidden by default; revealed when External is ticked. + html += ""; globalPrompt("Add Property to " + escapeHtml(typeName), html, "Create", function () { let name = $("#inputCreatePropName").val().trim(); @@ -1087,6 +1097,7 @@ function createProperty(typeName) { let hidden = $("#inputCreatePropHidden").prop("checked"); let readOnly = $("#inputCreatePropReadOnly").prop("checked"); let external = $("#inputCreatePropExternal").prop("checked"); + let compression = $("#inputCreatePropCompression").val(); let ifNotExists = $("#inputCreatePropIfNotExists").prop("checked"); let command = "CREATE PROPERTY `" + typeName + "`.`" + name + "`"; @@ -1100,6 +1111,7 @@ function createProperty(typeName) { if (hidden) constraints.push("HIDDEN true"); if (readOnly) constraints.push("READONLY true"); if (external) constraints.push("EXTERNAL true"); + if (external && compression && compression !== "none") constraints.push("COMPRESSION '" + compression + "'"); if (defaultVal != "") constraints.push("DEFAULT " + defaultVal); if (min != "") constraints.push("MIN " + min); if (max != "") constraints.push("MAX " + max); @@ -1131,8 +1143,17 @@ function createProperty(typeName) { else $("#createPropRegexpRow").hide(); } + function updateCompressionVisibility() { + // Compression policy is meaningful only for EXTERNAL properties. + if ($("#inputCreatePropExternal").prop("checked")) + $("#createPropCompressionRow").show(); + else + $("#createPropCompressionRow").hide(); + } $("#inputCreatePropType").on("change", updatePropVisibility); + $("#inputCreatePropExternal").on("change", updateCompressionVisibility); updatePropVisibility(); + updateCompressionVisibility(); }, 100); } @@ -3334,7 +3355,13 @@ function renderProperties(row, results) { let tooltip = pairs.length > 0 ? "Value stored in paired external bucket(s): " + pairs.join(", ") : "Value stored in a paired external bucket"; - panelHtml += " External"; + // Compression policy lives on the property; only emitted by schema:types when not 'none'. + let cmpLabel = ""; + if (property.compression) { + tooltip += ". Compression: " + property.compression; + cmpLabel = " (" + escapeHtml(property.compression) + ")"; + } + panelHtml += " External" + cmpLabel + ""; } else { panelHtml += "Inline"; } From 25dd79009f0ec87abc94350607182c0148b21660 Mon Sep 17 00:00:00 2001 From: Luca Garulli Date: Wed, 29 Apr 2026 15:00:02 -0400 Subject: [PATCH 07/12] Implemented claude review --- .../arcadedb/query/sql/grammar/SQLParser.g4 | 2 +- .../com/arcadedb/database/BaseDocument.java | 3 +- .../arcadedb/database/DocumentInternal.java | 38 ++++++++++++++++ .../database/ExternalValueRecord.java | 9 +++- .../com/arcadedb/database/LocalDatabase.java | 6 +-- .../com/arcadedb/engine/DatabaseChecker.java | 24 ++++++++-- .../java/com/arcadedb/engine/LocalBucket.java | 2 +- .../query/sql/antlr/SQLASTBuilder.java | 13 +++--- .../com/arcadedb/schema/LocalProperty.java | 14 +++++- .../arcadedb/serializer/BinarySerializer.java | 10 ++++- .../arcadedb/schema/ExternalPropertyTest.java | 35 ++++++--------- .../BinarySerializerTestHelper.java | 44 +++++++++++++++++++ ...RaftLeaderCrashWithExternalPropertyIT.java | 2 +- 13 files changed, 155 insertions(+), 47 deletions(-) create mode 100644 engine/src/main/java/com/arcadedb/database/DocumentInternal.java create mode 100644 engine/src/test/java/com/arcadedb/serializer/BinarySerializerTestHelper.java diff --git a/engine/src/main/antlr4/com/arcadedb/query/sql/grammar/SQLParser.g4 b/engine/src/main/antlr4/com/arcadedb/query/sql/grammar/SQLParser.g4 index f7e11940de..aaa2010c07 100644 --- a/engine/src/main/antlr4/com/arcadedb/query/sql/grammar/SQLParser.g4 +++ b/engine/src/main/antlr4/com/arcadedb/query/sql/grammar/SQLParser.g4 @@ -878,7 +878,7 @@ rebuildIndexStatement * Syntax: REBUILD TYPE typeName [POLYMORPHIC] */ rebuildTypeBody - : identifier POLYMORPHIC? (WITH identifier EQ expression (COMMA identifier EQ expression)*)? + : typeName=identifier POLYMORPHIC? (WITH settingKey+=identifier EQ settingValue+=expression (COMMA settingKey+=identifier EQ settingValue+=expression)*)? ; // ============================================================================ diff --git a/engine/src/main/java/com/arcadedb/database/BaseDocument.java b/engine/src/main/java/com/arcadedb/database/BaseDocument.java index 7ab77d51eb..53b3008f30 100644 --- a/engine/src/main/java/com/arcadedb/database/BaseDocument.java +++ b/engine/src/main/java/com/arcadedb/database/BaseDocument.java @@ -39,7 +39,7 @@ import java.util.List; import java.util.Map; -public abstract class BaseDocument extends BaseRecord implements Document, Serializable, Externalizable { +public abstract class BaseDocument extends BaseRecord implements Document, DocumentInternal, Serializable, Externalizable { protected final DocumentType type; protected int propertiesStartingPosition = 1; @@ -157,6 +157,7 @@ public DocumentType getType() { return type; } + @Override public int getPropertiesStartingPosition() { return propertiesStartingPosition; } diff --git a/engine/src/main/java/com/arcadedb/database/DocumentInternal.java b/engine/src/main/java/com/arcadedb/database/DocumentInternal.java new file mode 100644 index 0000000000..fc53004db8 --- /dev/null +++ b/engine/src/main/java/com/arcadedb/database/DocumentInternal.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.database; + +import com.arcadedb.utility.ExcludeFromJacocoGeneratedReport; + +/** + * Internal contract for {@link BaseDocument} that exposes serialization-format details to engine modules + * (notably {@code com.arcadedb.serializer.BinarySerializer}) without leaking those details into the public + * {@link Document} API. Code that consumes ArcadeDB as a library should never cast to this interface; cross-version + * stability is not guaranteed. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +@ExcludeFromJacocoGeneratedReport +public interface DocumentInternal { + /** + * Offset (in bytes) where the property header begins inside the record's binary buffer. Required by the + * serializer to scan for {@code TYPE_EXTERNAL} pointers without re-parsing the bucket-level record framing. + */ + int getPropertiesStartingPosition(); +} diff --git a/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java b/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java index 540172663e..35a3d326e5 100644 --- a/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java +++ b/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java @@ -53,7 +53,12 @@ public Binary getContent() { @Override public JSONObject toJSON(final boolean includeMetadata) { - // No user-visible JSON form - generic record dumpers should never reach an EXTERNAL value blob directly. - return new JSONObject(); + // ExternalValueRecord is engine-internal infrastructure. It's only reachable through a primary record's + // TYPE_EXTERNAL pointer, and every public read path (Document.toJSON, REST handlers, Studio) walks through + // BinarySerializer.deserializeProperty which materialises the value inline. If a generic record dumper hits + // this method directly, it's a bug: failing loud surfaces it instead of returning silent empty data. + throw new UnsupportedOperationException( + "ExternalValueRecord is an internal payload of an EXTERNAL property and has no JSON representation; " + + "read the value through its owning Document instead of dereferencing the external RID directly"); } } diff --git a/engine/src/main/java/com/arcadedb/database/LocalDatabase.java b/engine/src/main/java/com/arcadedb/database/LocalDatabase.java index bb3af6c0a2..298462102e 100644 --- a/engine/src/main/java/com/arcadedb/database/LocalDatabase.java +++ b/engine/src/main/java/com/arcadedb/database/LocalDatabase.java @@ -2068,11 +2068,7 @@ private void openInternal() { DatabaseContext.INSTANCE.init(this); setLockingEnabled(configuration.getValueAsBoolean(GlobalConfiguration.BACKUP_ENABLED)); - final String resolved = resolveExternalBucketPath(); - System.out.println("[DEBUG-FM] open db=" + name + " externalBucketPath=" + resolved - + " configValue=" + configuration.getValueAsString(GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH) - + " configHash=" + System.identityHashCode(configuration)); - fileManager = new FileManager(databasePath, mode, SUPPORTED_FILE_EXT, resolved); + fileManager = new FileManager(databasePath, mode, SUPPORTED_FILE_EXT, resolveExternalBucketPath()); transactionManager = new TransactionManager(wrappedDatabaseInstance); open = true; diff --git a/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java b/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java index 635c7937ac..ffbf86381d 100644 --- a/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java +++ b/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java @@ -316,18 +316,36 @@ private void checkExternalProperties() { final boolean startedNewTx = !database.isTransactionActive(); if (startedNewTx) database.begin(); + // All-or-nothing per bucket: any failure rolls back the whole batch (or, if a caller-supplied tx is + // already open, throws so the caller can decide). We never commit a partially-cleaned bucket. + boolean anyFailure = false; + long localFixed = 0L; for (final RID orphan : orphans) { try { extBucket.deleteRecord(orphan); // Mirror the accounting in LocalDatabase.cascadeDeleteExternalValues so count() stays consistent. database.getTransaction().updateBucketRecordDelta(extBucket.getFileId(), -1); - fixedCount++; + localFixed++; } catch (final Exception e) { warnings.add("could not delete orphan external record " + orphan + ": " + e.getMessage()); + anyFailure = true; + break; } } - if (startedNewTx) - database.commit(); + if (startedNewTx) { + if (anyFailure) + database.rollback(); + else { + database.commit(); + fixedCount += localFixed; + } + } else if (anyFailure) { + throw new com.arcadedb.exception.DatabaseOperationException( + "Failed to delete orphan external records in bucket '" + extBucket.getName() + + "'; aborting CHECK DATABASE FIX so the caller's transaction is not silently committed in a partial state"); + } else { + fixedCount += localFixed; + } } } diff --git a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java index 2ca2cae62f..c25024f844 100644 --- a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java +++ b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java @@ -178,7 +178,7 @@ public int getMaxRecordsInPage() { return maxRecordsInPage; } - /** Slot-table sizing for the bucket file format version. v0=2048 (legacy), v1=128 (paired external buckets). */ + /** Slot-table sizing for the bucket file format version. v0=2048 (legacy), v1=256 (paired external buckets). */ private static int maxRecordsInPageForVersion(final int version) { return version >= EXTERNAL_BUCKET_VERSION ? DEF_MAX_RECORDS_IN_PAGE_V1 : DEF_MAX_RECORDS_IN_PAGE; } diff --git a/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java b/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java index 944e692be0..ad3229ccf1 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java +++ b/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java @@ -5735,15 +5735,14 @@ public RebuildIndexStatement visitRebuildIndexStatement(final SQLParser.RebuildI public RebuildTypeStatement visitRebuildTypeStmt(final SQLParser.RebuildTypeStmtContext ctx) { final RebuildTypeStatement stmt = new RebuildTypeStatement(-1); final SQLParser.RebuildTypeBodyContext bodyCtx = ctx.rebuildTypeBody(); - // The type name is the first identifier; any further identifiers are WITH-setting keys. - final List ids = bodyCtx.identifier(); - stmt.typeName = (Identifier) visit(ids.get(0)); + // Grammar uses explicit labels (typeName=, settingKey+=, settingValue+=) so a future grammar tweak that + // introduces another identifier slot won't silently shift index-based bindings here. + stmt.typeName = (Identifier) visit(bodyCtx.typeName); stmt.polymorphic = bodyCtx.POLYMORPHIC() != null; if (bodyCtx.WITH() != null) { - final List values = bodyCtx.expression(); - for (int i = 0; i < values.size(); i++) { - final Expression key = new Expression((Identifier) visit(ids.get(i + 1))); - final Expression value = (Expression) visit(values.get(i)); + for (int i = 0; i < bodyCtx.settingValue.size(); i++) { + final Expression key = new Expression((Identifier) visit(bodyCtx.settingKey.get(i))); + final Expression value = (Expression) visit(bodyCtx.settingValue.get(i)); stmt.settings.put(key, value); } } diff --git a/engine/src/main/java/com/arcadedb/schema/LocalProperty.java b/engine/src/main/java/com/arcadedb/schema/LocalProperty.java index 71ec130c29..d9f7067568 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalProperty.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalProperty.java @@ -126,12 +126,22 @@ public Property setHidden(final boolean hidden) { public Property setExternal(final boolean external) { final boolean changed = !Objects.equals(this.external, external); if (changed) { - this.external = external; final LocalDocumentType localOwner = (LocalDocumentType) owner; if (external) { - localOwner.ownExternalPropertyCount.incrementAndGet(); + // ORDERING IS LOAD-BEARING: paired buckets must exist BEFORE any reader can observe + // ownExternalPropertyCount > 0. hasExternalProperties() short-circuits on the counter, and the + // serializer's write path uses that to decide whether to route a value through the external bucket. + // If we incremented first and another thread saw count > 0 before ensureExternalBucketsRecursive() + // ran, it could try to write into a bucket that doesn't exist yet. Schema mutations are typically + // serialised by the schema lock, but this ordering keeps the property setter safe even without it. localOwner.ensureExternalBucketsRecursive(); + this.external = external; + localOwner.ownExternalPropertyCount.incrementAndGet(); } else { + // Flip the flag first so concurrent serialisations stop writing externally; the counter is the gate + // hasExternalProperties() consults, so decrement last to keep the "count > 0 implies bucket exists" + // invariant directional. + this.external = external; localOwner.ownExternalPropertyCount.decrementAndGet(); } owner.getSchema().getEmbedded().saveConfiguration(); diff --git a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java index aa36d4a6bb..ce958f95c5 100644 --- a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java +++ b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java @@ -49,6 +49,7 @@ import com.arcadedb.query.sql.executor.Result; import com.arcadedb.function.sql.geo.GeoUtils; import com.arcadedb.database.BaseDocument; +import com.arcadedb.database.DocumentInternal; import com.arcadedb.schema.DocumentType; import com.arcadedb.schema.LocalDocumentType; import com.arcadedb.schema.Property; @@ -1002,7 +1003,12 @@ private static com.arcadedb.compression.LZ4Compression lz4() { * In "auto" mode compression is kept only when it saves more than 10% of bytes; otherwise the record is written * raw. The decision is per-record so a single property can mix compressed and uncompressed records freely. */ - public ExternalWriteResult writeExternalPropertyValue(final DatabaseInternal database, final int externalBucketId, + /** + * Internal serializer plumbing. Made package-private; the test-only entry point + * {@code BinarySerializerTestHelper#injectOrphanExternalRecord} forwards into this method without exposing + * it to library consumers. + */ + ExternalWriteResult writeExternalPropertyValue(final DatabaseInternal database, final int externalBucketId, final RID existingExternalRid, final byte valueType, final Object value, final String compressionPolicy) { if (existingExternalRid != null && existingExternalRid.getBucketId() != externalBucketId) throw new SerializationException( @@ -1105,7 +1111,7 @@ public Map findExistingExternalRids(final Database database, final try { final Binary buf = oldBuffer.copyOfContent(); - buf.position(((BaseDocument) record).getPropertiesStartingPosition()); + buf.position(((DocumentInternal) record).getPropertiesStartingPosition()); final int headerEndOffset = buf.getInt(); final int properties = (int) buf.getUnsignedNumber(); diff --git a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java index 1fe6997108..ce8f44904d 100644 --- a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java +++ b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java @@ -517,19 +517,13 @@ void externalBucketUsesLargerDefaultPageSize() { @Test void externalBucketPathOverridePlacesFileOnSecondaryDirectory() throws java.io.IOException { - // Use a tier directory outside the database path to simulate cheaper-storage placement. We set the override - // on the FACTORY's ContextConfiguration (not the global one) so the reopened database inherits it without - // mutating JVM-wide state that could leak into concurrent tests. + // Persist the override on the database itself via ALTER DATABASE. This writes the value into the + // database's configuration.json so it survives close+reopen without polluting the JVM-wide + // GlobalConfiguration (which would leak into concurrent tests). final java.nio.file.Path overrideDir = java.nio.file.Files.createTempDirectory("arcadedb-ext-tier-"); try { - System.out.println("[DEBUG-TEST] factory.cfg=" + System.identityHashCode(factory.getContextConfiguration()) - + " db.cfg=" + System.identityHashCode(database.getConfiguration())); - factory.getContextConfiguration().setValue(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH, - overrideDir.toString()); - database.getConfiguration().setValue(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH, - overrideDir.toString()); - System.out.println("[DEBUG-TEST] after setValue, factory.cfg.val=" - + factory.getContextConfiguration().getValueAsString(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH)); + database.command("sql", + "alter database `arcadedb.externalPropertyBucketPath` '" + overrideDir.toString() + "'"); final DocumentType type = database.getSchema().createDocumentType("Doc"); type.createProperty("blob", Type.STRING).setExternal(true); @@ -557,14 +551,9 @@ void externalBucketPathOverridePlacesFileOnSecondaryDirectory() throws java.io.I final java.io.File[] tieredFiles = tieredDbDir.listFiles((dir, name) -> name.startsWith(external.getName() + ".")); assertThat(tieredFiles).as("external bucket should be in //").isNotNull().isNotEmpty(); - // Reopen: the factory's per-instance ContextConfiguration carries the override into the new - // LocalDatabase, so FileManager rediscovers the tiered file via the secondary scan path. No global - // config mutation needed. - System.out.println("[DEBUG-TEST] before close, factory.cfg.val=" - + factory.getContextConfiguration().getValueAsString(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH)); + // Reopen: LocalDatabase.open() reloads configuration.json which contains our ALTER, so FileManager + // rediscovers the tiered file via the secondary scan path. database.close(); - System.out.println("[DEBUG-TEST] after close, factory.cfg.val=" - + factory.getContextConfiguration().getValueAsString(com.arcadedb.GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH)); database = factory.open(); final var loaded = database.lookupByRID(saved[0], true).asDocument(); @@ -588,10 +577,12 @@ void checkDatabaseDetectsAndFixesOrphanedExternalRecords() { // Inject an orphan: write a value blob directly to the external bucket, bypassing the property write path // so no primary record references it. compression="none" keeps the blob raw so cleanup just sees a normal - // unreferenced record. - database.transaction(() -> ((com.arcadedb.database.DatabaseInternal) database).getSerializer() - .writeExternalPropertyValue((com.arcadedb.database.DatabaseInternal) database, extBucketId, null, - com.arcadedb.serializer.BinaryTypes.TYPE_STRING, "orphan-payload", "none")); + // unreferenced record. The serializer write path is package-private; the test reaches it via the + // BinarySerializerTestHelper which lives in the same package under src/test/java. + database.transaction(() -> com.arcadedb.serializer.BinarySerializerTestHelper.injectOrphanExternalRecord( + ((com.arcadedb.database.DatabaseInternal) database).getSerializer(), + (com.arcadedb.database.DatabaseInternal) database, extBucketId, + com.arcadedb.serializer.BinaryTypes.TYPE_STRING, "orphan-payload", "none")); assertThat(externalBucket.count()).isEqualTo(extCountBefore + 1); diff --git a/engine/src/test/java/com/arcadedb/serializer/BinarySerializerTestHelper.java b/engine/src/test/java/com/arcadedb/serializer/BinarySerializerTestHelper.java new file mode 100644 index 0000000000..72ba1c00f0 --- /dev/null +++ b/engine/src/test/java/com/arcadedb/serializer/BinarySerializerTestHelper.java @@ -0,0 +1,44 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.serializer; + +import com.arcadedb.database.DatabaseInternal; +import com.arcadedb.database.RID; + +/** + * Test-only entry point for serializer internals. Lives in {@code src/test/java} under the + * {@code com.arcadedb.serializer} package so it can reach the package-private write path + * {@link BinarySerializer#writeExternalPropertyValue} without that method having to be public on the + * production API surface. Use only from regression tests. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public final class BinarySerializerTestHelper { + private BinarySerializerTestHelper() { + } + + /** + * Writes a value blob directly into a paired external bucket, bypassing the property write path. Used by + * orphan-cleanup tests to plant a record that no primary record references. + */ + public static RID injectOrphanExternalRecord(final BinarySerializer serializer, final DatabaseInternal database, + final int externalBucketId, final byte valueType, final Object value, final String compression) { + return serializer.writeExternalPropertyValue(database, externalBucketId, null, valueType, value, compression).rid; + } +} diff --git a/ha-raft/src/test/java/com/arcadedb/server/ha/raft/RaftLeaderCrashWithExternalPropertyIT.java b/ha-raft/src/test/java/com/arcadedb/server/ha/raft/RaftLeaderCrashWithExternalPropertyIT.java index f4375beef9..9be41b2343 100644 --- a/ha-raft/src/test/java/com/arcadedb/server/ha/raft/RaftLeaderCrashWithExternalPropertyIT.java +++ b/ha-raft/src/test/java/com/arcadedb/server/ha/raft/RaftLeaderCrashWithExternalPropertyIT.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-present Arcade Data Ltd (info@arcadedata.com) + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 3ae5fddcbf541774aad6062aa53f9ee1e407c0e1 Mon Sep 17 00:00:00 2001 From: Luca Garulli Date: Wed, 29 Apr 2026 15:26:06 -0400 Subject: [PATCH 08/12] Refactoring of compression --- .../arcadedb/compression/LZ4Compression.java | 34 +++- .../com/arcadedb/schema/AbstractProperty.java | 2 +- .../com/arcadedb/schema/LocalProperty.java | 9 +- .../java/com/arcadedb/schema/Property.java | 12 +- .../arcadedb/serializer/BinarySerializer.java | 69 +++++--- .../com/arcadedb/serializer/BinaryTypes.java | 5 +- .../arcadedb/schema/ExternalPropertyTest.java | 164 +++++++++++++++++- .../resources/static/js/studio-database.js | 55 ++---- 8 files changed, 274 insertions(+), 76 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/compression/LZ4Compression.java b/engine/src/main/java/com/arcadedb/compression/LZ4Compression.java index 4eeeba1d82..161b13dd0a 100644 --- a/engine/src/main/java/com/arcadedb/compression/LZ4Compression.java +++ b/engine/src/main/java/com/arcadedb/compression/LZ4Compression.java @@ -26,16 +26,24 @@ import java.util.*; /** - * Compression implementation that uses the popular LZ4 algorithm. + * Compression implementation that uses the popular LZ4 algorithm. Two compressors are exposed: the default + * {@code fast} encoder (block size optimised for throughput) and an on-demand {@code max} (LZ4 HC) encoder + * that spends more CPU on a deeper match search to produce a smaller output. Both share the same LZ4 byte + * format and the same decoder, so {@link #decompress(byte[], int)} works regardless of which encoder produced + * the input. */ public class LZ4Compression implements Compression { private static final byte[] EMPTY_BYTES = new byte[0]; private static final Binary EMPTY_BINARY = new Binary(EMPTY_BYTES); + private final LZ4Factory factory; private final LZ4Compressor compressor; private final LZ4FastDecompressor decompressor; + // High-compression encoder is built on first use: it allocates ~256KB of internal state and is rarely needed + // on the read path, so we don't want to pay for it when only the fast encoder is in use. + private volatile LZ4Compressor maxCompressor; public LZ4Compression() { - final LZ4Factory factory = LZ4Factory.fastestInstance(); + this.factory = LZ4Factory.fastestInstance(); this.compressor = factory.fastCompressor(); this.decompressor = factory.fastDecompressor(); } @@ -50,6 +58,28 @@ public byte[] compress(final byte[] data) { return compressed; } + /** + * Maximum-compression encoder (LZ4 HC). Compresses 8-20x slower than {@link #compress(byte[])} but produces + * meaningfully smaller output (~10pp on text); decompression is the same speed. Use for write-once / + * read-many EXTERNAL payloads where bucket file size dominates. + */ + public byte[] compressMax(final byte[] data) { + LZ4Compressor c = maxCompressor; + if (c == null) { + synchronized (this) { + c = maxCompressor; + if (c == null) + c = maxCompressor = factory.highCompressor(); + } + } + final int maxCompressedLength = c.maxCompressedLength(data.length); + byte[] compressed = new byte[maxCompressedLength]; + final int compressedLength = c.compress(data, 0, data.length, compressed, 0, maxCompressedLength); + if (compressedLength != maxCompressedLength) + compressed = Arrays.copyOf(compressed, compressedLength); + return compressed; + } + @Override public Binary compress(final Binary data) { final int decompressedLength = data.size() - data.position(); diff --git a/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java b/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java index 6cc97417da..6898cafa00 100644 --- a/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java +++ b/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java @@ -40,7 +40,7 @@ public abstract class AbstractProperty implements Property { protected boolean notNull = false; protected boolean hidden = false; protected boolean external = false; - // Compression policy for EXTERNAL property values: "none" | "auto" | "lz4". + // Compression policy for EXTERNAL property values: "none" | "fast" | "max" | "auto" (legacy alias: "lz4" -> "fast"). // STORAGE CONVENTION: null means "none" (the default), so toJSON omits the key. Read access MUST go through // getCompression(), which materialises null as the literal string "none". LocalProperty.setCompression // normalises "none" / null / "" all to null on write. Direct field reads from outside this class would see diff --git a/engine/src/main/java/com/arcadedb/schema/LocalProperty.java b/engine/src/main/java/com/arcadedb/schema/LocalProperty.java index d9f7067568..aa68c81d98 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalProperty.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalProperty.java @@ -154,11 +154,16 @@ public Property setCompression(final String compression) { final String normalized; if (compression == null || compression.isEmpty() || "none".equalsIgnoreCase(compression)) normalized = null; - else if ("auto".equalsIgnoreCase(compression) || "lz4".equalsIgnoreCase(compression)) + else if ("fast".equalsIgnoreCase(compression) || "max".equalsIgnoreCase(compression) + || "auto".equalsIgnoreCase(compression)) normalized = compression.toLowerCase(Locale.ENGLISH); + else if ("lz4".equalsIgnoreCase(compression)) + // Backwards alias: "lz4" was the original name for the fast tier; map it to "fast" so existing schema + // configs keep working without touching the schema.json on disk. + normalized = "fast"; else throw new IllegalArgumentException( - "Unsupported compression '" + compression + "' (supported: none, auto, lz4)"); + "Unsupported compression '" + compression + "' (supported: none, fast, max, auto)"); if (!Objects.equals(this.compression, normalized)) { this.compression = normalized; owner.getSchema().getEmbedded().saveConfiguration(); diff --git a/engine/src/main/java/com/arcadedb/schema/Property.java b/engine/src/main/java/com/arcadedb/schema/Property.java index 6398d4d57d..4fe536af17 100644 --- a/engine/src/main/java/com/arcadedb/schema/Property.java +++ b/engine/src/main/java/com/arcadedb/schema/Property.java @@ -81,8 +81,16 @@ public interface Property { boolean isExternal(); /** - * Compression policy for an EXTERNAL property's value: "none" (default), "auto" (try LZ4, keep compressed only - * if it saves >10%), or an explicit algorithm like "lz4". Ignored for non-EXTERNAL properties. + * Compression policy for an EXTERNAL property's value: + *
    + *
  • {@code none} (default) - store raw.
    • + *
  • {@code fast} - LZ4 fast encoder. ~1.2-1.5x faster compress than Snappy on text, identical decompress + * speed regardless of tier. Best default when writes are frequent.
    • + *
  • {@code max} - LZ4 HC encoder. ~10pp smaller output than {@code fast}, 8-20x slower compress; + * decompression speed is the same as {@code fast}. Best for write-once / read-many payloads.
    • + *
  • {@code auto} - try {@code fast}; keep compressed only when it saves more than 10% of bytes.
    • + *
+ * The legacy alias {@code lz4} is accepted and stored as {@code fast}. Ignored for non-EXTERNAL properties. */ Property setCompression(String compression); diff --git a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java index ce958f95c5..79129ce300 100644 --- a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java +++ b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java @@ -274,11 +274,11 @@ else if (properties == 0) embeddedModifier != null ? new EmbeddedModifierProperty(embeddedModifier.getOwner(), propertyName) : null; final Object propertyValue; - if (type == BinaryTypes.TYPE_EXTERNAL || type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_LZ4) { + if (isExternalType(type)) { final int extBucketId = (int) buffer.getNumber(); final long extPosition = buffer.getNumber(); propertyValue = readExternalValue((DatabaseInternal) database, extBucketId, extPosition, propertyModifier, - type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_LZ4); + isExternalCompressedType(type)); } else { propertyValue = deserializeValue(database, buffer, type, propertyModifier); } @@ -355,11 +355,11 @@ else if (properties == 0) final EmbeddedModifierProperty propertyModifier = embeddedModifier != null ? new EmbeddedModifierProperty(embeddedModifier.getOwner(), fieldName) : null; - if (type == BinaryTypes.TYPE_EXTERNAL || type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_LZ4) { + if (isExternalType(type)) { final int extBucketId = (int) buffer.getNumber(); final long extPosition = buffer.getNumber(); return readExternalValue((DatabaseInternal) database, extBucketId, extPosition, propertyModifier, - type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_LZ4); + isExternalCompressedType(type)); } return deserializeValue(database, buffer, type, propertyModifier); @@ -996,17 +996,36 @@ private static com.arcadedb.compression.LZ4Compression lz4() { return local; } + /** True for any of TYPE_EXTERNAL, TYPE_EXTERNAL_COMPRESSED_FAST, TYPE_EXTERNAL_COMPRESSED_MAX. */ + private static boolean isExternalType(final byte type) { + return type == BinaryTypes.TYPE_EXTERNAL + || type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_FAST + || type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_MAX; + } + /** - * Serialises an EXTERNAL property value, optionally compressing per the property's policy - * ("none"|"auto"|"lz4"), writes the resulting blob to the paired external bucket, and returns the type byte - * the caller should put in the main record (TYPE_EXTERNAL or TYPE_EXTERNAL_COMPRESSED_LZ4). - * In "auto" mode compression is kept only when it saves more than 10% of bytes; otherwise the record is written - * raw. The decision is per-record so a single property can mix compressed and uncompressed records freely. + * True only for the compressed external types. LZ4 fast and LZ4 HC share the same byte format, so the read + * path needs only the compressed/raw distinction; both compressed types decode through the same + * {@code LZ4Compression.decompress} call. */ + private static boolean isExternalCompressedType(final byte type) { + return type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_FAST + || type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_MAX; + } + /** - * Internal serializer plumbing. Made package-private; the test-only entry point - * {@code BinarySerializerTestHelper#injectOrphanExternalRecord} forwards into this method without exposing - * it to library consumers. + * Serialises an EXTERNAL property value per the property's compression policy, writes the resulting blob to + * the paired external bucket, and returns the type byte the caller should put in the main record. + *

+ * Policy values: + *

    + *
  • {@code none} (or null/empty) - store raw, type byte = TYPE_EXTERNAL.
    • + *
  • {@code fast} - LZ4 fast encoder, type byte = TYPE_EXTERNAL_COMPRESSED_FAST.
    • + *
  • {@code max} - LZ4 HC encoder (slower compress, ~10pp smaller), type byte = TYPE_EXTERNAL_COMPRESSED_MAX.
    • + *
  • {@code auto} - try LZ4 fast; keep only when it saves more than 10% of bytes, otherwise store raw.
    • + *
+ * The decision is per-record, so a single property may mix compressed and uncompressed records freely. + * Made package-private; tests reach it through {@code BinarySerializerTestHelper}. */ ExternalWriteResult writeExternalPropertyValue(final DatabaseInternal database, final int externalBucketId, final RID existingExternalRid, final byte valueType, final Object value, final String compressionPolicy) { @@ -1021,20 +1040,22 @@ ExternalWriteResult writeExternalPropertyValue(final DatabaseInternal database, serializeValue(database, rawValueBytes, valueType, value); rawValueBytes.flip(); - final boolean tryLz4 = compressionPolicy != null - && ("auto".equalsIgnoreCase(compressionPolicy) || "lz4".equalsIgnoreCase(compressionPolicy)); + final boolean fastMode = "fast".equalsIgnoreCase(compressionPolicy) || "lz4".equalsIgnoreCase(compressionPolicy); + final boolean maxMode = "max".equalsIgnoreCase(compressionPolicy); final boolean autoMode = "auto".equalsIgnoreCase(compressionPolicy); + final boolean tryCompress = fastMode || maxMode || autoMode; byte typeByte = BinaryTypes.TYPE_EXTERNAL; byte[] compressedPayload = null; int uncompressedSize = 0; - if (tryLz4 && rawValueBytes.size() > 0) { + if (tryCompress && rawValueBytes.size() > 0) { final byte[] raw = rawValueBytes.toByteArray(); - final byte[] compressed = lz4().compress(raw); - // In auto mode skip compression unless it saves >10%. Outside auto mode (explicit "lz4") always keep it. + final byte[] compressed = maxMode ? lz4().compressMax(raw) : lz4().compress(raw); + // In auto mode skip compression unless it saves >10%. In fast/max mode always keep the compressed form + // (the user explicitly asked to compress, even if a particular record happens not to gain much). if (!autoMode || compressed.length < raw.length * 0.9) { - typeByte = BinaryTypes.TYPE_EXTERNAL_COMPRESSED_LZ4; + typeByte = maxMode ? BinaryTypes.TYPE_EXTERNAL_COMPRESSED_MAX : BinaryTypes.TYPE_EXTERNAL_COMPRESSED_FAST; compressedPayload = compressed; uncompressedSize = raw.length; } @@ -1044,7 +1065,7 @@ ExternalWriteResult writeExternalPropertyValue(final DatabaseInternal database, final Binary blob = new Binary(); blob.putByte(ExternalValueRecord.RECORD_TYPE); blob.putByte(valueType); - if (typeByte == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_LZ4) { + if (isExternalCompressedType(typeByte)) { blob.putUnsignedNumber(uncompressedSize); // putByteArray writes the raw bytes without a length prefix; the compressed payload runs to end-of-record. blob.putByteArray(compressedPayload); @@ -1078,7 +1099,8 @@ public Object readExternalValue(final DatabaseInternal database, final int exter /** * Reads the value blob at the given external RID. When {@code compressed} is true, the blob's value-bytes are * LZ4-compressed and prefixed by an uncompressed-size varint. The compression flag is supplied by the caller - - * usually derived from the main record's type byte (TYPE_EXTERNAL vs TYPE_EXTERNAL_COMPRESSED_LZ4). + * usually derived from the main record's type byte. LZ4 fast and LZ4 HC share the same byte format, so this + * method does not need to know which encoder produced the bytes. */ public Object readExternalValue(final DatabaseInternal database, final int externalBucketId, final long position, final EmbeddedModifier embeddedModifier, final boolean compressed) { @@ -1128,10 +1150,9 @@ public Map findExistingExternalRids(final Database database, final buf.position(headerEndOffset + contentPosition); final byte type = buf.getByte(); - // Both raw and LZ4-compressed external pointers carry the same [bucketIdVarint][positionVarint] RID; - // the type byte differs only in how the blob is decoded - it doesn't change cascade-delete or orphan - // cleanup, both of which only need the RID. - if (type == BinaryTypes.TYPE_EXTERNAL || type == BinaryTypes.TYPE_EXTERNAL_COMPRESSED_LZ4) { + // All three external type bytes carry the same [bucketIdVarint][positionVarint] RID; only the blob + // decoder differs (raw / LZ4 fast / LZ4 HC). Cascade-delete and orphan cleanup only need the RID. + if (isExternalType(type)) { final int extBucketId = (int) buf.getNumber(); final long extPosition = buf.getNumber(); if (result == null) diff --git a/engine/src/main/java/com/arcadedb/serializer/BinaryTypes.java b/engine/src/main/java/com/arcadedb/serializer/BinaryTypes.java index ee2984001b..240b823ec2 100644 --- a/engine/src/main/java/com/arcadedb/serializer/BinaryTypes.java +++ b/engine/src/main/java/com/arcadedb/serializer/BinaryTypes.java @@ -65,8 +65,9 @@ public class BinaryTypes { public final static byte TYPE_ARRAY_OF_FLOATS = 26; // @SINCE 23.6.1 public final static byte TYPE_ARRAY_OF_DOUBLES = 27; // @SINCE 23.6.1 public final static byte TYPE_COMPRESSED_GEOMETRY = 28; // @SINCE 26.2.1 - Binary geometry storage (Point, Circle, Rectangle, etc.) - public final static byte TYPE_EXTERNAL = 29; // @SINCE 26.5.1 - Property value stored uncompressed in a paired external bucket. Followed by [bucketIdVarint][positionVarint]. - public final static byte TYPE_EXTERNAL_COMPRESSED_LZ4 = 30; // @SINCE 26.5.1 - Same as TYPE_EXTERNAL but the value bytes in the external blob are LZ4-compressed; the type byte is the dispatcher (no per-blob algo marker needed). + public final static byte TYPE_EXTERNAL = 29; // @SINCE 26.5.1 - Property value stored uncompressed in a paired external bucket. Followed by [bucketIdVarint][positionVarint]. + public final static byte TYPE_EXTERNAL_COMPRESSED_FAST = 30; // @SINCE 26.5.1 - Same as TYPE_EXTERNAL but the value bytes in the external blob are LZ4-fast-compressed; the type byte is the dispatcher (no per-blob algo marker needed). + public final static byte TYPE_EXTERNAL_COMPRESSED_MAX = 31; // @SINCE 26.5.1 - Same as TYPE_EXTERNAL_COMPRESSED_FAST but compressed with LZ4 HC (high compression, ~10pp smaller output, 8-20x slower compress; decompression is identical to FAST since LZ4 HC uses the same format). // Geometry subtypes for TYPE_COMPRESSED_GEOMETRY public final static byte GEOMETRY_SUBTYPE_POINT = 1; // Point: x(double), y(double) diff --git a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java index ce8f44904d..3d38630a9c 100644 --- a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java +++ b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java @@ -19,13 +19,22 @@ package com.arcadedb.schema; import com.arcadedb.TestHelper; +import com.arcadedb.database.Document; +import com.arcadedb.database.EmbeddedDocument; import com.arcadedb.database.MutableDocument; +import com.arcadedb.database.MutableEmbeddedDocument; import com.arcadedb.database.RID; import com.arcadedb.engine.LocalBucket; import com.arcadedb.graph.MutableVertex; import com.arcadedb.query.sql.executor.ResultSet; import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + import static org.assertj.core.api.Assertions.assertThat; /** @@ -691,8 +700,12 @@ void sqlDdlExternalCompression() { }); assertThat(database.getSchema().getType("Doc").getProperty("body").getCompression()).isEqualToIgnoringCase("auto"); + // Legacy alias: "lz4" must still be accepted and is normalised to the new "fast" tier name. database.transaction(() -> database.command("sql", "ALTER PROPERTY Doc.body COMPRESSION 'lz4'")); - assertThat(database.getSchema().getType("Doc").getProperty("body").getCompression()).isEqualToIgnoringCase("lz4"); + assertThat(database.getSchema().getType("Doc").getProperty("body").getCompression()).isEqualToIgnoringCase("fast"); + + database.transaction(() -> database.command("sql", "ALTER PROPERTY Doc.body COMPRESSION 'max'")); + assertThat(database.getSchema().getType("Doc").getProperty("body").getCompression()).isEqualToIgnoringCase("max"); database.transaction(() -> database.command("sql", "ALTER PROPERTY Doc.body COMPRESSION 'none'")); assertThat(database.getSchema().getType("Doc").getProperty("body").getCompression()).isEqualToIgnoringCase("none"); @@ -720,4 +733,153 @@ void rollbackDiscardsBothPrimaryAndExternal() { assertThat(external.count()).as("external bucket count must also be unchanged after rollback") .isEqualTo(externalCountBefore); } + + /** + * EXTERNAL must work for top-level Type.LIST: writeExternalPropertyValue routes through serializeValue, which + * already handles TYPE_LIST. The whole list lands in the paired bucket as one blob; reads materialise it back. + */ + @Test + void valueRoundTripListProperty() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("name", Type.STRING); + type.createProperty("tags", Type.LIST).setExternal(true); + + final List tags = new ArrayList<>(); + tags.add("alpha"); + tags.add(42); + tags.add(3.14); + tags.add("very-long-string-".repeat(50)); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Doc").set("name", "doc-with-list").set("tags", tags); + d.save(); + saved[0] = d.getIdentity(); + }); + + // Reopen so we exercise the deserializer, not an in-memory buffer. + database.close(); + database = factory.open(); + + final Document loaded = database.lookupByRID(saved[0], true).asDocument(); + assertThat(loaded.getString("name")).isEqualTo("doc-with-list"); + final List readBack = loaded.getList("tags"); + assertThat(readBack).containsExactlyElementsOf(tags); + + // Update the list in-place: only the external blob changes; the main record's pointer stays valid. + final List updated = new ArrayList<>(tags); + updated.add("appended"); + database.transaction(() -> { + final MutableDocument m = database.lookupByRID(saved[0], true).asDocument().modify(); + m.set("tags", updated); + m.save(); + }); + + database.close(); + database = factory.open(); + + final List readBackUpdated = database.lookupByRID(saved[0], true).asDocument().getList("tags"); + assertThat(readBackUpdated).containsExactlyElementsOf(updated); + } + + /** + * EXTERNAL must work for top-level Type.MAP. Same deal: the map serialises through TYPE_MAP into the external + * blob. Use a LinkedHashMap so iteration order is deterministic for the assertion. + */ + @Test + void valueRoundTripMapProperty() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("name", Type.STRING); + type.createProperty("attrs", Type.MAP).setExternal(true); + + final Map attrs = new LinkedHashMap<>(); + attrs.put("city", "Rome"); + attrs.put("zip", 100); + attrs.put("score", 9.81); + attrs.put("notes", "lorem ipsum ".repeat(80)); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument d = database.newDocument("Doc").set("name", "doc-with-map").set("attrs", attrs); + d.save(); + saved[0] = d.getIdentity(); + }); + + database.close(); + database = factory.open(); + + final Document loaded = database.lookupByRID(saved[0], true).asDocument(); + assertThat(loaded.getString("name")).isEqualTo("doc-with-map"); + final Map readBack = loaded.getMap("attrs"); + // Use containsAllEntriesOf for order-insensitive equality on the materialised map. + assertThat(readBack).containsAllEntriesOf(attrs); + assertThat(readBack.size()).isEqualTo(attrs.size()); + + // Mutate one entry; verify the external blob is rewritten in place and the change survives reopen. + database.transaction(() -> { + final MutableDocument m = database.lookupByRID(saved[0], true).asDocument().modify(); + final Map mutated = new LinkedHashMap<>(m.getMap("attrs")); + mutated.put("zip", 200); + m.set("attrs", mutated); + m.save(); + }); + + database.close(); + database = factory.open(); + + final Map readBackUpdated = database.lookupByRID(saved[0], true).asDocument().getMap("attrs"); + assertThat(readBackUpdated.get("zip")).isEqualTo(200); + assertThat(readBackUpdated.get("city")).isEqualTo("Rome"); + } + + /** + * EXTERNAL must work for top-level Type.EMBEDDED. The embedded document's own header lives in the external blob; + * the deserializer wires the EmbeddedModifier (parent + property) on read so the embedded knows its owner. + */ + @Test + void valueRoundTripEmbeddedProperty() { + database.getSchema().createDocumentType("Address"); + final DocumentType person = database.getSchema().createDocumentType("Person"); + person.createProperty("name", Type.STRING); + person.createProperty("address", Type.EMBEDDED).setExternal(true); + + final RID[] saved = new RID[1]; + database.transaction(() -> { + final MutableDocument p = database.newDocument("Person").set("name", "alice"); + final MutableEmbeddedDocument addr = p.newEmbeddedDocument("Address", "address"); + addr.set("street", "Via Roma"); + addr.set("number", 7); + addr.set("city", "Rome"); + p.save(); + saved[0] = p.getIdentity(); + }); + + database.close(); + database = factory.open(); + + final Document loaded = database.lookupByRID(saved[0], true).asDocument(); + assertThat(loaded.getString("name")).isEqualTo("alice"); + final EmbeddedDocument readBack = loaded.getEmbedded("address"); + assertThat(readBack).isNotNull(); + assertThat(readBack.getString("street")).isEqualTo("Via Roma"); + assertThat(readBack.getInteger("number")).isEqualTo(7); + assertThat(readBack.getString("city")).isEqualTo("Rome"); + + // Mutate the embedded by replacing it with a fresh MutableEmbeddedDocument; the rewrite hits the external bucket. + database.transaction(() -> { + final MutableDocument m = database.lookupByRID(saved[0], true).asDocument().modify(); + final MutableEmbeddedDocument addr = m.newEmbeddedDocument("Address", "address"); + addr.set("street", "Piazza Navona"); + addr.set("number", 99); + addr.set("city", "Rome"); + m.save(); + }); + + database.close(); + database = factory.open(); + + final EmbeddedDocument readBackUpdated = database.lookupByRID(saved[0], true).asDocument().getEmbedded("address"); + assertThat(readBackUpdated.getString("street")).isEqualTo("Piazza Navona"); + assertThat(readBackUpdated.getInteger("number")).isEqualTo(99); + } } diff --git a/studio/src/main/resources/static/js/studio-database.js b/studio/src/main/resources/static/js/studio-database.js index 9766eaa4ec..98afbc9a77 100644 --- a/studio/src/main/resources/static/js/studio-database.js +++ b/studio/src/main/resources/static/js/studio-database.js @@ -1072,10 +1072,11 @@ function createProperty(typeName) { html += ""; @@ -2693,48 +2694,20 @@ function browseType(typeName) { let limit = parseInt($("#inputLimit").val()) || 100; let query = "select from `" + typeName + "`"; - // If a graph already exists, append results to it - if (globalCy != null && globalResultset != null) { - $("#inputLanguage").val("sql"); - editor.setValue(query); - globalActivateTab("tab-query"); - globalActivateTab("tab-graph"); + $("#inputLanguage").val("sql"); + editor.setValue(query); + globalActivateTab("tab-query"); - $("#executeSpinner").show(); + let activeTab = $("#tabs-command .active").attr("id"); + let onGraph = (activeTab == "tab-graph-sel"); - jQuery.ajax({ - type: "POST", - url: "api/v1/command/" + database, - data: JSON.stringify({ language: "sql", command: escapeHtml(query), limit: limit, serializer: "studio" }), - beforeSend: function(xhr) { xhr.setRequestHeader("Authorization", globalCredentials); } - }).done(function(data) { - $("#executeSpinner").hide(); - appendToGraph(data.result); - $("#result-num").html(globalResultset.vertices.length + " vertices, " + globalTotalEdges + " edges"); - $("#resultJson").val(JSON.stringify({ result: globalResultset }, null, 2)); - }).fail(function(jqXHR) { - $("#executeSpinner").hide(); - globalNotifyError(jqXHR.responseText); - }); + // Honor the user's currently active result tab. Only on the graph tab + // do we keep the append-to-existing-graph behavior. + if (!onGraph) { + executeCommand("sql", query); return; } - // No existing graph - use normal executeCommand flow - executeCommand("sql", query); -} - -function browseType(typeName) { - let database = getCurrentDatabase(); - if (!database) return; - - let limit = parseInt($("#inputLimit").val()) || 100; - let query = "select from `" + typeName + "`"; - - $("#inputLanguage").val("sql"); - editor.setValue(query); - globalActivateTab("tab-query"); - globalActivateTab("tab-graph"); - $("#executeSpinner").show(); jQuery.ajax({ @@ -2747,11 +2720,9 @@ function browseType(typeName) { $("#resultJson").val(JSON.stringify(data, null, 2)); if (globalCy != null && globalResultset != null && data.result.vertices.length > 0) { - // Append to existing graph appendToGraph(data.result); $("#result-num").html(globalResultset.vertices.length + " vertices, " + globalTotalEdges + " edges"); } else { - // First click or no vertices - create fresh graph globalResultset = data.result; globalCy = null; $("#result-num").html(data.result.records.length); From b6a0248b0595650395724a602846efff5bc6d7b3 Mon Sep 17 00:00:00 2001 From: Luca Garulli Date: Wed, 29 Apr 2026 16:45:01 -0400 Subject: [PATCH 09/12] Fix from claude's report --- .../compression/CompressionFactory.java | 9 ++ .../database/ExternalValueRecord.java | 9 ++ .../com/arcadedb/engine/DatabaseChecker.java | 15 +- .../executor/FetchFromSchemaTypesStep.java | 5 +- .../sql/parser/RebuildTypeStatement.java | 52 ++++++- .../arcadedb/schema/LocalDocumentType.java | 41 +++++- .../arcadedb/serializer/BinarySerializer.java | 46 +++--- .../arcadedb/schema/ExternalPropertyTest.java | 133 +++++++++++++++++- 8 files changed, 274 insertions(+), 36 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/compression/CompressionFactory.java b/engine/src/main/java/com/arcadedb/compression/CompressionFactory.java index 7208bb1eb7..83fe1b5a22 100644 --- a/engine/src/main/java/com/arcadedb/compression/CompressionFactory.java +++ b/engine/src/main/java/com/arcadedb/compression/CompressionFactory.java @@ -27,4 +27,13 @@ public class CompressionFactory { public static Compression getDefault() { return defaultImplementation; } + + /** + * Returns the shared {@link LZ4Compression} instance. Exposes the concrete type for callers that need + * encoder variants beyond the {@link Compression} interface (e.g. {@link LZ4Compression#compressMax} for + * the LZ4 HC tier used by EXTERNAL property storage). + */ + public static LZ4Compression getLZ4() { + return defaultImplementation; + } } diff --git a/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java b/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java index 35a3d326e5..85a4a6df70 100644 --- a/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java +++ b/engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java @@ -26,6 +26,15 @@ * @author Luca Garulli (l.garulli@arcadedata.com) */ public class ExternalValueRecord extends BaseRecord implements RecordInternal { + // BaseRecord-subclass type tags (first byte of every record buffer): + // Document = 0 (Document.java) + // Vertex = 1 (Vertex.java) + // Edge = 2 (Edge.java) + // EdgeSegment = 3 (EdgeSegment.java / MutableEdgeSegment.java) + // EmbeddedDocument = 4 (EmbeddedDocument.java) + // ExternalValueRecord = 5 (this class - paired-bucket payload for EXTERNAL property values) + // LightEdge = 6 (LightEdge.java) + // Add new values at the end and update this list. public static final byte RECORD_TYPE = 5; public ExternalValueRecord(final Database database, final RID rid, final Binary buffer) { diff --git a/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java b/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java index ffbf86381d..a03641029c 100644 --- a/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java +++ b/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java @@ -34,6 +34,7 @@ import com.arcadedb.schema.LocalVertexType; import com.arcadedb.schema.Schema; import com.arcadedb.serializer.json.JSONObject; +import com.arcadedb.utility.LongHashSet; import java.io.*; import java.util.*; @@ -269,8 +270,14 @@ private void checkExternalProperties() { // For every external bucket, build the set of positions actually referenced from primary records of its // owning type. Orphan = an external record whose position is NOT in that set. - final Map> referencedByExtBucketId = new HashMap<>(); - final Map extBucketsToCheck = new HashMap<>(); + // + // We use LongHashSet (open-addressing primitive long set) instead of HashSet to skip Long-boxing on + // every add/contains and shrink the per-entry footprint from ~48 B (Long box + HashMap.Node + array slot) + // to ~8 B + load-factor slack. On a database with 10M referenced positions this is the difference between + // ~50 MB and ~10 MB of CHECK DATABASE working set. A future streaming variant could sort both bucket scans + // by position and walk them in lockstep, dropping the heap footprint to O(1) at the cost of a sort. + final Map referencedByExtBucketId = new HashMap<>(); + final Map extBucketsToCheck = new HashMap<>(); for (final DocumentType type : database.getSchema().getTypes()) { if (!(type instanceof LocalDocumentType ldt) || !ldt.hasExternalProperties()) @@ -284,7 +291,7 @@ private void checkExternalProperties() { continue; final LocalBucket extBucket = (LocalBucket) database.getSchema().getBucketById(extBucketId); extBucketsToCheck.put(extBucketId, extBucket); - final Set referenced = referencedByExtBucketId.computeIfAbsent(extBucketId, k -> new HashSet<>()); + final LongHashSet referenced = referencedByExtBucketId.computeIfAbsent(extBucketId, k -> new LongHashSet()); primaryBucket.scan((rid, view) -> { try { @@ -301,7 +308,7 @@ private void checkExternalProperties() { for (final Map.Entry entry : extBucketsToCheck.entrySet()) { final LocalBucket extBucket = entry.getValue(); - final Set referenced = referencedByExtBucketId.get(entry.getKey()); + final LongHashSet referenced = referencedByExtBucketId.get(entry.getKey()); final List orphans = new ArrayList<>(); extBucket.scan((rid, view) -> { diff --git a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java index 78aa9c8d0d..e563a74083 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java +++ b/engine/src/main/java/com/arcadedb/query/sql/executor/FetchFromSchemaTypesStep.java @@ -19,6 +19,7 @@ package com.arcadedb.query.sql.executor; import com.arcadedb.database.Document; +import com.arcadedb.engine.Bucket; import com.arcadedb.engine.timeseries.ColumnDefinition; import com.arcadedb.engine.timeseries.DownsamplingTier; import com.arcadedb.engine.timeseries.TimeSeriesEngine; @@ -94,10 +95,10 @@ else if (type.getType() == Edge.RECORD_TYPE) // (Studio etc.) tell the user where the externalised values for each primary bucket are stored. if (type instanceof LocalDocumentType ldt) { final Map extMap = new HashMap<>(); - for (final var b : type.getBuckets(false)) { + for (final Bucket b : type.getBuckets(false)) { final Integer extId = ldt.getExternalBucketIdFor(b.getFileId()); if (extId != null) { - final var extBucket = context.getDatabase().getSchema().getBucketById(extId); + final Bucket extBucket = context.getDatabase().getSchema().getBucketById(extId); if (extBucket != null) extMap.put(b.getName(), extBucket.getName()); } diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java b/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java index 2e09720212..956a778adf 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java @@ -22,17 +22,26 @@ import com.arcadedb.database.MutableDocument; import com.arcadedb.exception.CommandExecutionException; import com.arcadedb.exception.CommandSQLParsingException; +import com.arcadedb.log.LogManager; import com.arcadedb.query.sql.executor.CommandContext; import com.arcadedb.query.sql.executor.InternalResultSet; import com.arcadedb.query.sql.executor.ResultInternal; import com.arcadedb.query.sql.executor.ResultSet; import com.arcadedb.schema.DocumentType; +import com.arcadedb.schema.LocalDocumentType; import com.arcadedb.schema.Schema; import java.util.*; /** * REBUILD TYPE typeName [POLYMORPHIC] [WITH batchSize = N] - re-serialises records to apply schema layout changes. + *

+ * Behaviour under Raft HA. This statement extends {@link DDLStatement}, so + * {@code Statement.isDDL()} returns {@code true}. {@code RaftReplicatedDatabase.command()} therefore forwards + * any REBUILD TYPE issued on a follower to the leader via {@code forwardCommandToLeaderViaRaft}; the leader is + * the only node that runs the loop locally. Each in-loop {@code db.commit()} produces one Raft log entry; + * followers replay them in order. A follower that crashes mid-replay can resync from the leader's snapshot + * (the half-migrated intermediate state is recoverable by re-running REBUILD on the leader). * * @author Luca Garulli (l.garulli@arcadedata.com) */ @@ -58,9 +67,21 @@ public ResultSet executeDDL(final CommandContext context) { int batchSize = DEFAULT_BATCH_SIZE; for (final Map.Entry e : settings.entrySet()) { final String key = e.getKey().toString(); - if (key.equalsIgnoreCase("batchSize")) - batchSize = Integer.parseInt(e.getValue().value.toString()); - else + if (key.equalsIgnoreCase("batchSize")) { + final Object raw = e.getValue().value; + try { + batchSize = Integer.parseInt(raw == null ? "null" : raw.toString()); + } catch (final NumberFormatException nfe) { + throw new CommandSQLParsingException( + "REBUILD TYPE setting 'batchSize' must be a positive integer, got: " + raw); + } + // batchSize is the modulus for the in-rebuild commit cadence (count[0] % batchSize == 0). A zero or + // negative value would either ArithmeticException (mod-zero) or never trip the commit branch (modulo by + // a negative still works but the boundary semantics are nonsensical). Reject up front. + if (batchSize <= 0) + throw new CommandSQLParsingException( + "REBUILD TYPE setting 'batchSize' must be a positive integer, got: " + batchSize); + } else throw new CommandSQLParsingException( "Unrecognized setting '" + key + "' in REBUILD TYPE statement (supported: batchSize)"); } @@ -104,7 +125,10 @@ public ResultSet executeDDL(final CommandContext context) { "Error on rebuilding type '" + typeName.getStringValue() + "' after " + count[0] + " records (" + migratedAndKept + " committed in earlier batches and remain in the new layout, " + rolledBack + " rolled back from the in-flight batch). REBUILD TYPE is NOT atomic across batches; re-run the" - + " command to migrate the remaining records once the underlying issue is fixed.", e); + + " command to migrate the remaining records once the underlying issue is fixed." + + " If the failed batch left external-bucket blobs that no primary record references (typical when the" + + " rebuild crashes between writing the external value and re-serializing the primary record), run" + + " CHECK DATABASE FIX to delete the orphaned external records.", e); } // If the rebuild was triggered to revert a property from EXTERNAL to inline (i.e. the type no longer has @@ -115,14 +139,30 @@ public ResultSet executeDDL(final CommandContext context) { // queued record updates haven't flushed yet (LocalDatabase.updateRecord defers serialization to commit), // so the orphan cleanup hasn't run and the buckets are still non-empty. The caller can re-run REBUILD // outside their tx to trigger reclaim, or invoke it explicitly. - if (implicitTx && type instanceof com.arcadedb.schema.LocalDocumentType ldt && !ldt.hasExternalProperties()) - ldt.reclaimEmptyExternalBuckets(); + boolean reclaimSkipped = false; + if (type instanceof LocalDocumentType ldt && !ldt.hasExternalProperties()) { + if (implicitTx) + ldt.reclaimEmptyExternalBuckets(); + else if (ldt.hasExternalBuckets()) + // Caller-supplied tx with empty-EXTERNAL state and paired buckets still around: we cannot drop them + // here (records haven't flushed), so surface the situation in the result + log so the operator knows + // to re-run REBUILD outside a transaction to reclaim the buckets. + reclaimSkipped = true; + } final ResultInternal result = new ResultInternal(db); result.setProperty("operation", "rebuild type"); result.setProperty("typeName", typeName.getStringValue()); result.setProperty("polymorphic", polymorphic); result.setProperty("recordsRebuilt", count[0]); + if (reclaimSkipped) { + final String warning = "REBUILD TYPE " + typeName.getStringValue() + + " ran inside a caller-supplied transaction; paired external buckets were NOT reclaimed because " + + "queued record updates haven't been flushed yet. Re-run REBUILD TYPE outside a transaction (or " + + "commit and re-run) to drop the now-empty paired buckets."; + result.setProperty("warning", warning); + LogManager.instance().log(this, java.util.logging.Level.WARNING, warning); + } final InternalResultSet rs = new InternalResultSet(); rs.add(result); return rs; diff --git a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java index 7805e0602f..7e8be91209 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java @@ -46,6 +46,7 @@ import java.io.*; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.*; import java.util.logging.*; public class LocalDocumentType implements DocumentType { @@ -71,7 +72,7 @@ public class LocalDocumentType implements DocumentType { // Cached count of OWN properties (not inherited) currently flagged EXTERNAL. Avoids O(N) scans of // getPolymorphicProperties() on hot paths (cascadeDeleteExternalValues, addBucketInternal, addSuperType). // Maintained by LocalProperty.setExternal, dropProperty, and the schema-load path. - final java.util.concurrent.atomic.AtomicInteger ownExternalPropertyCount = new java.util.concurrent.atomic.AtomicInteger(0); + final AtomicInteger ownExternalPropertyCount = new AtomicInteger(0); public LocalDocumentType(final LocalSchema schema, final String name) { this.schema = schema; @@ -995,6 +996,11 @@ public Integer getExternalBucketIdFor(final int primaryBucketId) { return externalBucketIdByPrimaryBucketId.get(primaryBucketId); } + /** True when this type still owns at least one paired external-property bucket. */ + public boolean hasExternalBuckets() { + return !externalBucketIdByPrimaryBucketId.isEmpty(); + } + public void ensureExternalBuckets() { for (final Bucket b : buckets) ensureExternalBucketFor((LocalBucket) b); @@ -1022,7 +1028,12 @@ private void ensureExternalBucketFor(final LocalBucket primary) { && external.getPurpose() != LocalBucket.Purpose.EXTERNAL_PROPERTY) throw new SchemaException( "Cannot adopt bucket '" + extName + "' as the external-property bucket for type '" + name - + "': it is already a primary bucket of another user type. Rename the conflicting bucket and retry."); + + "': it is already a primary bucket of another user type. The paired external bucket is named" + + " _ext, so a primary bucket called '" + extName + "' (or one whose name" + + " ends in '_ext' that matches another type's primary bucket) collides with this convention." + + " Resolve by renaming the conflicting primary bucket via 'ALTER BUCKET " + extName + + " NAME ...' so its name no longer ends in '_ext' or no longer collides, then retry the EXTERNAL" + + " property change."); } else { // External buckets get larger pages (256KB vs 64KB primary), a smaller slot table (256 vs 2048: file-format // version EXTERNAL_BUCKET_VERSION), and optional placement on cheaper-storage tier via @@ -1040,10 +1051,27 @@ private void ensureExternalBucketFor(final LocalBucket primary) { /** * Drops paired external-property buckets that no longer back any record (typical case: a REBUILD TYPE that * moved every value back inline because the EXTERNAL flag was just toggled off). Skips buckets that still - * hold records so we never lose data; those point at persistent corruption and need investigation. Caller - * is expected to verify {@code !hasExternalProperties()} before calling. + * hold records so we never lose data; those point at persistent corruption and need investigation. + *

+ * Preconditions (caller-enforced). + *

    + *
  1. {@code !hasExternalProperties()} - the type no longer has any EXTERNAL property to write into the + * bucket. Without this, a concurrent insert could legitimately write into the bucket between the + * {@code count() == 0} check and {@code dropBucket}.
    2. + *
  2. No transaction is active (or any active transaction has been committed before this call). The + * {@code count()} read consults pageManager state that has not yet flushed dirty pages from a still-open + * transaction, so an open tx can hide records from this method and lead to a data-losing drop.
    3. + *
+ * The current production caller ({@code RebuildTypeStatement}) only invokes this on the + * {@code implicitTx == true} path, after committing its own transaction, so both preconditions are met. + * Any new caller MUST honour both, or restructure to take a schema-level write lock that blocks inserts + * for the duration of the count-then-drop sequence. */ public void reclaimEmptyExternalBuckets() { + if (hasExternalProperties()) + // Guard against accidental misuse from future callers; better to no-op than to silently drop a bucket + // that the schema still expects writes to flow into. + return; final List toDrop = new ArrayList<>(); for (final Map.Entry entry : externalBucketIdByPrimaryBucketId.entrySet()) { final LocalBucket extBucket = schema.getBucketById(entry.getValue(), false); @@ -1206,6 +1234,11 @@ else if (this instanceof LocalEdgeType edgeType) { if (!externalBucketIdByPrimaryBucketId.isEmpty()) { // PRIMARY BUCKET NAME -> EXTERNAL BUCKET NAME. NAMES (NOT IDS) ARE PERSISTED FOR HUMAN READABILITY AND // BECAUSE FILE IDS CAN BE REMAPPED ON FILE MIGRATION (LocalSchema.migratedFileIds). + // NAME DEPENDENCY: this serialised mapping is keyed by string. ArcadeDB does not currently expose a + // user-level RENAME BUCKET command, so the names are stable in practice. If a future feature lets a + // user rename a bucket, the rename code MUST update both sides of this mapping (or it will go stale on + // restart, and restoreExternalBuckets() will log SEVERE for the missing entry). Same constraint applies + // to the external bucket itself: its file name carries the primary's name + "_ext" suffix. final JSONObject extBuckets = new JSONObject(); for (final Map.Entry e : externalBucketIdByPrimaryBucketId.entrySet()) { final LocalBucket primary = schema.getBucketById(e.getKey(), false); diff --git a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java index 79129ce300..1941f93453 100644 --- a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java +++ b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java @@ -20,6 +20,8 @@ import com.arcadedb.ContextConfiguration; import com.arcadedb.GlobalConfiguration; +import com.arcadedb.compression.CompressionFactory; +import com.arcadedb.compression.LZ4Compression; import com.arcadedb.database.BaseRecord; import com.arcadedb.database.Binary; import com.arcadedb.database.DataEncryption; @@ -981,21 +983,6 @@ public ExternalWriteResult(final byte typeByte, final RID rid) { } } - // Lazy-init: LZ4Factory.fastestInstance() does JNI/SIMD probing on first call, so we cache the wrapper. - private static volatile com.arcadedb.compression.LZ4Compression lz4Singleton; - - private static com.arcadedb.compression.LZ4Compression lz4() { - com.arcadedb.compression.LZ4Compression local = lz4Singleton; - if (local == null) { - synchronized (BinarySerializer.class) { - local = lz4Singleton; - if (local == null) - lz4Singleton = local = new com.arcadedb.compression.LZ4Compression(); - } - } - return local; - } - /** True for any of TYPE_EXTERNAL, TYPE_EXTERNAL_COMPRESSED_FAST, TYPE_EXTERNAL_COMPRESSED_MAX. */ private static boolean isExternalType(final byte type) { return type == BinaryTypes.TYPE_EXTERNAL @@ -1040,7 +1027,9 @@ ExternalWriteResult writeExternalPropertyValue(final DatabaseInternal database, serializeValue(database, rawValueBytes, valueType, value); rawValueBytes.flip(); - final boolean fastMode = "fast".equalsIgnoreCase(compressionPolicy) || "lz4".equalsIgnoreCase(compressionPolicy); + // The "lz4" legacy alias is normalised to "fast" by LocalProperty.setCompression(), so by the time the + // policy reaches us it's already one of: null, "none", "fast", "max", "auto". No second alias check. + final boolean fastMode = "fast".equalsIgnoreCase(compressionPolicy); final boolean maxMode = "max".equalsIgnoreCase(compressionPolicy); final boolean autoMode = "auto".equalsIgnoreCase(compressionPolicy); final boolean tryCompress = fastMode || maxMode || autoMode; @@ -1051,7 +1040,8 @@ ExternalWriteResult writeExternalPropertyValue(final DatabaseInternal database, if (tryCompress && rawValueBytes.size() > 0) { final byte[] raw = rawValueBytes.toByteArray(); - final byte[] compressed = maxMode ? lz4().compressMax(raw) : lz4().compress(raw); + final LZ4Compression lz4 = CompressionFactory.getLZ4(); + final byte[] compressed = maxMode ? lz4.compressMax(raw) : lz4.compress(raw); // In auto mode skip compression unless it saves >10%. In fast/max mode always keep the compressed form // (the user explicitly asked to compress, even if a particular record happens not to gain much). if (!autoMode || compressed.length < raw.length * 0.9) { @@ -1105,6 +1095,15 @@ public Object readExternalValue(final DatabaseInternal database, final int exter public Object readExternalValue(final DatabaseInternal database, final int externalBucketId, final long position, final EmbeddedModifier embeddedModifier, final boolean compressed) { final LocalBucket externalBucket = database.getSchema().getEmbedded().getBucketById(externalBucketId); + if (externalBucket == null) + // Typical cause: the paired external bucket was created on a tiered path (configured via + // arcadedb.externalPropertyBucketPath) and the database was reopened with that config unset or + // pointing elsewhere, so FileManager's secondary scan never picked the file up. The schema still + // references the old bucket id, but the file isn't loaded -> we'd otherwise NPE on getRecord. + throw new SerializationException( + "Cannot read EXTERNAL property: external bucket id=" + externalBucketId + " is not loaded. " + + "If the bucket was tiered to a secondary path, set 'arcadedb.externalPropertyBucketPath' " + + "to the same value used at creation time and reopen the database."); final RID rid = RID.create(database, externalBucketId, position); final Binary buffer = externalBucket.getRecord(rid).copyOfContent(); buffer.position(Binary.BYTE_SERIALIZED_SIZE); // SKIP RECORD TYPE BYTE @@ -1116,11 +1115,20 @@ public Object readExternalValue(final DatabaseInternal database, final int exter final int compressedLen = buffer.size() - buffer.position(); final byte[] compressedBytes = new byte[compressedLen]; System.arraycopy(buffer.getContent(), buffer.position(), compressedBytes, 0, compressedLen); - final byte[] decompressed = lz4().decompress(compressedBytes, uncompressedSize); + final byte[] decompressed = CompressionFactory.getLZ4().decompress(compressedBytes, uncompressedSize); return deserializeValue(database, new Binary(decompressed), valueType, embeddedModifier); } - /** Reused by cascade-delete: scans the OLD buffer for TYPE_EXTERNAL pointers, keyed by property name. */ + /** + * Reused by cascade-delete and the orphan-cleanup-on-update path inside {@link #serializeProperties}: scans + * the OLD buffer for TYPE_EXTERNAL pointers, keyed by property name. + *

+ * Do NOT add a {@code hasExternalProperties()} early-out here. The + * {@link #serializeProperties} caller invokes this during the EXTERNAL→inline migration (REBUILD TYPE after + * {@code setExternal(false)}), when the type's current schema reports zero EXTERNAL properties but the OLD + * record buffer still carries TYPE_EXTERNAL pointers that must be discovered so the paired blobs can be + * deleted as orphans. The schema flag and the buffer contents are decoupled; the buffer is ground truth. + */ public Map findExistingExternalRids(final Database database, final Document record) { final RID identity = record.getIdentity(); if (identity == null) diff --git a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java index 3d38630a9c..3728e5a80a 100644 --- a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java +++ b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java @@ -36,6 +36,7 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** * Verifies the EXTERNAL property storage feature: when a property is flagged EXTERNAL, its value lives in a paired @@ -434,6 +435,39 @@ void rebuildTypeReversesExternalToInlineAndCleansOrphans() { .as("schema should no longer expose the dropped external bucket").isFalse(); } + /** + * REBUILD TYPE inside a caller-supplied transaction cannot reclaim now-empty paired buckets (the queued + * record updates haven't flushed). The result row must surface a {@code warning} so the operator knows to + * re-run REBUILD outside a transaction. + */ + @Test + void rebuildTypeWarnsWhenInsideCallerTransactionAndExternalBucketsRemain() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + database.transaction(() -> { + for (int i = 0; i < 5; i++) + database.newDocument("Doc").set("blob", "v-" + i).save(); + }); + type.getProperty("blob").setExternal(false); + + database.begin(); + final ResultSet rs = database.command("sql", "REBUILD TYPE Doc"); + assertThat(rs.hasNext()).isTrue(); + final var row = rs.next(); + assertThat((String) row.getProperty("warning")) + .as("warning must be set so the operator knows reclaim was deferred") + .contains("paired external buckets were NOT reclaimed"); + database.commit(); + + // After committing the caller tx, re-running REBUILD outside any transaction must complete the reclaim. + final ResultSet rs2 = database.command("sql", "REBUILD TYPE Doc"); + assertThat(rs2.hasNext()).isTrue(); + final var row2 = rs2.next(); + assertThat((String) row2.getProperty("warning")).as("second run owns the tx and reclaims; no warning").isNull(); + assertThat(((LocalDocumentType) database.getSchema().getType("Doc")).hasExternalBuckets()) + .as("paired external buckets should be dropped after the second REBUILD").isFalse(); + } + @Test void rebuildTypePolymorphicWalksSubtypes() { final DocumentType parent = database.getSchema().createDocumentType("Parent"); @@ -518,7 +552,7 @@ void externalBucketUsesLargerDefaultPageSize() { assertThat(external.getPageSize()).isEqualTo(262_144); assertThat(external.getPageSize()).isGreaterThan(primary.getPageSize()); - // External buckets carry few but heavy records, so the slot table is sized down (128 vs 2048): saves about + // External buckets carry few but heavy records, so the slot table is sized down (256 vs 2048): saves about // 7.5KB of header overhead per page (file-format version EXTERNAL_BUCKET_VERSION encodes this). assertThat(primary.getMaxRecordsInPage()).isEqualTo(2048); assertThat(external.getMaxRecordsInPage()).isEqualTo(256); @@ -882,4 +916,101 @@ void valueRoundTripEmbeddedProperty() { assertThat(readBackUpdated.getString("street")).isEqualTo("Piazza Navona"); assertThat(readBackUpdated.getInteger("number")).isEqualTo(99); } + + /** + * Two write semantics on an EXTERNAL property: + *

    + *
  • {@code set("field", null)} REUSES the paired external slot as a TYPE_NULL marker. The bucket count + * stays the same; on read the property returns null. This mirrors how a null inline value still + * occupies a property-header slot - the schema still tracks the property as present, just with a + * null value.
    • + *
  • {@code remove("field")} drops the property from the property header entirely; orphan-cleanup in + * BinarySerializer.serializeProperties detects the previously-paired RID is no longer consumed and + * deletes the external record, decrementing the bucket count.
    • + *
+ * Read-back must reflect the difference: null after set-null, "no such property" after remove. + */ + @Test + void externalPropertyNullVsRemoveSemantics() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + + final var primary = type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket external = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + + // Two records, each with an EXTERNAL value. + final RID[] saved = new RID[2]; + database.transaction(() -> { + final MutableDocument a = database.newDocument("Doc").set("blob", "value-a"); + a.save(); + saved[0] = a.getIdentity(); + final MutableDocument b = database.newDocument("Doc").set("blob", "value-b"); + b.save(); + saved[1] = b.getIdentity(); + }); + assertThat(external.count()).as("two external records expected").isEqualTo(2L); + + // set-null: slot is reused as TYPE_NULL, bucket count unchanged. + database.transaction(() -> { + final MutableDocument m = database.lookupByRID(saved[0], true).asDocument().modify(); + m.set("blob", (Object) null); + m.save(); + }); + assertThat(external.count()).as("set-null reuses the paired slot, count unchanged").isEqualTo(2L); + assertThat(database.lookupByRID(saved[0], true).asDocument().get("blob")) + .as("set-null reads back as null").isNull(); + + // remove(): drops the property entirely, orphan-cleanup releases the bucket entry. + database.transaction(() -> { + final MutableDocument m = database.lookupByRID(saved[1], true).asDocument().modify(); + m.remove("blob"); + m.save(); + }); + assertThat(external.count()).as("remove() releases the paired slot, count decreases").isEqualTo(1L); + assertThat(database.lookupByRID(saved[1], true).asDocument().get("blob")) + .as("remove() makes the property absent (and absence reads as null)").isNull(); + + // Reopen and confirm the on-disk state is consistent with what the in-memory bucket showed. + database.close(); + database = factory.open(); + final var primary2 = database.getSchema().getType("Doc").getBuckets(false).getFirst(); + final Integer extId2 = ((LocalDocumentType) database.getSchema().getType("Doc")).getExternalBucketIdFor( + primary2.getFileId()); + final LocalBucket external2 = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId2); + assertThat(external2.count()).as("post-reopen: one slot retained for set-null, one dropped by remove") + .isEqualTo(1L); + } + + /** + * REBUILD TYPE on a database opened READ_ONLY must surface a clean DatabaseIsReadOnlyException (the + * implicit {@code db.begin()} cannot start a write transaction). The error must wrap to a + * CommandExecutionException that includes the read-only signal so an operator inspecting Studio sees it. + */ + @Test + void rebuildTypeOnReadOnlyDatabaseFailsCleanly() { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + database.transaction(() -> database.newDocument("Doc").set("blob", "v").save()); + + database.close(); + database = factory.open(com.arcadedb.engine.ComponentFile.MODE.READ_ONLY); + try { + assertThatThrownBy(() -> database.command("sql", "REBUILD TYPE Doc")) + .isInstanceOfAny(com.arcadedb.exception.DatabaseIsReadOnlyException.class, + com.arcadedb.exception.CommandExecutionException.class) + .satisfies(t -> { + final String msg = (t.getMessage() == null ? "" : t.getMessage()) + + (t.getCause() == null || t.getCause().getMessage() == null ? "" : " " + t.getCause().getMessage()); + assertThat(msg.toLowerCase(java.util.Locale.ENGLISH)) + .as("error must mention read-only state so the operator sees the actual cause") + .containsAnyOf("read-only", "read only", "readonly"); + }); + } finally { + // TestHelper.afterTest runs CHECK DATABASE which requires write access; reopen RW so the harness can + // tear down cleanly without inheriting our READ_ONLY mode. + database.close(); + database = factory.open(); + } + } } From 46e852cf5199645f999f172baef98460390cb543 Mon Sep 17 00:00:00 2001 From: Luca Garulli Date: Wed, 29 Apr 2026 17:30:52 -0400 Subject: [PATCH 10/12] Implemented latest claude reviews --- .../java/com/arcadedb/engine/LocalBucket.java | 14 +++- .../sql/parser/AlterPropertyStatement.java | 4 + .../CreatePropertyAttributeStatement.java | 2 + .../arcadedb/schema/LocalDocumentType.java | 20 ++++- .../java/com/arcadedb/schema/Property.java | 5 +- .../arcadedb/serializer/BinarySerializer.java | 77 ++++++++++++++----- 6 files changed, 95 insertions(+), 27 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java index c25024f844..76c09ccec2 100644 --- a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java +++ b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java @@ -97,17 +97,23 @@ public class LocalBucket extends PaginatedComponent implements Bucket { protected final int contentHeaderSize; private final int maxRecordsInPage; private final AtomicLong cachedRecordCount = new AtomicLong(-1); - // Buckets are PRIMARY by default (they hold the primary records of a type and are user-targetable via DML). - // Internal kinds (e.g. EXTERNAL_PROPERTY) hold serializer infrastructure that user-facing DML must not target. - // The purpose is persisted in schema.json (per-type) and restored at load time, see LocalDocumentType. - private Purpose purpose = Purpose.PRIMARY; + /** + * Bucket purpose tag. Declared up here (next to the {@link #purpose} field that uses it) so the enum is the + * first thing a reader sees alongside the other bucket-level constants - the alternative was burying it + * mid-file between unrelated state, which made the EXTERNAL property contract hard to discover. + */ public enum Purpose { /** Bucket holding the primary records of a type (vertex/edge/document). Targetable by user DML. */ PRIMARY, /** Paired infrastructure bucket holding externalised property values. NOT targetable by user DML. */ EXTERNAL_PROPERTY } + + // Buckets are PRIMARY by default (they hold the primary records of a type and are user-targetable via DML). + // Internal kinds (e.g. EXTERNAL_PROPERTY) hold serializer infrastructure that user-facing DML must not target. + // The purpose is persisted in schema.json (per-type) and restored at load time, see LocalDocumentType. + private Purpose purpose = Purpose.PRIMARY; // pageId → free-space-bytes. TreeMap ordering is unused (verified by grep), so a primitive // open-addressing map saves memory and avoids Integer boxing on every read/write/remove on // the page-allocation hot path. Bounded by MAX_PAGES_GATHER_STATS (100). Single-threaded diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java b/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java index ef5b5b2333..b8f4ac08f8 100755 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java @@ -92,6 +92,10 @@ public ResultSet executeDDL(final CommandContext context) { oldValue = property.isExternal(); property.setExternal((boolean) finalValue); } else if (setting.equalsIgnoreCase("compression") || setting.equalsIgnoreCase("external_compression")) { + // CANONICAL NAME: 'compression'. The 'external_compression' alias is accepted for symmetry with the + // 'external' attribute (so users can spell out "this knob applies only to EXTERNAL properties"); new + // SQL should prefer 'compression' since the setting is meaningless for non-EXTERNAL properties anyway. + // Studio and our own DDL examples emit 'compression'. oldValue = property.getCompression(); property.setCompression(String.valueOf(finalValue)); } else if (setting.equalsIgnoreCase("max")) { diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java b/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java index cc39e1df7f..bc602c2622 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java @@ -74,6 +74,8 @@ public Object setOnProperty(final Property internalProp, final CommandContext co } else if (attrName.equalsIgnoreCase("external")) { internalProp.setExternal((boolean) attrValue); } else if (attrName.equalsIgnoreCase("compression") || attrName.equalsIgnoreCase("external_compression")) { + // CANONICAL NAME: 'compression' (mirrors AlterPropertyStatement). 'external_compression' is a legacy + // alias kept so existing scripts keep parsing; new DDL should use 'compression'. internalProp.setCompression(String.valueOf(attrValue)); } else if (attrName.equalsIgnoreCase("max")) { internalProp.setMax("" + attrValue); diff --git a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java index 7e8be91209..5c6d3d2f19 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java @@ -1068,10 +1068,20 @@ private void ensureExternalBucketFor(final LocalBucket primary) { * for the duration of the count-then-drop sequence. */ public void reclaimEmptyExternalBuckets() { + // Fail-fast on precondition #1 from the javadoc. A no-op return would silently mask the bug; throwing + // surfaces it during development AND production. The cost is one O(1) atomic read on a hot path that + // only runs after schema mutations and REBUILD TYPE, so it is negligible. if (hasExternalProperties()) - // Guard against accidental misuse from future callers; better to no-op than to silently drop a bucket - // that the schema still expects writes to flow into. - return; + throw new IllegalStateException( + "reclaimEmptyExternalBuckets() requires !hasExternalProperties() but type '" + name + "' still has at" + + " least one EXTERNAL property. Calling this with EXTERNAL properties present would race with" + + " concurrent inserts that legitimately write into the bucket between the count() check and" + + " dropBucket(). Drop the EXTERNAL flag on every property of the type before reclaiming."); + // The TOCTOU window between count() == 0 and dropBucket() is closed when the caller honours the second + // precondition (no active transaction). With the EXTERNAL flag off, no new code path will write into + // these buckets; with no in-flight tx, no queued update can have a record waiting to flush either. + assert !((LocalDatabase) schema.getDatabase()).isTransactionActive() : + "reclaimEmptyExternalBuckets() must run with no active transaction; see javadoc precondition #2"; final List toDrop = new ArrayList<>(); for (final Map.Entry entry : externalBucketIdByPrimaryBucketId.entrySet()) { final LocalBucket extBucket = schema.getBucketById(entry.getValue(), false); @@ -1239,6 +1249,10 @@ else if (this instanceof LocalEdgeType edgeType) { // user rename a bucket, the rename code MUST update both sides of this mapping (or it will go stale on // restart, and restoreExternalBuckets() will log SEVERE for the missing entry). Same constraint applies // to the external bucket itself: its file name carries the primary's name + "_ext" suffix. + // TODO(rename-bucket): when a user-level RENAME BUCKET is introduced (see LocalSchema), it MUST also + // (a) re-key this map's primary entry, (b) rename the paired '_ext' bucket file to + // '_ext' to keep the naming convention consistent, and (c) re-save the schema so the JSON + // mirrors the new state. A grep for "TODO(rename-bucket)" surfaces every site that needs updating. final JSONObject extBuckets = new JSONObject(); for (final Map.Entry e : externalBucketIdByPrimaryBucketId.entrySet()) { final LocalBucket primary = schema.getBucketById(e.getKey(), false); diff --git a/engine/src/main/java/com/arcadedb/schema/Property.java b/engine/src/main/java/com/arcadedb/schema/Property.java index 4fe536af17..f61cecd875 100644 --- a/engine/src/main/java/com/arcadedb/schema/Property.java +++ b/engine/src/main/java/com/arcadedb/schema/Property.java @@ -88,7 +88,10 @@ public interface Property { * speed regardless of tier. Best default when writes are frequent. *
  • {@code max} - LZ4 HC encoder. ~10pp smaller output than {@code fast}, 8-20x slower compress; * decompression speed is the same as {@code fast}. Best for write-once / read-many payloads.
    • - *
  • {@code auto} - try {@code fast}; keep compressed only when it saves more than 10% of bytes.
    • + *
  • {@code auto} - try {@code fast}; keep compressed only when it saves more than 10% of bytes. + * Cost: on no-win records (e.g. dense float32 embeddings) the work is "compress, measure, throw + * it away, fall back to raw". You pay one wasted LZ4 compress + one extra byte-array copy per record + * compared to {@code none}. Use {@code none} explicitly when the workload tips no-win consistently.
    • * * The legacy alias {@code lz4} is accepted and stored as {@code fast}. Ignored for non-EXTERNAL properties. */ diff --git a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java index 1941f93453..5eef7085a2 100644 --- a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java +++ b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java @@ -972,12 +972,15 @@ public Binary serializeProperties(final Database database, final Document record return header; } - /** Holder for {@link #writeExternalPropertyValue}: the bytes written and the type byte to put in the main record. */ - public static final class ExternalWriteResult { - public final byte typeByte; - public final RID rid; + /** + * Holder for {@link #writeExternalPropertyValue}: the bytes written and the type byte to put in the main + * record. Package-private along with the writer; tests reach it via {@code BinarySerializerTestHelper}. + */ + static final class ExternalWriteResult { + final byte typeByte; + final RID rid; - public ExternalWriteResult(final byte typeByte, final RID rid) { + ExternalWriteResult(final byte typeByte, final RID rid) { this.typeByte = typeByte; this.rid = rid; } @@ -1021,12 +1024,6 @@ ExternalWriteResult writeExternalPropertyValue(final DatabaseInternal database, "Existing external RID " + existingExternalRid + " does not match the paired external bucket id " + externalBucketId + " for this record. The schema's external bucket mapping is inconsistent."); - // Step 1: serialise the raw value bytes. We do this even when compressing, because we need both the raw size - // (uncompressed-size header) and the option to fall back to raw on auto-mode no-win. - final Binary rawValueBytes = new Binary(); - serializeValue(database, rawValueBytes, valueType, value); - rawValueBytes.flip(); - // The "lz4" legacy alias is normalised to "fast" by LocalProperty.setCompression(), so by the time the // policy reaches us it's already one of: null, "none", "fast", "max", "auto". No second alias check. final boolean fastMode = "fast".equalsIgnoreCase(compressionPolicy); @@ -1034,11 +1031,41 @@ ExternalWriteResult writeExternalPropertyValue(final DatabaseInternal database, final boolean autoMode = "auto".equalsIgnoreCase(compressionPolicy); final boolean tryCompress = fastMode || maxMode || autoMode; + final Binary blob = new Binary(); + + if (!tryCompress) { + // Fast path (no compression policy): serialise straight into the blob. Avoids the extra Binary buffer + + // toByteArray() roundtrip the compressed path needs. This is the dense-vector default (compression is a + // loss for embeddings) and the throughput-critical case for write-heavy workloads. + blob.putByte(ExternalValueRecord.RECORD_TYPE); + blob.putByte(valueType); + serializeValue(database, blob, valueType, value); + blob.flip(); + return finalizeExternalWrite(database, externalBucketId, existingExternalRid, blob, BinaryTypes.TYPE_EXTERNAL); + } + + // Compressed path: we need the raw bytes both to compress and to fall back on auto-mode no-win, so this is + // the one case where the intermediate Binary is unavoidable. + // + // AUTO-MODE OVERHEAD (accepted, not optimised). When auto-mode picks the no-win branch the work done is: + // 1. serializeValue once into rawValueBytes (always paid) + // 2. rawValueBytes.toByteArray() byte[] copy (paid even when we end up not keeping the compressed form) + // 3. lz4.compress() call (CPU spent for nothing) + // 4. blob.append(rawValueBytes) byte copy (effectively a third copy of the raw bytes) + // versus the non-compress fast path's single serialise-straight-into-blob. The serialise itself runs once, + // not twice; the cost is the extra byte[] copy + the discarded compress call. For workloads that actually + // tip into the no-win branch routinely (e.g. dense float32 embeddings), use {@code none} explicitly. Auto + // is correct for mixed workloads where most records benefit and the throughput hit on the minority is + // acceptable. We do not memoise the compressor output across calls because it is per-record state. + final Binary rawValueBytes = new Binary(); + serializeValue(database, rawValueBytes, valueType, value); + rawValueBytes.flip(); + byte typeByte = BinaryTypes.TYPE_EXTERNAL; byte[] compressedPayload = null; int uncompressedSize = 0; - if (tryCompress && rawValueBytes.size() > 0) { + if (rawValueBytes.size() > 0) { final byte[] raw = rawValueBytes.toByteArray(); final LZ4Compression lz4 = CompressionFactory.getLZ4(); final byte[] compressed = maxMode ? lz4.compressMax(raw) : lz4.compress(raw); @@ -1051,8 +1078,6 @@ ExternalWriteResult writeExternalPropertyValue(final DatabaseInternal database, } } - // Step 2: build the blob the bucket will store. - final Binary blob = new Binary(); blob.putByte(ExternalValueRecord.RECORD_TYPE); blob.putByte(valueType); if (isExternalCompressedType(typeByte)) { @@ -1063,8 +1088,16 @@ ExternalWriteResult writeExternalPropertyValue(final DatabaseInternal database, blob.append(rawValueBytes); } blob.flip(); + return finalizeExternalWrite(database, externalBucketId, existingExternalRid, blob, typeByte); + } - // Step 3: insert or update in the paired bucket, with delta accounting consistent with cascade-delete. + /** + * Inserts or updates the given blob in the paired external bucket and returns the type byte + RID for the + * caller to embed in the main record. Extracted from {@link #writeExternalPropertyValue} to keep the two + * write paths (raw / compressed) short and to avoid duplicating the bucket-side accounting. + */ + private ExternalWriteResult finalizeExternalWrite(final DatabaseInternal database, final int externalBucketId, + final RID existingExternalRid, final Binary blob, final byte typeByte) { final LocalBucket externalBucket = database.getSchema().getEmbedded().getBucketById(externalBucketId); final RID rid; if (existingExternalRid == null) { @@ -1077,7 +1110,6 @@ ExternalWriteResult writeExternalPropertyValue(final DatabaseInternal database, externalBucket.updateRecord(rec, true); rid = existingExternalRid; } - return new ExternalWriteResult(typeByte, rid); } @@ -1135,12 +1167,17 @@ public Map findExistingExternalRids(final Database database, final return Collections.emptyMap(); if (!(record instanceof BaseDocument)) return Collections.emptyMap(); - final Binary oldBuffer = ((BaseRecord) record).getBuffer(); - if (oldBuffer == null) + final Binary buf = ((BaseRecord) record).getBuffer(); + if (buf == null) return Collections.emptyMap(); + // Scan the record's own buffer in place (no copy). The buffer is the read-side of the record's content, + // which by the time serializeProperties reaches us has finished its own reads (it pulls the property map + // first via propertiesAsMap, which deserialises and caches into a Map, then calls us). Cascade-delete and + // CHECK DATABASE both call us outside any other buffer iteration. We still save/restore the position so + // a future caller mid-iteration would not be disturbed - cheap insurance for a hot update path. + final int savedPosition = buf.position(); try { - final Binary buf = oldBuffer.copyOfContent(); buf.position(((DocumentInternal) record).getPropertiesStartingPosition()); final int headerEndOffset = buf.getInt(); @@ -1177,6 +1214,8 @@ public Map findExistingExternalRids(final Database database, final + "record may be orphaned in the paired bucket.", e, identity, e.getMessage()); return Collections.emptyMap(); + } finally { + buf.position(savedPosition); } } From 688b88571467192349d2065faed2e1d5402d4da6 Mon Sep 17 00:00:00 2001 From: Luca Garulli Date: Wed, 29 Apr 2026 20:04:20 -0400 Subject: [PATCH 11/12] Last fix from claude review --- .../sql/parser/AlterPropertyStatement.java | 6 +- .../CreatePropertyAttributeStatement.java | 4 +- .../arcadedb/schema/LocalDocumentType.java | 66 ++++++++++++-- .../java/com/arcadedb/schema/LocalSchema.java | 10 ++- .../java/com/arcadedb/schema/Property.java | 12 +++ .../arcadedb/serializer/BinarySerializer.java | 86 +++++++++++-------- .../arcadedb/schema/ExternalPropertyTest.java | 78 +++++++++++++---- 7 files changed, 190 insertions(+), 72 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java b/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java index b8f4ac08f8..a684b833b9 100755 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/AlterPropertyStatement.java @@ -91,11 +91,7 @@ public ResultSet executeDDL(final CommandContext context) { } else if (setting.equalsIgnoreCase("external")) { oldValue = property.isExternal(); property.setExternal((boolean) finalValue); - } else if (setting.equalsIgnoreCase("compression") || setting.equalsIgnoreCase("external_compression")) { - // CANONICAL NAME: 'compression'. The 'external_compression' alias is accepted for symmetry with the - // 'external' attribute (so users can spell out "this knob applies only to EXTERNAL properties"); new - // SQL should prefer 'compression' since the setting is meaningless for non-EXTERNAL properties anyway. - // Studio and our own DDL examples emit 'compression'. + } else if (setting.equalsIgnoreCase("compression")) { oldValue = property.getCompression(); property.setCompression(String.valueOf(finalValue)); } else if (setting.equalsIgnoreCase("max")) { diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java b/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java index bc602c2622..d50a3b39c7 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/CreatePropertyAttributeStatement.java @@ -73,9 +73,7 @@ public Object setOnProperty(final Property internalProp, final CommandContext co internalProp.setHidden((boolean) attrValue); } else if (attrName.equalsIgnoreCase("external")) { internalProp.setExternal((boolean) attrValue); - } else if (attrName.equalsIgnoreCase("compression") || attrName.equalsIgnoreCase("external_compression")) { - // CANONICAL NAME: 'compression' (mirrors AlterPropertyStatement). 'external_compression' is a legacy - // alias kept so existing scripts keep parsing; new DDL should use 'compression'. + } else if (attrName.equalsIgnoreCase("compression")) { internalProp.setCompression(String.valueOf(attrValue)); } else if (attrName.equalsIgnoreCase("max")) { internalProp.setMax("" + attrValue); diff --git a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java index 5c6d3d2f19..95cdbb0775 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalDocumentType.java @@ -1021,11 +1021,16 @@ private void ensureExternalBucketFor(final LocalBucket primary) { final LocalBucket external; if (schema.bucketMap.containsKey(extName)) { external = schema.bucketMap.get(extName); - // Refuse to adopt a bucket that is already registered as the primary bucket of some user type. A bucket - // freshly loaded from disk shows purpose=PRIMARY (transient field), so we cannot rely on purpose alone; - // bucketId2TypeMap is the authoritative source of "this bucket is a user type's primary bucket". - if (schema.getTypeByBucketId(external.getFileId()) != null - && external.getPurpose() != LocalBucket.Purpose.EXTERNAL_PROPERTY) + // Refuse to adopt a bucket that is already registered as the primary bucket of some user type. + // {@code bucketId2TypeMap} is the authoritative source of "this bucket is a user type's primary + // bucket": it is rebuilt from each type's {@code getBuckets(false)} list (primary buckets only; + // adopted external buckets are NOT in this map, they live in per-type externalBucketIdByPrimaryBucketId). + // We therefore do NOT consult {@link LocalBucket#getPurpose}: purpose is transient and reset to PRIMARY + // on load, and depending on it would create a fragile ordering requirement against + // restoreExternalBuckets() running first. {@code computeIfAbsent} above already short-circuits when + // this type's own ext bucket is already adopted, so any hit here means the candidate name belongs to + // a different type's primary bucket, full stop. + if (schema.getTypeByBucketId(external.getFileId()) != null) throw new SchemaException( "Cannot adopt bucket '" + extName + "' as the external-property bucket for type '" + name + "': it is already a primary bucket of another user type. The paired external bucket is named" @@ -1080,8 +1085,15 @@ public void reclaimEmptyExternalBuckets() { // The TOCTOU window between count() == 0 and dropBucket() is closed when the caller honours the second // precondition (no active transaction). With the EXTERNAL flag off, no new code path will write into // these buckets; with no in-flight tx, no queued update can have a record waiting to flush either. - assert !((LocalDatabase) schema.getDatabase()).isTransactionActive() : - "reclaimEmptyExternalBuckets() must run with no active transaction; see javadoc precondition #2"; + // Use an explicit throw instead of `assert`: production JVMs run without -ea, and a violation here can + // lose data (we'd drop a bucket that has a queued tx record about to flush into it). Same pattern as + // the hasExternalProperties() guard two lines above. + if (((LocalDatabase) schema.getDatabase()).isTransactionActive()) + throw new IllegalStateException( + "reclaimEmptyExternalBuckets() requires no active transaction but one is open on database '" + + schema.getDatabase().getName() + "'. Queued record updates have not flushed yet, so the" + + " count() check could miss records about to land in the bucket and we'd drop it under them." + + " Commit (or rollback) the active transaction before calling reclaimEmptyExternalBuckets()."); final List toDrop = new ArrayList<>(); for (final Map.Entry entry : externalBucketIdByPrimaryBucketId.entrySet()) { final LocalBucket extBucket = schema.getBucketById(entry.getValue(), false); @@ -1099,7 +1111,16 @@ public void reclaimEmptyExternalBuckets() { schema.saveConfiguration(); } - /** Re-applies EXTERNAL_PROPERTY purpose (transient on LocalBucket) and rebuilds the map from JSON at load time. */ + /** + * Re-applies EXTERNAL_PROPERTY purpose (transient on {@link LocalBucket}) and rebuilds the map from JSON at + * load time. After processing the JSON-driven entries, runs a name-based heuristic sweep over this type's + * primary buckets: for any primary bucket whose paired '_ext' sibling exists in the schema's + * bucketMap but was missing from the JSON (corruption, partial migration, JSON edited by hand), adopt the + * sibling as the external bucket and tag its purpose. Without this fallback the {@code purpose} field would + * default to {@code PRIMARY}, the DML write guard ({@code LocalDatabase.createRecordNoLock}) would let users + * target the bucket directly, and an INSERT could corrupt internal payload bytes. Adoption is refused for + * any '_ext' bucket that bucketId2TypeMap already claims as another type's primary bucket. + */ void restoreExternalBuckets(final Map primaryNameToExternalName) { externalBucketIdByPrimaryBucketId.clear(); for (final Map.Entry entry : primaryNameToExternalName.entrySet()) { @@ -1125,6 +1146,35 @@ void restoreExternalBuckets(final Map primaryNameToExternalName) external.setPurpose(LocalBucket.Purpose.EXTERNAL_PROPERTY); externalBucketIdByPrimaryBucketId.put(primary.getFileId(), external.getFileId()); } + + // Heuristic recovery: for every primary bucket of this type that does NOT yet have a paired entry in + // externalBucketIdByPrimaryBucketId, look for '_ext' in bucketMap and adopt it. Defends + // against schema.json missing the externalBuckets key (corruption, migration from an older snapshot, or + // a hand-edit). Refuses to adopt if the candidate is already registered as another type's primary + // bucket (bucketId2TypeMap is authoritative; the field-level Purpose is transient and unreliable here). + for (final Bucket primaryBucket : buckets) { + if (externalBucketIdByPrimaryBucketId.containsKey(primaryBucket.getFileId())) + continue; + final String candidateName = primaryBucket.getName() + "_ext"; + final LocalBucket candidate = schema.bucketMap.get(candidateName); + if (candidate == null) + continue; + if (schema.getTypeByBucketId(candidate.getFileId()) != null) { + // candidate is some other type's primary bucket; refuse to repurpose, just log so the operator sees it. + LogManager.instance().log(this, Level.WARNING, + "Heuristic recovery for type '%s': bucket '%s' looks like a paired external bucket by name but is" + + " already a primary bucket of another user type. Skipping adoption.", + null, name, candidateName); + continue; + } + candidate.setPurpose(LocalBucket.Purpose.EXTERNAL_PROPERTY); + externalBucketIdByPrimaryBucketId.put(primaryBucket.getFileId(), candidate.getFileId()); + LogManager.instance().log(this, Level.WARNING, + "Heuristic recovery for type '%s': adopted bucket '%s' as the external-property bucket for primary" + + " '%s'. The schema.json was missing the matching externalBuckets entry; it will be re-saved on" + + " the next schema mutation.", + null, name, candidateName, primaryBucket.getName()); + } } protected void removeBucketInternal(final Bucket bucket) { diff --git a/engine/src/main/java/com/arcadedb/schema/LocalSchema.java b/engine/src/main/java/com/arcadedb/schema/LocalSchema.java index bbd9e50d41..e94082a5ac 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalSchema.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalSchema.java @@ -1498,14 +1498,18 @@ protected synchronized void readConfiguration() { } // RESTORE THE primaryBucket -> externalBucket MAP BEFORE PROPERTIES ARE LOADED, SO THAT setExternal(true) ON A - // PROPERTY DOES NOT TRY TO LAZY-CREATE BUCKETS THAT ALREADY EXIST. + // PROPERTY DOES NOT TRY TO LAZY-CREATE BUCKETS THAT ALREADY EXIST. Always call restoreExternalBuckets + // - even when the JSON has no externalBuckets key - so the name-based heuristic inside it can adopt + // any orphan '_ext' files that exist on disk but were lost from the JSON (partial corruption, + // migration from an older snapshot, etc.). Without that pass the affected buckets would default to + // purpose=PRIMARY and our DML write guard would let users target them. + final Map primaryToExternal = new HashMap<>(); if (schemaType.has("externalBuckets")) { final JSONObject extBuckets = schemaType.getJSONObject("externalBuckets"); - final Map primaryToExternal = new HashMap<>(); for (final String primaryName : extBuckets.keySet()) primaryToExternal.put(primaryName, extBuckets.getString(primaryName)); - type.restoreExternalBuckets(primaryToExternal); } + type.restoreExternalBuckets(primaryToExternal); type.custom.clear(); if (schemaType.has("custom")) diff --git a/engine/src/main/java/com/arcadedb/schema/Property.java b/engine/src/main/java/com/arcadedb/schema/Property.java index f61cecd875..12ffb93c8f 100644 --- a/engine/src/main/java/com/arcadedb/schema/Property.java +++ b/engine/src/main/java/com/arcadedb/schema/Property.java @@ -76,6 +76,18 @@ public interface Property { boolean isHidden(); + /** + * When true, the property's value is stored in a paired external bucket instead of inline in the primary + * record. The main record carries only a small TYPE_EXTERNAL pointer, so traversal-only queries that don't + * project this property never load its bytes. Best for vector embeddings, large strings, embedded JSON, and + * full-text payloads. + *

    + * Null semantics. {@code set("field", null)} on an EXTERNAL property does NOT consume external bucket + * space: the serializer writes an inline TYPE_NULL byte and releases any pre-existing paired blob via the + * orphan-cleanup pass. {@code set(field, null)} and {@code remove(field)} are therefore equivalent in terms + * of paired-bucket storage; they differ only in whether the property header slot is retained (set-null + * keeps it, remove drops it). Reads return null in both cases. + */ Property setExternal(boolean external); boolean isExternal(); diff --git a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java index 5eef7085a2..fca92e392f 100644 --- a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java +++ b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java @@ -121,6 +121,7 @@ public Binary serializeDocument(final DatabaseInternal database, final Document header.position(Binary.BYTE_SERIALIZED_SIZE); serializeProperties = false; } + if (serializeProperties) return serializeProperties(database, document, header, context.getTemporaryBuffer2()); @@ -893,39 +894,50 @@ public Binary serializeProperties(final Database database, final Document record final Property propertyDef = documentType.getPropertyIfExists(propertyName); if (propertyDef != null && propertyDef.isExternal()) { - // Externalised property: write the value to the paired external bucket and put a TYPE_EXTERNAL marker (with - // the external RID) in the main record's content. The main record stays small and traversal-only reads never - // hit the external bucket. See LocalDocumentType.getExternalBucketIdFor. - final RID identity = record.getIdentity(); - if (identity == null) - throw new SerializationException( - "Cannot serialize EXTERNAL property '" + propertyName + "' on type '" + documentType.getName() - + "': record has no target bucket. The bucket layer must set a provisional identity before serialize."); - final int primaryBucketId = identity.getBucketId(); - // Look up the external bucket via the type that ACTUALLY owns the primary bucket. This may differ from - // record.getType(): polymorphic scans (scanType POLYMORPHIC, MATCH, etc.) tag every record with the queried - // parent type even when the record physically lives in a subtype's bucket. Trusting documentType in that - // case would miss the subtype's external bucket map. - final LocalDocumentType ownerType = (LocalDocumentType) database.getSchema().getEmbedded().getTypeByBucketId(primaryBucketId); - final Integer extBucketId = (ownerType != null ? ownerType : (LocalDocumentType) documentType) - .getExternalBucketIdFor(primaryBucketId); - if (extBucketId == null) - throw new SerializationException( - "Cannot serialize EXTERNAL property '" + propertyName + "' on type '" + documentType.getName() - + "': no external bucket is paired with primary bucket " + primaryBucketId); + // NULL is not externalised. set("field", null) is treated semantically the same as remove("field") + // for storage purposes: we write a TYPE_NULL byte INLINE in the main record and let the orphan- + // cleanup pass at the end of this method delete any pre-existing external blob (the property is + // intentionally NOT added to consumedExternalProperties below, so orphan cleanup runs). This keeps + // user mental model intuitive (null means "no payload") and avoids charging external-bucket space + // for null markers, which on a paired-bucket layout would otherwise force one external record per + // null-valued record. + if (value == null) { + content.putByte(BinaryTypes.TYPE_NULL); + } else { + // Externalised property: write the value to the paired external bucket and put a TYPE_EXTERNAL marker (with + // the external RID) in the main record's content. The main record stays small and traversal-only reads never + // hit the external bucket. See LocalDocumentType.getExternalBucketIdFor. + final RID identity = record.getIdentity(); + if (identity == null) + throw new SerializationException( + "Cannot serialize EXTERNAL property '" + propertyName + "' on type '" + documentType.getName() + + "': record has no target bucket. The bucket layer must set a provisional identity before serialize."); + final int primaryBucketId = identity.getBucketId(); + // Look up the external bucket via the type that ACTUALLY owns the primary bucket. This may differ from + // record.getType(): polymorphic scans (scanType POLYMORPHIC, MATCH, etc.) tag every record with the queried + // parent type even when the record physically lives in a subtype's bucket. Trusting documentType in that + // case would miss the subtype's external bucket map. + final LocalDocumentType ownerType = (LocalDocumentType) database.getSchema().getEmbedded().getTypeByBucketId(primaryBucketId); + final Integer extBucketId = (ownerType != null ? ownerType : (LocalDocumentType) documentType) + .getExternalBucketIdFor(primaryBucketId); + if (extBucketId == null) + throw new SerializationException( + "Cannot serialize EXTERNAL property '" + propertyName + "' on type '" + documentType.getName() + + "': no external bucket is paired with primary bucket " + primaryBucketId); - final RID existingExtRid = existingExternalRids.get(propertyName); - if (consumedExternalProperties != null && existingExtRid != null) - consumedExternalProperties.add(propertyName); + final RID existingExtRid = existingExternalRids.get(propertyName); + if (consumedExternalProperties != null && existingExtRid != null) + consumedExternalProperties.add(propertyName); - final ExternalWriteResult written = writeExternalPropertyValue((DatabaseInternal) database, extBucketId, - existingExtRid, type, value, propertyDef.getCompression()); + final ExternalWriteResult written = writeExternalPropertyValue((DatabaseInternal) database, extBucketId, + existingExtRid, type, value, propertyDef.getCompression()); - // The persisted type byte tells the reader which decoder to use. Bucket id and position are varints, - // mirroring TYPE_COMPRESSED_RID, so each pointer averages 3-7 bytes vs 12 fixed. - content.putByte(written.typeByte); - content.putNumber(written.rid.getBucketId()); - content.putNumber(written.rid.getPosition()); + // The persisted type byte tells the reader which decoder to use. Bucket id and position are varints, + // mirroring TYPE_COMPRESSED_RID, so each pointer averages 3-7 bytes vs 12 fixed. + content.putByte(written.typeByte); + content.putNumber(written.rid.getBucketId()); + content.putNumber(written.rid.getPosition()); + } } else { if (value instanceof String stringValue && type == BinaryTypes.TYPE_STRING) { final int id = dictionary.getIdByName(stringValue, false); @@ -973,16 +985,18 @@ public Binary serializeProperties(final Database database, final Document record } /** - * Holder for {@link #writeExternalPropertyValue}: the bytes written and the type byte to put in the main - * record. Package-private along with the writer; tests reach it via {@code BinarySerializerTestHelper}. + * Holder for {@link #writeExternalPropertyValue}: the RID where the value blob lives and the type byte the + * caller should embed in the main record. Field order matches the conceptual weight (the RID is the bulk of + * the pointer; the type byte is the 1-byte discriminator). Package-private along with the writer; tests + * reach it via {@code BinarySerializerTestHelper}. */ static final class ExternalWriteResult { - final byte typeByte; final RID rid; + final byte typeByte; - ExternalWriteResult(final byte typeByte, final RID rid) { - this.typeByte = typeByte; + ExternalWriteResult(final RID rid, final byte typeByte) { this.rid = rid; + this.typeByte = typeByte; } } @@ -1110,7 +1124,7 @@ private ExternalWriteResult finalizeExternalWrite(final DatabaseInternal databas externalBucket.updateRecord(rec, true); rid = existingExternalRid; } - return new ExternalWriteResult(typeByte, rid); + return new ExternalWriteResult(rid, typeByte); } public Object readExternalValue(final DatabaseInternal database, final int externalBucketId, final long position, diff --git a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java index 3728e5a80a..211c204ff9 100644 --- a/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java +++ b/engine/src/test/java/com/arcadedb/schema/ExternalPropertyTest.java @@ -918,20 +918,17 @@ void valueRoundTripEmbeddedProperty() { } /** - * Two write semantics on an EXTERNAL property: - *

      - *
    • {@code set("field", null)} REUSES the paired external slot as a TYPE_NULL marker. The bucket count - * stays the same; on read the property returns null. This mirrors how a null inline value still - * occupies a property-header slot - the schema still tracks the property as present, just with a - * null value.
      • - *
    • {@code remove("field")} drops the property from the property header entirely; orphan-cleanup in - * BinarySerializer.serializeProperties detects the previously-paired RID is no longer consumed and - * deletes the external record, decrementing the bucket count.
      • - *
    - * Read-back must reflect the difference: null after set-null, "no such property" after remove. + * On an EXTERNAL property, BOTH {@code set("field", null)} and {@code remove("field")} release the paired + * external blob (the bucket count drops). Rationale: charging external-bucket space for a null marker is + * surprising; users expect null to mean "no payload". The serializer special-cases null on the EXTERNAL + * write path: it writes an inline TYPE_NULL byte in the main record's content (not a TYPE_EXTERNAL pointer) + * and lets the orphan-cleanup pass delete any pre-existing external blob. + *

    + * Read-back semantics still differ: set-null leaves the property present (returns null), remove() makes the + * property absent (also reads as null on get(), but the property-header slot is gone). */ @Test - void externalPropertyNullVsRemoveSemantics() { + void externalPropertyNullAndRemoveBothReleasePairedBlob() { final DocumentType type = database.getSchema().createDocumentType("Doc"); type.createProperty("blob", Type.STRING).setExternal(true); @@ -951,13 +948,13 @@ void externalPropertyNullVsRemoveSemantics() { }); assertThat(external.count()).as("two external records expected").isEqualTo(2L); - // set-null: slot is reused as TYPE_NULL, bucket count unchanged. + // set-null: orphan-cleanup releases the paired bucket entry. database.transaction(() -> { final MutableDocument m = database.lookupByRID(saved[0], true).asDocument().modify(); m.set("blob", (Object) null); m.save(); }); - assertThat(external.count()).as("set-null reuses the paired slot, count unchanged").isEqualTo(2L); + assertThat(external.count()).as("set-null releases the paired slot, count decreases").isEqualTo(1L); assertThat(database.lookupByRID(saved[0], true).asDocument().get("blob")) .as("set-null reads back as null").isNull(); @@ -967,7 +964,7 @@ void externalPropertyNullVsRemoveSemantics() { m.remove("blob"); m.save(); }); - assertThat(external.count()).as("remove() releases the paired slot, count decreases").isEqualTo(1L); + assertThat(external.count()).as("remove() releases the paired slot, count decreases to zero").isEqualTo(0L); assertThat(database.lookupByRID(saved[1], true).asDocument().get("blob")) .as("remove() makes the property absent (and absence reads as null)").isNull(); @@ -978,8 +975,11 @@ void externalPropertyNullVsRemoveSemantics() { final Integer extId2 = ((LocalDocumentType) database.getSchema().getType("Doc")).getExternalBucketIdFor( primary2.getFileId()); final LocalBucket external2 = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId2); - assertThat(external2.count()).as("post-reopen: one slot retained for set-null, one dropped by remove") - .isEqualTo(1L); + assertThat(external2.count()).as("post-reopen: both slots dropped (one by set-null, one by remove)") + .isEqualTo(0L); + // set-null record still reads back as null after reopen; the inline TYPE_NULL byte is the canonical marker. + assertThat(database.lookupByRID(saved[0], true).asDocument().get("blob")) + .as("set-null record reads as null even after reopen").isNull(); } /** @@ -1013,4 +1013,48 @@ void rebuildTypeOnReadOnlyDatabaseFailsCleanly() { database = factory.open(); } } + + /** + * Defends the DML write guard against a corrupted/old schema.json where the {@code externalBuckets} key was + * lost. The on-disk '_ext' bucket file is still there, but the JSON map is empty. On reopen, the + * name-based heuristic in {@code restoreExternalBuckets} must re-tag the '_ext' bucket as + * {@code EXTERNAL_PROPERTY} so an INSERT INTO bucket:Doc_0_ext stays rejected. + */ + @Test + void heuristicRecoveryAdoptsOrphanExtBucketOnSchemaJsonMissingEntry() throws java.io.IOException { + final DocumentType type = database.getSchema().createDocumentType("Doc"); + type.createProperty("blob", Type.STRING).setExternal(true); + database.transaction(() -> database.newDocument("Doc").set("blob", "v").save()); + + final var primary = type.getBuckets(false).getFirst(); + final Integer extId = ((LocalDocumentType) type).getExternalBucketIdFor(primary.getFileId()); + final LocalBucket extBucket = ((LocalSchema) database.getSchema().getEmbedded()).getBucketById(extId); + final String extBucketName = extBucket.getName(); + + database.close(); + + // Strip the externalBuckets key from schema.json to simulate an older snapshot or partial corruption. + final java.io.File schemaJson = new java.io.File(database.getDatabasePath(), "schema.json"); + String content = java.nio.file.Files.readString(schemaJson.toPath()); + final com.arcadedb.serializer.json.JSONObject schema = new com.arcadedb.serializer.json.JSONObject(content); + final com.arcadedb.serializer.json.JSONObject docType = schema.getJSONObject("types").getJSONObject("Doc"); + docType.remove("externalBuckets"); + java.nio.file.Files.writeString(schemaJson.toPath(), schema.toString()); + + database = factory.open(); + + // Heuristic recovery should have re-tagged the _ext bucket; the user-bucket DML guard now rejects writes. + // Use the Java path (more deterministic error: IllegalArgumentException with "internal" in the message). + final MutableDocument fresh = database.newDocument("Doc").set("blob", "x"); + assertThatThrownBy(() -> database.transaction(() -> + ((com.arcadedb.database.DatabaseInternal) database).createRecord(fresh, extBucketName))) + .as("the heuristic must re-tag the bucket so user DML is still refused after schema.json loss") + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("internal"); + + // The mapping is back in memory; record reads on the type still resolve correctly. + final ResultSet rs = database.query("sql", "SELECT blob FROM Doc"); + assertThat(rs.hasNext()).isTrue(); + assertThat((String) rs.next().getProperty("blob")).isEqualTo("v"); + } } From 0210da9ecbae80d4b70ded38a5f42ac1ed930c3a Mon Sep 17 00:00:00 2001 From: Luca Garulli Date: Wed, 29 Apr 2026 21:26:53 -0400 Subject: [PATCH 12/12] More fixes after claude review --- .../com/arcadedb/engine/DatabaseChecker.java | 6 ++ .../java/com/arcadedb/engine/LocalBucket.java | 12 ++++ .../query/sql/antlr/SQLASTBuilder.java | 15 ++++- .../sql/parser/RebuildTypeStatement.java | 12 ++++ .../com/arcadedb/schema/AbstractProperty.java | 7 ++- .../java/com/arcadedb/schema/LocalSchema.java | 14 +++++ .../arcadedb/serializer/BinarySerializer.java | 59 +++++++++++++++---- 7 files changed, 110 insertions(+), 15 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java b/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java index a03641029c..f0b4fcad02 100644 --- a/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java +++ b/engine/src/main/java/com/arcadedb/engine/DatabaseChecker.java @@ -358,6 +358,12 @@ private void checkExternalProperties() { result.put("orphanedExternalRecords", (long) orphanedExternalRecords.size()); result.put("orphanedExternalRecordsFixed", fixedCount); + // BinarySerializer.findExistingExternalRids() catches parse failures and returns an empty map, which + // means the caller skips orphan-cleanup for that record. Surfacing the JVM-cumulative count here lets the + // operator notice corruption-driven leak rates climbing without having to grep WARN logs. The counter is + // process-static, so re-runs of CHECK report the running total since startup. + result.put("externalRidScanFailuresCumulative", + com.arcadedb.serializer.BinarySerializer.getExternalRidScanFailures()); ((LinkedHashSet) result.get("warnings")).addAll(warnings); if (fix) ((LinkedHashSet) result.get("deletedRecordsAfterFix")).addAll(orphanedExternalRecords); diff --git a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java index 76c09ccec2..720d297da1 100644 --- a/engine/src/main/java/com/arcadedb/engine/LocalBucket.java +++ b/engine/src/main/java/com/arcadedb/engine/LocalBucket.java @@ -159,6 +159,7 @@ public LocalBucket(final DatabaseInternal database, final String name, final Str this.contentHeaderSize = PAGE_RECORD_TABLE_OFFSET + (maxRecordsInPage * INT_SERIALIZED_SIZE); this.cachedRecordCount.set(0); this.reuseSpaceMode = REUSE_SPACE_MODE.valueOf(GlobalConfiguration.BUCKET_REUSE_SPACE_MODE.getValueAsString().toUpperCase()); + this.purpose = purposeForVersion(version); } /** @@ -170,10 +171,21 @@ public LocalBucket(final DatabaseInternal database, final String name, final Str this.maxRecordsInPage = maxRecordsInPageForVersion(version); contentHeaderSize = PAGE_RECORD_TABLE_OFFSET + (maxRecordsInPage * INT_SERIALIZED_SIZE); this.reuseSpaceMode = REUSE_SPACE_MODE.valueOf(GlobalConfiguration.BUCKET_REUSE_SPACE_MODE.getValueAsString().toUpperCase()); + // Derive purpose from the bucket file version - the version itself is persisted in the on-disk file name + // (e.g. Doc_0_ext...v1.bucket), so this assignment is reliable as soon as the LocalBucket + // is constructed - long before LocalDocumentType.restoreExternalBuckets() runs. That closes the gap where + // a write path firing between FileManager scan and schema-load completion would have seen purpose=PRIMARY + // by default and bypassed the user-DML guard. Schema JSON still maps primary->external by name (which is + // an orthogonal concern), but the write guard now no longer depends on schema-load ordering. + this.purpose = purposeForVersion(version); if (this.reuseSpaceMode.ordinal() >= REUSE_SPACE_MODE.HIGH.ordinal()) gatherPageStatistics(); } + private static Purpose purposeForVersion(final int version) { + return version >= EXTERNAL_BUCKET_VERSION ? Purpose.EXTERNAL_PROPERTY : Purpose.PRIMARY; + } + @Override public void close() { super.close(); diff --git a/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java b/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java index ad3229ccf1..11b7654fc7 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java +++ b/engine/src/main/java/com/arcadedb/query/sql/antlr/SQLASTBuilder.java @@ -34,7 +34,9 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; @@ -5740,10 +5742,19 @@ public RebuildTypeStatement visitRebuildTypeStmt(final SQLParser.RebuildTypeStmt stmt.typeName = (Identifier) visit(bodyCtx.typeName); stmt.polymorphic = bodyCtx.POLYMORPHIC() != null; if (bodyCtx.WITH() != null) { + // Track which settingKey strings we've already seen so a duplicate (e.g. WITH batchSize=1, batchSize=2) + // doesn't silently shadow the first via Map.put. Bare Expression equality on the key would only catch + // exact AST matches; using the lowercased identifier string also catches case variants. + final Set seenKeys = new HashSet<>(); for (int i = 0; i < bodyCtx.settingValue.size(); i++) { - final Expression key = new Expression((Identifier) visit(bodyCtx.settingKey.get(i))); + final Identifier keyId = (Identifier) visit(bodyCtx.settingKey.get(i)); + final String keyName = keyId.getStringValue().toLowerCase(Locale.ENGLISH); + if (!seenKeys.add(keyName)) + throw new CommandSQLParsingException( + "REBUILD TYPE WITH clause has duplicate setting '" + keyId.getStringValue() + + "'. Each setting must appear at most once."); final Expression value = (Expression) visit(bodyCtx.settingValue.get(i)); - stmt.settings.put(key, value); + stmt.settings.put(new Expression(keyId), value); } } return stmt; diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java b/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java index 956a778adf..b204b9234c 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildTypeStatement.java @@ -95,6 +95,7 @@ public ResultSet executeDDL(final CommandContext context) { if (implicitTx) db.begin(); + final long startNanos = System.nanoTime(); try { db.scanType(typeName.getStringValue(), polymorphic, rec -> { final MutableDocument m = (MutableDocument) rec.modify(); @@ -106,6 +107,13 @@ public ResultSet executeDDL(final CommandContext context) { if (implicitTx && count[0] % finalBatchSize == 0) { db.commit(); committedBefore[0] = count[0]; + // Per-batch progress line so an operator running REBUILD on a 100M-record type can observe forward + // motion via the server log instead of staring at a frozen prompt for many minutes. Logged at INFO + // so it's on by default but easy to silence by raising the level for this class. + final long elapsedMs = (System.nanoTime() - startNanos) / 1_000_000L; + LogManager.instance().log(this, java.util.logging.Level.INFO, + "REBUILD TYPE '%s': %,d records re-serialised so far (last batch=%,d, elapsed=%,d ms)", + null, typeName.getStringValue(), count[0], finalBatchSize, elapsedMs); db.begin(); } return true; @@ -113,6 +121,10 @@ public ResultSet executeDDL(final CommandContext context) { if (implicitTx) db.commit(); + final long elapsedMs = (System.nanoTime() - startNanos) / 1_000_000L; + LogManager.instance().log(this, java.util.logging.Level.INFO, + "REBUILD TYPE '%s' completed: %,d records re-serialised in %,d ms", + null, typeName.getStringValue(), count[0], elapsedMs); } catch (Exception e) { if (implicitTx && db.isTransactionActive()) db.rollback(); diff --git a/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java b/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java index 6898cafa00..aba1122a55 100644 --- a/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java +++ b/engine/src/main/java/com/arcadedb/schema/AbstractProperty.java @@ -39,7 +39,12 @@ public abstract class AbstractProperty implements Property { protected boolean mandatory = false; protected boolean notNull = false; protected boolean hidden = false; - protected boolean external = false; + // Volatile because BinarySerializer.serializeProperties reads isExternal() outside the schema write lock + // on the per-record write hot path. The schema lock serialises mutations, but a reader that came in just + // before setExternal() flipped the bit must observe the latest value to route the value through the + // correct write path (inline vs paired bucket). volatile is the cheapest correctness fix and matches the + // memory-model role of {@link LocalDocumentType#ownExternalPropertyCount}'s atomic. + protected volatile boolean external = false; // Compression policy for EXTERNAL property values: "none" | "fast" | "max" | "auto" (legacy alias: "lz4" -> "fast"). // STORAGE CONVENTION: null means "none" (the default), so toJSON omits the key. Read access MUST go through // getCompression(), which materialises null as the literal string "none". LocalProperty.setCompression diff --git a/engine/src/main/java/com/arcadedb/schema/LocalSchema.java b/engine/src/main/java/com/arcadedb/schema/LocalSchema.java index e94082a5ac..26c9eee18a 100644 --- a/engine/src/main/java/com/arcadedb/schema/LocalSchema.java +++ b/engine/src/main/java/com/arcadedb/schema/LocalSchema.java @@ -391,6 +391,20 @@ public LocalBucket createBucket(final String bucketName, final int pageSize, fin if (bucketMap.containsKey(bucketName)) throw new SchemaException("Cannot create bucket '" + bucketName + "' because already exists"); + // Discoverability warning for the EXTERNAL property naming convention. The engine creates paired buckets + // as '_ext' with file-format version EXTERNAL_BUCKET_VERSION, so version == CURRENT_VERSION (0) + // here means a user-driven CREATE BUCKET. A user bucket named '*_ext' will collide if a primary bucket + // with the matching prefix later gains an EXTERNAL property; ensureExternalBucketFor() rejects with a + // SchemaException at that point. Surfacing the constraint at create time is much cheaper than debugging + // the later failure. + if (version == LocalBucket.CURRENT_VERSION && bucketName.endsWith("_ext")) + LogManager.instance().log(this, Level.WARNING, + "Bucket name '%s' ends with '_ext'. The engine reserves the '_ext' suffix for paired" + + " EXTERNAL-property buckets. If a primary bucket whose name + '_ext' equals this name later" + + " gains an EXTERNAL property, that property change will fail with a SchemaException. Consider" + + " renaming this bucket to avoid the collision.", + null, bucketName); + final String dir = (parentDirectory == null || parentDirectory.isEmpty()) ? databasePath : parentDirectory; return recordFileChanges(() -> { diff --git a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java index fca92e392f..d9a20baf71 100644 --- a/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java +++ b/engine/src/main/java/com/arcadedb/serializer/BinarySerializer.java @@ -71,6 +71,7 @@ import java.time.*; import java.time.temporal.*; import java.util.*; +import java.util.concurrent.atomic.*; import java.util.logging.*; /** @@ -89,6 +90,20 @@ public class BinarySerializer { // Cached WKT writer for fast Point serialization (avoid recreating writer for each Point) private static volatile ShapeWriter cachedWktWriter; + /** + * Process-cumulative count of records whose OLD buffer could not be parsed by + * {@link #findExistingExternalRids}. Each parse failure means the record's external blobs (if any) were + * NOT discovered and could be left as orphans in the paired bucket. CHECK DATABASE surfaces this counter + * so an operator can see corruption-driven leaks accumulating without waiting for the next bucket scan. + * Static so the value survives instance churn and tracks the JVM lifetime. + */ + private static final AtomicLong externalRidScanFailures = new AtomicLong(0); + + /** Returns the JVM-cumulative count of {@link #findExistingExternalRids} parse failures since process start. */ + public static long getExternalRidScanFailures() { + return externalRidScanFailures.get(); + } + public BinarySerializer(final ContextConfiguration configuration) throws ClassNotFoundException { setDateImplementation(configuration.getValue(GlobalConfiguration.DATE_IMPLEMENTATION)); setDateTimeImplementation(configuration.getValue(GlobalConfiguration.DATE_TIME_IMPLEMENTATION)); @@ -1096,8 +1111,12 @@ ExternalWriteResult writeExternalPropertyValue(final DatabaseInternal database, blob.putByte(valueType); if (isExternalCompressedType(typeByte)) { blob.putUnsignedNumber(uncompressedSize); - // putByteArray writes the raw bytes without a length prefix; the compressed payload runs to end-of-record. - blob.putByteArray(compressedPayload); + // Length-prefixed payload (Binary.putBytes writes a varint length followed by the bytes). The earlier + // version used putByteArray and reconstructed the length from buffer.size() - position(), which tied + // the format to "compressed bytes fill the rest of the record". Storing the length explicitly costs + // 1-4 bytes per record and lets future versions append fields (checksum, footer, etc.) without + // requiring a one-shot data migration on every external bucket. + blob.putBytes(compressedPayload); } else { blob.append(rawValueBytes); } @@ -1158,9 +1177,9 @@ public Object readExternalValue(final DatabaseInternal database, final int exter return deserializeValue(database, buffer, valueType, embeddedModifier); final int uncompressedSize = (int) buffer.getUnsignedNumber(); - final int compressedLen = buffer.size() - buffer.position(); - final byte[] compressedBytes = new byte[compressedLen]; - System.arraycopy(buffer.getContent(), buffer.position(), compressedBytes, 0, compressedLen); + // Length-prefixed payload (Binary.getBytes reads varint length + the bytes). Trailing space (if any) is + // ignored, leaving the format extensible to a future footer/checksum without a migration. + final byte[] compressedBytes = buffer.getBytes(); final byte[] decompressed = CompressionFactory.getLZ4().decompress(compressedBytes, uncompressedSize); return deserializeValue(database, new Binary(decompressed), valueType, embeddedModifier); } @@ -1169,11 +1188,21 @@ public Object readExternalValue(final DatabaseInternal database, final int exter * Reused by cascade-delete and the orphan-cleanup-on-update path inside {@link #serializeProperties}: scans * the OLD buffer for TYPE_EXTERNAL pointers, keyed by property name. *

    - * Do NOT add a {@code hasExternalProperties()} early-out here. The - * {@link #serializeProperties} caller invokes this during the EXTERNAL→inline migration (REBUILD TYPE after - * {@code setExternal(false)}), when the type's current schema reports zero EXTERNAL properties but the OLD - * record buffer still carries TYPE_EXTERNAL pointers that must be discovered so the paired blobs can be - * deleted as orphans. The schema flag and the buffer contents are decoupled; the buffer is ground truth. + * Gating check: {@code hasExternalBuckets()}, not {@code hasExternalProperties()}. + * The two are different and the distinction is load-bearing: + *

      + *
    • {@code hasExternalProperties()} reflects the CURRENT schema (count of properties currently flagged + * EXTERNAL). It flips to false the instant {@code setExternal(false)} commits, before any record has + * been re-serialised. Using it as the gate would skip orphan-cleanup during the EXTERNAL→inline + * migration window and leak external blobs.
      • + *
    • {@code hasExternalBuckets()} reflects the lifetime of the paired-bucket mapping + * ({@code externalBucketIdByPrimaryBucketId} non-empty). It stays true through the entire migration + * window: schema flag flipped → REBUILD TYPE re-serialises every record → orphan-cleanup empties the + * paired buckets → {@code reclaimEmptyExternalBuckets()} drops them and clears the map. Only at that + * point does it return false, and by then no record can carry a stale TYPE_EXTERNAL pointer.
      • + *
    + * Types that have never used the EXTERNAL feature have an empty map, so this fast-exit pays for itself on + * the (very common) update hot path of plain-vanilla types. */ public Map findExistingExternalRids(final Database database, final Document record) { final RID identity = record.getIdentity(); @@ -1181,6 +1210,8 @@ public Map findExistingExternalRids(final Database database, final return Collections.emptyMap(); if (!(record instanceof BaseDocument)) return Collections.emptyMap(); + if (!(record.getType() instanceof LocalDocumentType ldt) || !ldt.hasExternalBuckets()) + return Collections.emptyMap(); final Binary buf = ((BaseRecord) record).getBuffer(); if (buf == null) return Collections.emptyMap(); @@ -1223,10 +1254,14 @@ public Map findExistingExternalRids(final Database database, final } return result == null ? Collections.emptyMap() : result; } catch (Exception e) { + // Bump the JVM-cumulative counter so CHECK DATABASE surfaces the leak rate without waiting for the + // next paired-bucket scan. The orphan blob (if any) will eventually be caught by the orphan scan, + // but counting failures gives an early signal that something is corrupting record buffers. + externalRidScanFailures.incrementAndGet(); LogManager.instance().log(this, Level.WARNING, "Could not parse old buffer to recover external RIDs for record %s: %s. External records linked to this " - + "record may be orphaned in the paired bucket.", - e, identity, e.getMessage()); + + "record may be orphaned in the paired bucket. (cumulative scan failures since process start: %d)", + e, identity, e.getMessage(), externalRidScanFailures.get()); return Collections.emptyMap(); } finally { buf.position(savedPosition);