External properties

engine/src/main/antlr4/com/arcadedb/query/sql/grammar/SQLParser.g4

@@ -136,6 +136,9 @@ statement
     // Index Management
     | rebuildIndexStatement                          # rebuildIndexStmt
 
+    // Type Re-serialisation (e.g. relocate property values after toggling EXTERNAL flag)
+    | REBUILD TYPE rebuildTypeBody                   # rebuildTypeStmt
+
     // Transaction Statements
     | beginStatement                                 # beginStmt
     | commitStatement                                # commitStmt
@@ -868,6 +871,16 @@ rebuildIndexStatement
     : REBUILD INDEX (identifier | STAR) (WITH identifier EQ expression (COMMA identifier EQ expression)*)?
     ;
 
+/**
+ * REBUILD TYPE statement
+ * Re-serialises every record of a type so that schema changes (e.g. toggling a property's EXTERNAL flag) are applied
+ * to existing records on disk. POLYMORPHIC additionally walks subtypes.
+ * Syntax: REBUILD TYPE typeName [POLYMORPHIC]
+ */
+rebuildTypeBody
+    : typeName=identifier POLYMORPHIC? (WITH settingKey+=identifier EQ settingValue+=expression (COMMA settingKey+=identifier EQ settingValue+=expression)*)?
+    ;
+
 // ============================================================================
 // CONTROL FLOW STATEMENTS
 // ============================================================================

engine/src/main/java/com/arcadedb/GlobalConfiguration.java

@@ -210,6 +210,14 @@ public Object call(final Object value) {
   BUCKET_DEFAULT_PAGE_SIZE("arcadedb.bucketDefaultPageSize", SCOPE.DATABASE,
       "Default page size in bytes for buckets. Default is 64KB", Integer.class, 65_536),
 
+  EXTERNAL_PROPERTY_BUCKET_DEFAULT_PAGE_SIZE("arcadedb.externalPropertyBucketDefaultPageSize", SCOPE.DATABASE,
+      "Default page size in bytes for paired external-property buckets. They hold heavy property payloads (vector embeddings, large strings, embedded JSON) so the default is larger than for primary buckets to reduce multi-page chunking. Matches the LSM-index default (256KB)",
+      Integer.class, 262_144),
+
+  EXTERNAL_PROPERTY_BUCKET_PATH("arcadedb.externalPropertyBucketPath", SCOPE.DATABASE,
+      "Filesystem directory where new paired external-property buckets are created. If empty (default), external buckets sit alongside primary buckets in the database directory. Set to a path on cheaper/slower storage (HDD, network mount) to tier the heavy payloads away from the topology files. The directory must exist and be writable. Existing external buckets are not relocated when this changes.",
+      String.class, ""),
+
   BUCKET_REUSE_SPACE_MODE("arcadedb.bucketReuseSpaceMode", SCOPE.DATABASE,
       "How to reuse space in pages. 'high' = more space saved, but slower opening and update/delete time. 'medium' to still reuse space without the initial scan at opening time. 'low' for faster performance, but less space reused. Default is 'high'",
       String.class, "high", Set.of("low", "medium", "high")),

engine/src/main/java/com/arcadedb/compression/CompressionFactory.java

@@ -27,4 +27,13 @@ public class CompressionFactory {
   public static Compression getDefault() {
     return defaultImplementation;
   }
+
+  /**
+   * Returns the shared {@link LZ4Compression} instance. Exposes the concrete type for callers that need
+   * encoder variants beyond the {@link Compression} interface (e.g. {@link LZ4Compression#compressMax} for
+   * the LZ4 HC tier used by EXTERNAL property storage).
+   */
+  public static LZ4Compression getLZ4() {
+    return defaultImplementation;
+  }
 }

engine/src/main/java/com/arcadedb/compression/LZ4Compression.java

@@ -26,16 +26,24 @@
 import java.util.*;
 
 /**
- * Compression implementation that uses the popular LZ4 algorithm.
+ * Compression implementation that uses the popular LZ4 algorithm. Two compressors are exposed: the default
+ * {@code fast} encoder (block size optimised for throughput) and an on-demand {@code max} (LZ4 HC) encoder
+ * that spends more CPU on a deeper match search to produce a smaller output. Both share the same LZ4 byte
+ * format and the same decoder, so {@link #decompress(byte[], int)} works regardless of which encoder produced
+ * the input.
  */
 public class LZ4Compression implements Compression {
   private static final byte[]              EMPTY_BYTES  = new byte[0];
   private static final Binary              EMPTY_BINARY = new Binary(EMPTY_BYTES);
+  private final        LZ4Factory          factory;
   private final        LZ4Compressor       compressor;
   private final        LZ4FastDecompressor decompressor;
+  // High-compression encoder is built on first use: it allocates ~256KB of internal state and is rarely needed
+  // on the read path, so we don't want to pay for it when only the fast encoder is in use.
+  private volatile     LZ4Compressor       maxCompressor;
 
   public LZ4Compression() {
-    final LZ4Factory factory = LZ4Factory.fastestInstance();
+    this.factory = LZ4Factory.fastestInstance();
     this.compressor = factory.fastCompressor();
     this.decompressor = factory.fastDecompressor();
   }
@@ -50,6 +58,28 @@ public byte[] compress(final byte[] data) {
     return compressed;
   }
 
+  /**
+   * Maximum-compression encoder (LZ4 HC). Compresses 8-20x slower than {@link #compress(byte[])} but produces
+   * meaningfully smaller output (~10pp on text); decompression is the same speed. Use for write-once /
+   * read-many EXTERNAL payloads where bucket file size dominates.
+   */
+  public byte[] compressMax(final byte[] data) {
+    LZ4Compressor c = maxCompressor;
+    if (c == null) {
+      synchronized (this) {
+        c = maxCompressor;
+        if (c == null)
+          c = maxCompressor = factory.highCompressor();
+      }
+    }
+    final int maxCompressedLength = c.maxCompressedLength(data.length);
+    byte[] compressed = new byte[maxCompressedLength];
+    final int compressedLength = c.compress(data, 0, data.length, compressed, 0, maxCompressedLength);
+    if (compressedLength != maxCompressedLength)
+      compressed = Arrays.copyOf(compressed, compressedLength);
+    return compressed;
+  }
+
   @Override
   public Binary compress(final Binary data) {
     final int decompressedLength = data.size() - data.position();

engine/src/main/java/com/arcadedb/database/BaseDocument.java

@@ -39,7 +39,7 @@
 import java.util.List;
 import java.util.Map;
 
-public abstract class BaseDocument extends BaseRecord implements Document, Serializable, Externalizable {
+public abstract class BaseDocument extends BaseRecord implements Document, DocumentInternal, Serializable, Externalizable {
   protected final DocumentType type;
   protected       int          propertiesStartingPosition = 1;
 
@@ -157,6 +157,11 @@ public DocumentType getType() {
     return type;
   }
 
+  @Override
+  public int getPropertiesStartingPosition() {
+    return propertiesStartingPosition;
+  }
+
   public String getTypeName() {
     return type.getName();
   }

engine/src/main/java/com/arcadedb/database/DocumentInternal.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com)
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package com.arcadedb.database;
+
+import com.arcadedb.utility.ExcludeFromJacocoGeneratedReport;
+
+/**
+ * Internal contract for {@link BaseDocument} that exposes serialization-format details to engine modules
+ * (notably {@code com.arcadedb.serializer.BinarySerializer}) without leaking those details into the public
+ * {@link Document} API. Code that consumes ArcadeDB as a library should never cast to this interface; cross-version
+ * stability is not guaranteed.
+ *
+ * @author Luca Garulli (l.garulli@arcadedata.com)
+ */
+@ExcludeFromJacocoGeneratedReport
+public interface DocumentInternal {
+  /**
+   * Offset (in bytes) where the property header begins inside the record's binary buffer. Required by the
+   * serializer to scan for {@code TYPE_EXTERNAL} pointers without re-parsing the bucket-level record framing.
+   */
+  int getPropertiesStartingPosition();
+}

engine/src/main/java/com/arcadedb/database/ExternalValueRecord.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com)
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package com.arcadedb.database;
+
+import com.arcadedb.serializer.json.JSONObject;
+
+/**
+ * Opaque payload record for EXTERNAL property values. Buffer = [RECORD_TYPE][value type][value bytes].
+ *
+ * @author Luca Garulli (l.garulli@arcadedata.com)
+ */
+public class ExternalValueRecord extends BaseRecord implements RecordInternal {
+  // BaseRecord-subclass type tags (first byte of every record buffer):
+  //   Document            = 0  (Document.java)
+  //   Vertex              = 1  (Vertex.java)
+  //   Edge                = 2  (Edge.java)
+  //   EdgeSegment         = 3  (EdgeSegment.java / MutableEdgeSegment.java)
+  //   EmbeddedDocument    = 4  (EmbeddedDocument.java)
+  //   ExternalValueRecord = 5  (this class - paired-bucket payload for EXTERNAL property values)
+  //   LightEdge           = 6  (LightEdge.java)
+  // Add new values at the end and update this list.
+  public static final byte RECORD_TYPE = 5;
+
+  public ExternalValueRecord(final Database database, final RID rid, final Binary buffer) {
+    super(database, rid, buffer);
+  }
+
+  @Override
+  public byte getRecordType() {
+    return RECORD_TYPE;
+  }
+
+  @Override
+  public void setIdentity(final RID rid) {
+    this.rid = upgradeRID(rid);
+  }
+
+  @Override
+  public void unsetDirty() {
+    // NO-OP: BUFFER IS BUILT FRESH ON EACH SERIALIZE
+  }
+
+  public Binary getContent() {
+    return buffer;
+  }
+
+  @Override
+  public JSONObject toJSON(final boolean includeMetadata) {
+    // ExternalValueRecord is engine-internal infrastructure. It's only reachable through a primary record's
+    // TYPE_EXTERNAL pointer, and every public read path (Document.toJSON, REST handlers, Studio) walks through
+    // BinarySerializer.deserializeProperty which materialises the value inline. If a generic record dumper hits
+    // this method directly, it's a bug: failing loud surfaces it instead of returning silent empty data.
+    throw new UnsupportedOperationException(
+        "ExternalValueRecord is an internal payload of an EXTERNAL property and has no JSON representation; "
+            + "read the value through its owning Document instead of dereferencing the external RID directly");
+  }
+}

engine/src/main/java/com/arcadedb/database/LocalDatabase.java

@@ -962,8 +962,15 @@ public void createRecordNoLock(final Record record, final String bucketName, fin
       if (bucketName == null && record instanceof Document doc)
         bucket = (LocalBucket) doc.getType().getBucketIdByRecord(doc,
             DatabaseContext.INSTANCE.getContext(databasePath).asyncMode);
-      else
+      else {
         bucket = (LocalBucket) schema.getBucketByName(bucketName);
+        // Reject direct writes to internal buckets (e.g. paired external-property buckets). They are infrastructure
+        // for the serializer, not user data containers; allowing user DML to write here would corrupt the schema's
+        // accounting of which records are real records vs. payload blobs.
+        if (bucket.getPurpose() != LocalBucket.Purpose.PRIMARY)
+          throw new IllegalArgumentException(
+              "Bucket '" + bucketName + "' is internal (purpose=" + bucket.getPurpose() + ") and cannot be written to directly");
+      }
 
       ((RecordInternal) record).setIdentity(bucket.createRecord(record, discardRecordAfter));
 
@@ -1124,8 +1131,12 @@ public void deleteRecordNoLock(final Record record) {
     try {
       final LocalBucket bucket = schema.getBucketById(record.getIdentity().getBucketId());
 
-      if (record instanceof Document document)
+      if (record instanceof Document document) {
         indexer.deleteDocument(document);
+        // Cascade-delete EXTERNAL property values living in paired external buckets. This must run BEFORE the primary
+        // record is deleted, so the buffer is still readable. Both deletes ride the same transaction.
+        cascadeDeleteExternalValues(document);
+      }
 
       if (record instanceof Edge edge) {
         graphEngine.deleteEdge(edge);
@@ -1155,6 +1166,27 @@ public void deleteRecordNoLock(final Record record) {
     }
   }
 
+  /**
+   * Deletes all external-bucket records referenced by the given document's TYPE_EXTERNAL property pointers, in the same
+   * transaction as the primary delete. No-op if the type has no EXTERNAL properties or the document was not loaded with
+   * a buffer.
+   */
+  private void cascadeDeleteExternalValues(final Document document) {
+    if (!(document.getType() instanceof LocalDocumentType localType))
+      return;
+    if (!localType.hasExternalProperties())
+      return;
+    final Map<String, RID> externalRids = serializer.findExistingExternalRids(this, document);
+    for (final RID extRid : externalRids.values()) {
+      final LocalBucket externalBucket = schema.getBucketById(extRid.getBucketId(), false);
+      if (externalBucket != null) {
+        externalBucket.deleteRecord(extRid);
+        // Keep the external bucket's count consistent (mirrors the +1 in BinarySerializer.writeExternalPropertyValue).
+        getTransaction().updateBucketRecordDelta(externalBucket.getFileId(), -1);
+      }
+    }
+  }
+
   @Override
   public boolean isTransactionActive() {
     final Transaction tx = getTransactionIfExists();
@@ -1436,6 +1468,14 @@ public String getName() {
     return name;
   }
 
+  /** Override root + dbName subdir, or null if not configured. The subdir prevents collisions across databases. */
+  public String resolveExternalBucketPath() {
+    final String configured = configuration.getValueAsString(GlobalConfiguration.EXTERNAL_PROPERTY_BUCKET_PATH);
+    if (configured == null || configured.isEmpty())
+      return null;
+    return configured + File.separator + name;
+  }
+
   @Override
   public ComponentFile.MODE getMode() {
     return mode;
@@ -2028,7 +2068,7 @@ private void openInternal() {
       DatabaseContext.INSTANCE.init(this);
       setLockingEnabled(configuration.getValueAsBoolean(GlobalConfiguration.BACKUP_ENABLED));
 
-      fileManager = new FileManager(databasePath, mode, SUPPORTED_FILE_EXT);
+      fileManager = new FileManager(databasePath, mode, SUPPORTED_FILE_EXT, resolveExternalBucketPath());
       transactionManager = new TransactionManager(wrappedDatabaseInstance);
 
       open = true;

engine/src/main/java/com/arcadedb/database/MutableDocument.java

@@ -70,6 +70,15 @@ public boolean isDirty() {
     return dirty;
   }
 
+  /**
+   * Forces the next save to re-serialize this record even if no property has changed. Used by maintenance commands
+   * like `REBUILD TYPE` to apply schema layout changes (e.g. EXTERNAL flag toggles) to existing records on disk.
+   */
+  public MutableDocument markDirty() {
+    dirty = true;
+    return this;
+  }
+
   @Override
   public void setBuffer(final Binary buffer) {
     super.setBuffer(buffer);