From 181bf24c8bf2308048bd69a5f2ebc88c16cca3dd Mon Sep 17 00:00:00 2001 From: azim afroozeh <13484327+azimafroozeh@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:05:57 +0200 Subject: [PATCH 1/5] refactor: inline the schema contract in the catalog --- .github/workflows/ci.yml | 2 +- AGENTS.md | 4 +- .../schema-contract-inlining.breaking.md | 9 + crates/omnigraph-catalog/src/commit.rs | 18 +- crates/omnigraph-catalog/src/graph.rs | 30 +- crates/omnigraph-catalog/src/lib.rs | 342 ++- crates/omnigraph-catalog/src/migrations.rs | 264 ++- crates/omnigraph-catalog/src/publisher.rs | 170 +- crates/omnigraph-catalog/src/record.rs | 179 +- crates/omnigraph-catalog/src/retention.rs | 1 + crates/omnigraph-catalog/src/state.rs | 598 ++++- crates/omnigraph-catalog/src/tests.rs | 1194 +++++++++- .../omnigraph-cli/tests/cli_schema_config.rs | 2 +- .../tests/crossversion_upgrade.rs | 86 +- crates/omnigraph-cluster/src/tests.rs | 4 +- crates/omnigraph-cluster/tests/failpoints.rs | 31 +- crates/omnigraph-core/src/branch_control.rs | 69 +- crates/omnigraph-core/src/branch_names.rs | 6 - crates/omnigraph-core/src/instrumentation.rs | 9 +- .../instrumentation/small_manifest_reads.rs | 427 ++++ crates/omnigraph-core/src/lance_access.rs | 20 +- crates/omnigraph-dst/README.md | 32 +- crates/omnigraph-dst/cost_table.txt | 107 +- crates/omnigraph-dst/src/catalog.rs | 8 +- crates/omnigraph-dst/src/cost.rs | 4 +- crates/omnigraph-dst/src/harness.rs | 375 +++- crates/omnigraph-dst/src/write_census.rs | 6 - crates/omnigraph-dst/tests/scenarios.rs | 486 ++--- crates/omnigraph-gqt/src/dst_runner.rs | 2 +- crates/omnigraph-server/src/lib.rs | 20 +- .../omnigraph-server/tests/boot_settings.rs | 51 +- crates/omnigraph-server/tests/data_routes.rs | 35 +- .../omnigraph-server/tests/schema_routes.rs | 94 +- crates/omnigraph-server/tests/support/mod.rs | 6 - crates/omnigraph/src/db/catalog_tests.rs | 41 +- crates/omnigraph/src/db/graph_coordinator.rs | 323 ++- crates/omnigraph/src/db/mod.rs | 14 +- crates/omnigraph/src/db/omnigraph.rs | 1932 +++++++---------- .../omnigraph/src/db/omnigraph/collector.rs | 6 - crates/omnigraph/src/db/omnigraph/export.rs | 10 +- crates/omnigraph/src/db/omnigraph/optimize.rs | 5 - crates/omnigraph/src/db/omnigraph/repair.rs | 2 - .../src/db/omnigraph/schema_apply.rs | 414 +--- .../src/db/omnigraph/system_column_upgrade.rs | 157 +- .../omnigraph/src/db/omnigraph/table_ops.rs | 6 - crates/omnigraph/src/db/schema_state.rs | 866 ++------ crates/omnigraph/src/db/snapshot.rs | 26 + crates/omnigraph/src/db/upgrade.rs | 336 ++- .../omnigraph/src/db/upgrade/detached_only.rs | 28 +- .../src/db/upgrade/legacy_schema_files.rs | 217 ++ .../src/db/upgrade/legacy_sidecars.rs | 24 +- crates/omnigraph/src/db/upgrade/tests.rs | 1003 +++++++-- crates/omnigraph/src/db/write_queue.rs | 18 +- crates/omnigraph/src/exec/merge.rs | 30 +- crates/omnigraph/src/exec/mod.rs | 2 +- crates/omnigraph/src/exec/mutation.rs | 283 ++- crates/omnigraph/src/loader/mod.rs | 26 +- crates/omnigraph/src/runtime_cache.rs | 50 +- crates/omnigraph/src/seams/catalog.rs | 7 +- crates/omnigraph/src/storage_layer.rs | 15 + crates/omnigraph/src/table_store.rs | 92 + .../omnigraph/tests/detached_commit_matrix.rs | 29 +- crates/omnigraph/tests/end_to_end.rs | 10 +- crates/omnigraph/tests/failpoints.rs | 914 ++++---- crates/omnigraph/tests/helpers/failpoint.rs | 4 + crates/omnigraph/tests/helpers/mod.rs | 41 + .../omnigraph/tests/lance_surface_guards.rs | 37 + crates/omnigraph/tests/legacy_columns.rs | 49 +- crates/omnigraph/tests/lifecycle.rs | 656 +++--- .../omnigraph/tests/merge_projection_cache.rs | 23 +- crates/omnigraph/tests/point_in_time.rs | 29 + crates/omnigraph/tests/recovery.rs | 46 +- crates/omnigraph/tests/recovery_probe.rs | 20 +- crates/omnigraph/tests/schema_apply.rs | 76 +- .../omnigraph/tests/system_column_upgrade.rs | 134 +- crates/omnigraph/tests/warm_read_cost.rs | 287 ++- crates/omnigraph/tests/write_cost.rs | 169 +- crates/omnigraph/tests/writes.rs | 40 +- docs/dev/invariants.md | 44 +- docs/dev/lance.md | 6 +- docs/dev/recovery.md | 155 +- docs/dev/testing.md | 56 +- docs/dev/versioning.md | 139 +- docs/dev/writes.md | 103 +- docs/releases/v0.12.0.md | 120 +- .../0037-deterministic-simulation-harness.md | 17 +- docs/rfcs/2026-09-18-shared-schema-gate.md | 2 + docs/rfcs/2026-09-21-detached-only-tables.md | 2 + .../2026-09-30-schema-contract-in-manifest.md | 398 ++++ docs/rfcs/README.md | 1 + docs/user/operations/upgrade.md | 138 +- scripts/check-storage-upgrade-ci.py | 12 + 92 files changed, 9070 insertions(+), 5311 deletions(-) create mode 100644 changelog.d/schema-contract-inlining.breaking.md create mode 100644 crates/omnigraph-core/src/instrumentation/small_manifest_reads.rs create mode 100644 crates/omnigraph/src/db/upgrade/legacy_schema_files.rs create mode 100644 docs/rfcs/2026-09-30-schema-contract-in-manifest.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11c850618..fa10e1f43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1279,7 +1279,7 @@ jobs: done run_cell azure_write_detached_pin_survives_next_write -p omnigraph-engine --features failpoints --test failpoints run_cell azure_optimize_detached_pin_survives_next_write -p omnigraph-engine --features failpoints --test failpoints - run_cell azure_schema_apply_recovers_source_and_destination_after_partial_rename -p omnigraph-engine --features failpoints --test failpoints + run_cell azure_schema_apply_post_commit_failure_leaves_readable_contract -p omnigraph-engine --features failpoints --test failpoints run_cell azure_cluster_full_lifecycle_import_apply_serve_evolve_delete -p omnigraph-cluster --test s3_cluster run_cell server_opens_azure_graph_directly_and_serves_snapshot_and_read -p omnigraph-server --test s3 run_cell server_boots_azure_cluster_from_bare_storage_uri_and_serves_query -p omnigraph-server --test s3 diff --git a/AGENTS.md b/AGENTS.md index 25b81c16f..7e929b82b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -104,8 +104,8 @@ working memory: fresh rather than mixing old and new authority. 3. A mutation, load, schema apply, merge, or maintenance batch publishes once. 4. Pre-publication durable effects stay unreachable (detached commits, staged - files); what is published carries the identity that finishes it (pins, - the staged contract's publishing commit); ambiguity fails closed. + files). Table pins and the accepted schema contract are complete in one + manifest publication; ambiguity fails closed. 5. Stable schema identity survives supported renames, not drop/re-add. Never infer identity from names, paths, versions, field IDs, or branch refs. 6. Indexes, caches, topology, fragment layout, and compaction are derived diff --git a/changelog.d/schema-contract-inlining.breaking.md b/changelog.d/schema-contract-inlining.breaking.md new file mode 100644 index 000000000..2a12d6bc4 --- /dev/null +++ b/changelog.d/schema-contract-inlining.breaking.md @@ -0,0 +1,9 @@ +- Schema apply and system-column upgrades now publish the schema source, + compiled schema and identity atomically with their table references in the + catalog. Storage format 13 requires an explicit offline upgrade from supported + older formats; ordinary open refuses those layouts. Stop readers, writers and + maintenance, retain a whole-root backup, and follow the + [storage upgrade guide][schema-contract-inlining-upgrade]. Fresh graphs no + longer create separate root schema files. + +[schema-contract-inlining-upgrade]: ../docs/user/operations/upgrade.md diff --git a/crates/omnigraph-catalog/src/commit.rs b/crates/omnigraph-catalog/src/commit.rs index 4ca8a29b2..0b911b5e2 100644 --- a/crates/omnigraph-catalog/src/commit.rs +++ b/crates/omnigraph-catalog/src/commit.rs @@ -26,14 +26,14 @@ use std::sync::Arc; use arrow_array::{BooleanArray, RecordBatch}; use arrow_schema::Schema; -use datafusion::arrow::compute::filter_record_batch; +use datafusion::arrow::compute::{concat_batches, filter_record_batch}; use lance::Dataset; use lance::dataset::{CommitBuilder, InsertBuilder, WriteMode, WriteParams}; use lance_file::version::LanceFileVersion; use crate::error::{OmniError, Result}; use crate::migrations::{INTERNAL_MANIFEST_SCHEMA_VERSION, read_stamp, stamp_entry}; -use crate::publisher::map_lance_publish_error; +use crate::publisher::{PUBLISHED_ROWS_CACHE_BYTES, map_lance_publish_error}; use crate::record::{ StoredShape, compact_to_storage, flat_manifest_schema, flat_to_storage, manifest_storage_schema, written_shape, @@ -49,7 +49,7 @@ pub(crate) async fn overwrite( dataset: Dataset, pending: RecordBatch, live_rows: Vec, -) -> Result { +) -> Result<(Dataset, Option)> { let replaced: HashSet<&str> = string_column(&pending, "object_id")? .iter() .flatten() @@ -90,7 +90,17 @@ pub(crate) async fn overwrite( } } batches.push(to_storage(&pending)?); - commit_overwrite(dataset, batches).await + let retained_bytes = batches.iter().fold(0usize, |bytes, batch| { + bytes.saturating_add(batch.get_array_memory_size()) + }); + let retained = if shape == StoredShape::Packed && retained_bytes <= PUBLISHED_ROWS_CACHE_BYTES { + let empty = RecordBatch::new_empty(schema.clone()); + concat_batches(&schema, batches.iter().chain([&empty])).ok() + } else { + None + }; + let dataset = commit_overwrite(dataset, batches).await?; + Ok((dataset, retained)) } /// Commit `batches` as the whole stored row set at `dataset`'s version + 1 (the module doc: zero diff --git a/crates/omnigraph-catalog/src/graph.rs b/crates/omnigraph-catalog/src/graph.rs index 5a959a42e..f6730c604 100644 --- a/crates/omnigraph-catalog/src/graph.rs +++ b/crates/omnigraph-catalog/src/graph.rs @@ -19,8 +19,8 @@ use super::layout::{ use super::metadata::TableVersionMetadata; use super::migrations::{guard_stamp, stamp_entry, stamp_for_system_columns}; use super::state::{ - DatasetEntry, GraphLineageRow, ManifestState, entries_to_batch, graph_lineage_row_parts, - read_manifest_state, read_manifest_state_and_lineage, + DatasetEntry, GraphLineageRow, ManifestState, SchemaContractRow, entries_to_batch, + graph_lineage_row_parts, read_manifest_state, read_manifest_state_and_lineage, }; use super::{TableIdentity, table_path_for_identity}; use crate::record::{compact_to_storage, manifest_storage_schema}; @@ -113,13 +113,15 @@ impl From for OmniError { } } -/// Assembles the initial entries, genesis lineage, and internal-schema stamp, -/// and lands them all in the one `__manifest` Create commit. A returned error -/// from the Create call is acknowledgement-unknown; reading the state back is +/// Assembles the initial entries, genesis lineage, the `schema_contract` row +/// and the internal-schema stamp, and lands them all in the one `__manifest` +/// Create commit. A returned error from the Create call is +/// acknowledgement-unknown; reading the state back is /// `load_initial_manifest_state`, on the confirmed post-commit side. pub(crate) async fn init_manifest_graph( root_uri: &str, catalog: &Catalog, + contract: &SchemaContractRow, control_session: &Arc, attempt: &GenesisManifestAttempt, ) -> std::result::Result { @@ -131,7 +133,12 @@ pub(crate) async fn init_manifest_graph( // commit as every table entry and the internal-schema stamp. let genesis_lineage = graph_lineage_row_parts(attempt.lineage(), None)?; - let manifest_batch = entries_to_batch(&entries, &version_metadata, &genesis_lineage)?; + let manifest_batch = entries_to_batch( + &entries, + &version_metadata, + &genesis_lineage, + Some(contract), + )?; // The internal-schema stamp rides the Create write's schema metadata, so // the stamp is atomic with manifest birth: no crash window can leave // `__manifest` durable but unstamped. The Create commit is the manifest's @@ -236,6 +243,12 @@ pub(crate) async fn open_exact_genesis_manifest( "main graph head does not match this initialization attempt", )); } + if known_state.schema_contract.is_none() { + return Err(genesis_probe_mismatch( + root_uri, + "genesis carries no schema_contract row", + )); + } Ok((dataset, known_state, lineage_rows)) } @@ -308,7 +321,7 @@ pub(crate) async fn snapshot_state_at( root_uri: &str, branch: Option<&str>, version: u64, -) -> Result { +) -> Result<(Dataset, ManifestState)> { let control_session = crate::lance_access::control_session(); let dataset = open_manifest_dataset_with_session( root_uri.trim_end_matches('/'), @@ -320,7 +333,8 @@ pub(crate) async fn snapshot_state_at( .checkout_version(version) .await .map_err(OmniError::storage)?; - read_manifest_state(&dataset).await + let state = read_manifest_state(&dataset).await?; + Ok((dataset, state)) } async fn build_initial_entries( diff --git a/crates/omnigraph-catalog/src/lib.rs b/crates/omnigraph-catalog/src/lib.rs index 0d7feb6ca..0f6a24d31 100644 --- a/crates/omnigraph-catalog/src/lib.rs +++ b/crates/omnigraph-catalog/src/lib.rs @@ -78,10 +78,11 @@ use publisher::{GraphNamespacePublisher, ManifestBatchPublisher, PublishOutcome} pub use state::DatasetEntry; #[cfg(test)] use state::string_column; -pub use state::{GraphLineageRow, read_graph_lineage}; +pub use state::{GraphLineageRow, SchemaContractHead, SchemaContractRow, read_graph_lineage}; use state::{ ManifestState, ProjectionAccumulator, fold_projection_delta, read_manifest_projection, - read_manifest_state, read_object_identities_at_offsets, + read_manifest_projection_with_contract, read_manifest_state, read_object_identities_at_offsets, + read_schema_contract_row, }; /// The maximum supported storage-format stamp, the one this binary writes; the @@ -107,6 +108,14 @@ pub const OBJECT_TYPE_GRAPH_HEAD: &str = "graph_head"; /// `object_id` prefix of the head rows — one constant for row minting, row /// decode, and the incremental fold's dead-row classification. pub(crate) const GRAPH_HEAD_OBJECT_ID_PREFIX: &str = "graph_head:"; +/// The one live schema-contract row of main's `__manifest` (stamp 13): the +/// `.pg` source and the IR text in the `schema_source` and `schema_ir` +/// columns, the IR hash and the identity version and domain in `metadata` +/// ([`SchemaContractHead`]). A schema apply's publish replaces it in the same +/// commit as the table rows and the head, the way `graph_head:` moves. +pub const OBJECT_TYPE_SCHEMA_CONTRACT: &str = "schema_contract"; +/// The fixed `object_id` of the `schema_contract` row. +pub const SCHEMA_CONTRACT_OBJECT_ID: &str = "schema_contract"; /// Stable head-key segment for the main branch in `graph_head:` rows. /// `table_branch`/`manifest_branch` encode main as null, but `object_id` must be @@ -149,6 +158,12 @@ pub async fn read_supported_internal_schema_version(root_uri: &str) -> Result, + /// The `schema_contract` row's head from this SAME pinned manifest + /// version (see `ManifestState::schema_contract`). + schema_contract: Option, + manifest_dataset: Option, + captured_contract: Option>, /// Logical graph branch used to capture this snapshot, including historical reads. graph_branch: Option, /// Native ref of the live branch coordinator; named writes record it as fork owner. - /// `None` on main, time-travel reads, and directly built test snapshots. + /// `None` on main and directly built test snapshots. native_branch: Option, /// Per-graph read caches (shared `Session` + held-handle cache), injected by /// `Omnigraph::resolved_target` for live Branch reads so dataset opens reuse @@ -464,6 +484,30 @@ impl Snapshot { self.graph_heads.get(branch_key).map(String::as_str) } + /// The schema contract's head (IR hash, identity version and domain) from + /// this snapshot's own pinned manifest version. `None` only for a version + /// written before the contract lived in `__manifest` (a time-travel read + /// of a stamp-12 version) or a directly built test snapshot. + pub fn schema_contract(&self) -> Option<&SchemaContractHead> { + self.schema_contract.as_ref() + } + + /// Whether both snapshots captured the same immutable manifest image. + pub fn same_manifest_image(&self, other: &Self) -> bool { + self.root_uri == other.root_uri + && self.version == other.version + && self.native_branch == other.native_branch + && self.schema_contract == other.schema_contract + && match (&self.manifest_dataset, &other.manifest_dataset) { + (Some(left), Some(right)) => publisher::manifest_image_matches( + left.manifest(), + left.manifest_location(), + right, + ), + _ => false, + } + } + /// Bind the current accepted catalog's aliases onto a historical snapshot /// by immutable table identity for query execution. /// @@ -863,6 +907,7 @@ impl CollectorBranch { ); snapshot.graph_branch = self.branch.clone(); snapshot.native_branch = self.dataset.manifest().branch.clone(); + snapshot.manifest_dataset = Some(self.dataset.clone()); Ok(snapshot) } } @@ -886,6 +931,9 @@ pub enum ManifestChange { RegisterTable(TableRegistration), RenameTable(TableRename), Tombstone(TableTombstone), + /// Replace the live `schema_contract` row with this contract, in the same + /// publish as the batch's table rows and head. At most one per batch. + SchemaContract(SchemaContractRow), } /// One table-version authority assertion supplied to a publish attempt. @@ -1012,6 +1060,7 @@ async fn fragment_deletion_offsets( pub struct ManifestCoordinator { root_uri: String, dataset: Dataset, + captured_contract: Option>, pub known_state: ManifestState, active_branch: Option, /// The native Lance ref `active_branch` resolved to at open time @@ -1059,6 +1108,7 @@ impl ManifestCoordinator { Self { root_uri: self.root_uri.clone(), dataset: self.dataset.clone(), + captured_contract: self.captured_contract.clone(), known_state: self.known_state.clone(), active_branch: self.active_branch.clone(), native_branch: self.native_branch.clone(), @@ -1098,6 +1148,7 @@ impl ManifestCoordinator { root_uri: root_uri.trim_end_matches('/').to_string(), dataset, known_state, + captured_contract: None, active_branch, native_branch, branch_identifier, @@ -1137,6 +1188,9 @@ impl ManifestCoordinator { .map(|entry| (entry.type_key.clone(), entry)) .collect(), graph_heads: state.graph_heads, + schema_contract: state.schema_contract, + manifest_dataset: None, + captured_contract: None, graph_branch: None, native_branch: None, read_caches: None, @@ -1160,7 +1214,8 @@ impl ManifestCoordinator { /// Test-only composition of the two init halves; production init goes /// through them separately so the commit point is a caller-visible - /// boundary (issue #495). + /// boundary (issue #495). The genesis `schema_contract` row carries + /// [`SchemaContractRow::for_test_catalog`]. #[cfg(any(test, feature = "test-util"))] pub async fn init_with_lineage( root_uri: &str, @@ -1168,28 +1223,115 @@ impl ManifestCoordinator { control_session: &Arc, ) -> Result<(Self, Vec)> { let attempt = GenesisManifestAttempt::mint(catalog.system_columns)?; - let dataset = Self::init_commit(root_uri, catalog, control_session, &attempt).await?; + let contract = SchemaContractRow::for_test_catalog(catalog)?; + let dataset = + Self::init_commit(root_uri, catalog, &contract, control_session, &attempt).await?; Self::finish_init(root_uri, dataset).await } /// Commit half of manifest init; ends at the `__manifest` Create commit - /// (assembled in `init_manifest_graph`). + /// (assembled in `init_manifest_graph`), which carries `contract` as the + /// genesis `schema_contract` row. #[doc(hidden)] pub async fn init_commit( root_uri: &str, catalog: &Catalog, + contract: &SchemaContractRow, control_session: &Arc, attempt: &GenesisManifestAttempt, ) -> std::result::Result { init_manifest_graph( root_uri.trim_end_matches('/'), catalog, + contract, control_session, attempt, ) .await } + /// The `schema_contract` row at this coordinator's pinned manifest + /// version, texts included: captured content or one filtered scan, + /// checked against the head folded from the same version. + pub async fn read_schema_contract(&self) -> Result { + if let Some(row) = &self.captured_contract + && self.known_state.schema_contract.as_ref() == Some(&row.head) + { + return Ok((**row).clone()); + } + if let Some(batch) = self.publisher.cached_rows(&self.dataset) { + return state::schema_contract_from_batch( + &self.dataset, + &batch, + self.known_state.schema_contract.as_ref(), + ); + } + read_schema_contract_row(&self.dataset, self.known_state.schema_contract.as_ref()).await + } + + /// Read contract content from the exact native branch and version already + /// captured by a live snapshot. A retained retired ref returns its original + /// contract; a missing captured ref fails without selecting a replacement. + pub async fn read_schema_contract_for_snapshot( + root_uri: &str, + snapshot: &Snapshot, + ) -> Result { + if root_uri.trim_end_matches('/') != snapshot.root_uri { + return Err(OmniError::manifest( + "schema-contract snapshot belongs to another root", + )); + } + if let Some(row) = &snapshot.captured_contract + && snapshot.schema_contract() == Some(&row.head) + { + return Ok((**row).clone()); + } + if let Some(dataset) = &snapshot.manifest_dataset { + return read_schema_contract_row(dataset, snapshot.schema_contract()).await; + } + if snapshot.graph_branch().is_some() && snapshot.native_branch().is_none() { + return Err(OmniError::manifest( + "named schema-contract snapshot lacks native branch provenance", + )); + } + let control_session = crate::lance_access::control_session(); + let dataset = open_manifest_dataset_native_with_session( + root_uri.trim_end_matches('/'), + snapshot.native_branch(), + &control_session, + ) + .await?; + let dataset = dataset + .checkout_version(snapshot.graph_manifest_version()) + .await + .map_err(OmniError::storage)?; + read_schema_contract_row(&dataset, snapshot.schema_contract()).await + } + + /// The `schema_contract` row of `branch` (main when `None`) at `version`, + /// texts included, read from a fresh open without a coordinator. The row + /// is checked against the head folded from a state read of that same + /// version, so the read is two scans. + pub async fn read_schema_contract_at( + root_uri: &str, + branch: Option<&str>, + version: u64, + ) -> Result { + let control_session = crate::lance_access::control_session(); + let dataset = open_manifest_dataset_with_session( + root_uri.trim_end_matches('/'), + branch, + &control_session, + ) + .await?; + let dataset = dataset + .checkout_version(version) + .await + .map_err(OmniError::storage)?; + let state = read_manifest_state(&dataset).await?; + read_schema_contract_row(&dataset, state.schema_contract.as_ref()).await + } + /// Probe an acknowledgement-unknown manifest Create and accept only the /// exact immutable genesis receipt minted by this initialization attempt. /// A transport/read/mismatch error remains indeterminate to the caller; @@ -1298,19 +1440,125 @@ impl ManifestCoordinator { // `refresh_with_lineage`: this body now builds the projection // accumulators and is awaited inside merge authority capture and // publication. - Box::pin(Self::open_with_lineage_inner( + let (coordinator, lineage, _) = Box::pin(Self::open_with_lineage_inner( root_uri, branch, control_session, + false, + )) + .await?; + Ok((coordinator, lineage)) + } + + /// Capture state, lineage and contract from one pinned image. Content errors are + /// returned separately so callers can refresh before admitting that image. + pub async fn open_with_lineage_and_contract( + root_uri: &str, + branch: Option<&str>, + control_session: &Arc, + ) -> Result<(Self, Vec, Result)> { + let (coordinator, lineage, contract) = Box::pin(Self::open_with_lineage_inner( + root_uri, + branch, + control_session, + true, + )) + .await?; + Ok(( + coordinator, + lineage, + contract.expect("contract projection returns its complete schema row"), + )) + } + + /// Check the held image against ordinary serving bounds, independently of conversion admission. + pub fn validate_serving_format(&self) -> Result<()> { + migrations::refuse_if_stamp_unsupported(migrations::guard_stamp(&self.dataset)?) + } + + /// Validate ordinary serving before the schema queue without discarding the opened dataset. + pub async fn prepare_open_with_contract( + root_uri: &str, + control_session: &Arc, + ) -> Result { + let root = root_uri.trim_end_matches('/'); + let dataset = open_manifest_dataset_with_session(root, None, control_session).await?; + migrations::refuse_if_stamp_unsupported(migrations::guard_stamp(&dataset)?)?; + Ok(PreparedManifestOpen { + root_uri: root.to_string(), + dataset, + }) + } + + /// Consume the admitted main image; retry a failed old scan only after proving it changed. + pub async fn open_prepared_with_lineage_and_contract( + root_uri: &str, + prepared: PreparedManifestOpen, + ) -> Result<(Self, Vec, Result)> { + let root = root_uri.trim_end_matches('/'); + if prepared.root_uri != root { + return Err(OmniError::manifest_internal( + "prepared manifest belongs to a different graph root", + )); + } + let dataset = prepared.dataset; + let control_session = dataset.session(); + let captured_manifest = dataset.manifest().clone(); + let captured_location = dataset.manifest_location().clone(); + let captured = Box::pin(Self::project_opened_with_lineage( + root, + None, + dataset, + lance::dataset::refs::BranchIdentifier::main(), + None, + true, + )) + .await; + let (coordinator, lineage, contract) = match captured { + Ok(captured) => captured, + Err(error) => { + let current = + open_manifest_dataset_with_session(root, None, &control_session).await?; + let location = current.manifest_location(); + if captured_location.path == location.path + && captured_location.version == location.version + && captured_location.naming_scheme == location.naming_scheme + && captured_location.e_tag == location.e_tag + && captured_location.size == location.size + && &captured_manifest == current.manifest() + && captured_manifest.schema.metadata == current.schema().metadata + { + return Err(error); + } + migrations::refuse_if_stamp_unsupported(migrations::guard_stamp(¤t)?)?; + Box::pin(Self::project_opened_with_lineage( + root, + None, + current, + lance::dataset::refs::BranchIdentifier::main(), + None, + true, + )) + .await? + } + }; + Ok(( + coordinator, + lineage, + contract.expect("contract projection returns its complete schema row"), )) - .await } async fn open_with_lineage_inner( root_uri: &str, branch: Option<&str>, control_session: &Arc, - ) -> Result<(Self, Vec)> { + read_contract: bool, + ) -> Result<( + Self, + Vec, + Option>, + )> { let root = root_uri.trim_end_matches('/'); let branch = branch.filter(|branch| *branch != "main"); // Retain the fold accumulators alongside the state (the incremental merge-authority projection): the @@ -1318,7 +1566,39 @@ impl ManifestCoordinator { // deltas into, instead of a sunk cost repeated per refresh. let (dataset, branch_identifier, native_branch) = open_manifest_branch_with_identifier(root, branch, control_session).await?; - let (known_state, projection, lineage_rows) = read_manifest_projection(&dataset).await?; + Self::project_opened_with_lineage( + root, + branch, + dataset, + branch_identifier, + native_branch, + read_contract, + ) + .await + } + + async fn project_opened_with_lineage( + root: &str, + branch: Option<&str>, + dataset: Dataset, + branch_identifier: lance::dataset::refs::BranchIdentifier, + native_branch: Option, + read_contract: bool, + ) -> Result<( + Self, + Vec, + Option>, + )> { + let (known_state, projection, lineage_rows, contract) = if read_contract + || migrations::read_stamp(&dataset) == Some(INTERNAL_MANIFEST_SCHEMA_VERSION) + { + let (state, projection, lineage, contract) = + read_manifest_projection_with_contract(&dataset).await?; + (state, projection, lineage, Some(contract)) + } else { + let (state, projection, lineage) = read_manifest_projection(&dataset).await?; + (state, projection, lineage, None) + }; let projection_version = dataset.version().version; let mut coordinator = Self::from_parts_with_default_publisher( root, @@ -1329,7 +1609,11 @@ impl ManifestCoordinator { branch_identifier, ); coordinator.projection = Some((projection_version, projection)); - Ok((coordinator, lineage_rows)) + coordinator.captured_contract = contract + .as_ref() + .and_then(|row| row.as_ref().ok()) + .map(|row| Arc::new(row.clone())); + Ok((coordinator, lineage_rows, contract)) } pub async fn snapshot_at( @@ -1338,8 +1622,10 @@ impl ManifestCoordinator { version: u64, ) -> Result { let root = root_uri.trim_end_matches('/'); - let mut snapshot = - Self::snapshot_from_state(root, snapshot_state_at(root, branch, version).await?); + let (dataset, state) = snapshot_state_at(root, branch, version).await?; + let mut snapshot = Self::snapshot_from_state(root, state); + snapshot.native_branch = dataset.manifest().branch.clone(); + snapshot.manifest_dataset = Some(dataset); snapshot.graph_branch = branch .filter(|branch| *branch != "main") .map(str::to_string); @@ -1381,6 +1667,8 @@ impl ManifestCoordinator { let mut snapshot = Self::snapshot_from_state(&self.root_uri, self.known_state.clone()); snapshot.native_branch = self.native_branch.clone(); snapshot.graph_branch = self.active_branch.clone(); + snapshot.manifest_dataset = Some(self.dataset.clone()); + snapshot.captured_contract = self.captured_contract.clone(); snapshot } @@ -1420,9 +1708,11 @@ impl ManifestCoordinator { &control_session, ) .await?; - let (known_state, projection, lineage_rows) = read_manifest_projection(&dataset).await?; + let (known_state, projection, lineage_rows, contract) = + read_manifest_projection_with_contract(&dataset).await?; let projection_version = dataset.version().version; self.dataset = dataset; + self.captured_contract = contract.ok().map(Arc::new); self.known_state = known_state; self.branch_identifier = branch_identifier; self.native_branch = native_branch; @@ -1535,6 +1825,10 @@ impl ManifestCoordinator { return Ok(None); } for (object_type, object_id) in identities { + if object_type == OBJECT_TYPE_SCHEMA_CONTRACT { + folded.remove_schema_contract(); + continue; + } if object_type != OBJECT_TYPE_GRAPH_HEAD { // A deleted row this fold does not model (retention, // repair, future machinery): the append assumption is @@ -1573,6 +1867,7 @@ impl ManifestCoordinator { crate::instrumentation::record_projection_incremental_refresh(); let projection_version = new_dataset.version().version; self.dataset = new_dataset; + self.captured_contract = None; self.known_state = known_state; self.projection = Some((projection_version, folded)); Ok(Some(lineage_rows)) @@ -1605,7 +1900,7 @@ impl ManifestCoordinator { &control_session, ) .await?; - let known_state = read_manifest_state(&dataset).await?; + let (known_state, contract) = state::read_manifest_state_with_contract(&dataset).await?; let branch_key = self .active_branch .as_deref() @@ -1622,6 +1917,7 @@ impl ManifestCoordinator { self.known_state = known_state; self.branch_identifier = branch_identifier; self.native_branch = native_branch; + self.captured_contract = contract.ok().map(Arc::new); // Same staleness rule as the post-publish fold: this refresh advances // `dataset` without folding the projection accumulators, so they must // not survive it. @@ -1748,6 +2044,18 @@ impl ManifestCoordinator { // RFC-013 PR2 #1b: the publisher folded the new visible state in-memory // (byte-identical to a re-scan via the shared `assemble_manifest_state`), // so adopt it directly instead of an O(fragments) `read_manifest_state`. + self.captured_contract = self + .publisher + .cached_rows(&dataset) + .and_then(|batch| { + state::schema_contract_from_batch( + &dataset, + &batch, + known_state.schema_contract.as_ref(), + ) + .ok() + }) + .map(Arc::new); self.dataset = dataset; self.known_state = known_state; // Until the caller has adopted the committed lineage, refresh must @@ -1922,10 +2230,6 @@ impl ManifestCoordinator { crate::branch_control::retire_branch_recoverably(&ds, &native, expected_identifier).await } - pub async fn schema_apply_locked(&self) -> Result { - crate::branch_control::schema_apply_locked(&self.dataset).await - } - /// Logical graph branches, `main` first. Each live native ref maps to /// exactly one logical name; a duplicate incarnation fails loudly. pub async fn list_graph_branches(&self) -> Result> { diff --git a/crates/omnigraph-catalog/src/migrations.rs b/crates/omnigraph-catalog/src/migrations.rs index 0bb458038..e1312dcd7 100644 --- a/crates/omnigraph-catalog/src/migrations.rs +++ b/crates/omnigraph-catalog/src/migrations.rs @@ -22,7 +22,7 @@ //! vintages (`id`/`src`/`dst` and `__id`/`__src`/`__dst`) live under that //! stamp; the vintage is read from the schema IR, never from the stamp. //! Normal open refuses an active storage-upgrade intent. The explicit -//! offline upgrade entry point converts supported v6 to v10 graphs to v11 +//! offline upgrade entry point converts supported v6 to v12 graphs to v13 //! before serving; its v10 step opens the graph through an engine handle, //! admitted at the stamp it converts from by [`admit_conversion_source`], //! before it fences. Retained v6 snapshots use the legacy decoder after root @@ -42,23 +42,22 @@ use lance::dataset::refs::BranchIdentifier; use lance::dataset::transaction::{Operation, Transaction, UpdateMap}; use omnigraph_compiler::{SYSTEM_COLUMNS_LEGACY, SYSTEM_COLUMNS_V3, SystemColumns}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; -#[cfg(any(test, feature = "test-util"))] use std::sync::Arc; -#[cfg(any(test, feature = "test-util"))] use arrow_array::RecordBatch; -#[cfg(any(test, feature = "test-util"))] -use arrow_schema::Schema; +use arrow_schema::{Schema, SchemaRef}; -#[cfg(any(test, feature = "test-util"))] +#[cfg(any(test, feature = "test-util", feature = "dst"))] use crate::commit::commit_overwrite; use crate::error::{OmniError, Result}; -use crate::record::RECORD_COLUMN; -#[cfg(any(test, feature = "test-util"))] -use crate::record::{expand_from_storage, flat_manifest_schema, flat_to_storage}; -#[cfg(any(test, feature = "test-util"))] -use crate::state::read_publish_scan; +use crate::record::{ + RECORD_COLUMN, compact_to_storage, flat_manifest_schema, manifest_storage_schema, +}; +#[cfg(any(test, feature = "test-util", feature = "dst"))] +use crate::record::{expand_from_storage, flat_to_storage}; +use crate::state::{SchemaContractHead, SchemaContractRow, entries_to_batch, read_publish_scan}; /// The internal schema version this binary writes for a new-vintage graph and /// the ceiling it serves. @@ -105,20 +104,36 @@ use crate::state::read_publish_scan; /// reached; rows at or below it keep the v10 twin rule, rows above it open /// their detached version directly. A v10 binary would read a v11 pin's /// target as reclaimed history, so the stamp refuses it before any open. +/// - v12 — the `__manifest` rows are stored as `object_id`, `object_type` and +/// one packed `record` struct (`record.rs`); `base_objects` is dropped. +/// - v13 — the schema contract is one live `schema_contract` row in main's +/// `__manifest`: the `.pg` source and the IR text in the `schema_source` +/// and `schema_ir` columns beside `record`, the IR hash and the identity +/// version and domain in the row's `metadata`. A publish that applies a +/// schema replaces the row in the same commit as the table rows and the +/// head. A v12 binary would open a v13 graph without reading the row, so +/// the stamp refuses it before any open. /// -/// v1–v10 graphs are not served by this binary (see `MIN_SUPPORTED`); the +/// v1–v12 graphs are not served by this binary (see `MIN_SUPPORTED`); the /// history is kept for provenance and to document what each stamp value meant. -pub const INTERNAL_MANIFEST_SCHEMA_VERSION: u32 = 12; +pub const INTERNAL_MANIFEST_SCHEMA_VERSION: u32 = 13; /// The first stamp whose `__manifest` rows are stored as `object_id`, /// `object_type` and one packed `record` struct (`record.rs`). A stamp-11 -/// manifest keeps its flat columns until its next publish rewrites it. +/// manifest keeps its flat columns until a publish rewrites it. pub const PACKED_RECORD_STAMP: u32 = 12; -/// The oldest main-manifest stamp accepted by normal open: v11, the target of -/// every registered upgrade route. Explicit conversion and retained-snapshot -/// decoding do not lower this gate. -pub const MIN_SUPPORTED_INTERNAL_SCHEMA_VERSION: u32 = 11; +/// The last stamp stored flat, and the oldest stamp a publish rewrites into +/// the packed shape at [`INTERNAL_MANIFEST_SCHEMA_VERSION`]; a publish over a +/// lower stamp keeps the flat shape and the source stamp, the state a storage +/// conversion reads while it runs. +pub const LAST_FLAT_STAMP: u32 = PACKED_RECORD_STAMP - 1; + +/// The oldest main-manifest stamp accepted by normal open: v13, the stamp +/// every graph is born with since the schema contract moved into +/// `__manifest`. Explicit conversion and retained-snapshot decoding do not +/// lower this gate. +pub const MIN_SUPPORTED_INTERNAL_SCHEMA_VERSION: u32 = 13; /// The stamp a fresh graph of the given system column vintage is born with: /// CURRENT for both `id`/`src`/`dst` and `__id`/`__src`/`__dst` since v10. @@ -147,9 +162,10 @@ pub fn stamp_for_system_columns(system_columns: SystemColumns) -> Result { /// v1 ≤ 0.3.1, v2 0.4.1–0.6.1, v3 0.6.2–0.7.2, v4 0.8.x, v5 was /// unreleased (final source commit pinned below), v6 is 0.9.x–0.10.x, /// v7 an unreleased development build, v8 and v9 are the two 0.11.x -/// system column vintages, and v10 is the 0.11.x line after RFC 0067 (the +/// system column vintages, v10 is the 0.11.x line after RFC 0067 (the /// stamp of every graph the 0.11.0 crate version wrote with detached table -/// commits). The fallback keeps this map total. +/// commits), and v11 and v12 are the 0.11.x development line before the +/// schema contract moved into `__manifest`. The fallback keeps this map total. pub fn release_for_internal_schema_version(stamp: u32) -> &'static str { match stamp { 1 => "0.3.1 or earlier", @@ -164,6 +180,8 @@ pub fn release_for_internal_schema_version(stamp: u32) -> &'static str { 8 => "0.11.x (legacy system column spellings)", 9 => "0.11.x", 10 => "0.11.x (detached table commits)", + 11 => "0.11.x (detached-only tables)", + 12 => "0.11.x (packed catalog record)", _ => "an unrecognized older release", } } @@ -323,7 +341,8 @@ pub fn is_served_stamp(stamp: u32) -> bool { /// /// - `stamp > CURRENT`: the graph was written by a newer binary — upgrade omnigraph. /// - `stamp < MIN_SUPPORTED`: the graph was made by an older omnigraph whose -/// storage format this binary does not read — rebuild it via export/import. +/// storage format this binary does not read — use a registered offline +/// upgrade route when available, otherwise rebuild via export/import. /// - `MIN_SUPPORTED..=CURRENT` is served as-is and never migrated on open. pub fn refuse_if_stamp_unsupported(stamp: u32) -> Result<()> { if stamp > INTERNAL_MANIFEST_SCHEMA_VERSION { @@ -334,8 +353,8 @@ pub fn refuse_if_stamp_unsupported(stamp: u32) -> Result<()> { ))); } if stamp < MIN_SUPPORTED_INTERNAL_SCHEMA_VERSION { - let explicit_upgrade = if matches!(stamp, 6..=10) { - " A registered in-place route is also available: stop all writers and maintenance, retain a verified backup, and run `omnigraph upgrade --check` before execution; the route keeps the graph's branches and system column spellings." + let explicit_upgrade = if matches!(stamp, 6..=12) { + " A registered in-place route is also available: stop all readers, writers and maintenance, retain a verified backup, and run `omnigraph upgrade --check` before execution; the route keeps the graph's branches and system column spellings." } else { "" }; @@ -378,6 +397,36 @@ pub struct UpgradeIntent { pub target_format: u32, pub graph_identity: String, pub branches: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema_contract: Option, +} + +/// Protocol 5 binds semantic identity and exact text across branch conversions. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct UpgradeSchemaContract { + pub identity: SchemaContractHead, + pub source_sha256: String, + pub ir_sha256: String, +} + +impl UpgradeSchemaContract { + pub fn from_row(row: &SchemaContractRow) -> Self { + Self { + identity: row.head.clone(), + source_sha256: format!("{:x}", Sha256::digest(row.source.as_bytes())), + ir_sha256: format!("{:x}", Sha256::digest(row.ir.as_bytes())), + } + } + + pub fn validate_row(&self, row: &SchemaContractRow) -> Result<()> { + if *self != Self::from_row(row) { + return Err(invalid( + "upgrade schema contract identity or exact text changed", + )); + } + Ok(()) + } } #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -392,6 +441,10 @@ pub fn invalid(message: impl Into) -> OmniError { OmniError::manifest(message) } +fn valid_sha256(hash: &str) -> bool { + hash.len() == 64 && hash.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + pub fn intent_from(dataset: &Dataset) -> Result> { let Some(json) = dataset.schema().metadata.get(UPGRADE_PENDING_KEY) else { return Ok(None); @@ -403,11 +456,28 @@ pub fn intent_from(dataset: &Dataset) -> Result> { } let intent: UpgradeIntent = serde_json::from_str(json) .map_err(|e| invalid(format!("unrecognized upgrade ownership: {e}")))?; + let contract_valid = match &intent.schema_contract { + Some(contract) => { + intent.protocol == 5 + && contract.identity.schema_identity_domain == intent.graph_identity + && contract.identity.schema_identity_version != 0 + && contract + .identity + .schema_ir_hash + .strip_prefix("sha256:") + .is_some_and(valid_sha256) + && [&contract.source_sha256, &contract.ir_sha256] + .iter() + .all(|hash| valid_sha256(hash)) + } + None => intent.protocol != 5, + }; let mut names = HashSet::new(); if !matches!( (intent.protocol, intent.source_format, intent.target_format), - (1, 6, 7) | (2, 7, 8) | (3, 8, 10) | (3, 9, 10) | (4, 10, 11) - ) || intent.attempt.parse::().is_err() + (1, 6, 7) | (2, 7, 8) | (3, 8, 10) | (3, 9, 10) | (4, 10, 11) | (5, 11, 13) | (5, 12, 13) + ) || !contract_valid + || intent.attempt.parse::().is_err() || intent.graph_identity.is_empty() || intent.branches.is_empty() || intent.branches.len() > MAX_BRANCHES @@ -430,7 +500,7 @@ pub fn recovery_guidance(dataset: &Dataset) -> String { Ok(Some(intent)) => format!( "storage upgrade recovery required: stop all writers and maintenance and rerun `omnigraph upgrade --to-format {}` with this upgrade-capable executable; preserve the existing attempt.{}", intent.target_format, - if intent.target_format == 7 { " After completion, run `omnigraph upgrade --to-format 8` before serving with this executable." } else { "" }, + if intent.target_format < INTERNAL_MANIFEST_SCHEMA_VERSION { " After completion, run `omnigraph upgrade ` to reach the served format before opening with this executable." } else { "" }, ), _ => "storage upgrade ownership is unknown: preserve the graph and original upgrade options; use this upgrade-capable executable for read-only `omnigraph upgrade --check` diagnostics before recovery".into(), } @@ -496,12 +566,14 @@ pub fn branch_completed( } pub async fn historical_source(snapshot: Dataset, source_format: u32) -> Result { - if source_format != 7 || !snapshot.schema().metadata.contains_key(UPGRADE_PENDING_KEY) { + if !matches!(source_format, 7 | 13) + || !snapshot.schema().metadata.contains_key(UPGRADE_PENDING_KEY) + { return Ok(snapshot); } let intent = intent_from(&snapshot)?.ok_or_else(|| invalid("historical upgrade intent disappeared"))?; - if intent.protocol != 1 || snapshot.manifest().branch.is_some() { + if !matches!(intent.protocol, 1 | 5) || snapshot.manifest().branch.is_some() { return Err(invalid("unsupported historical upgrade ownership")); } let main = intent @@ -526,9 +598,13 @@ pub async fn historical_source(snapshot: Dataset, source_format: u32) -> Result< .get(UPGRADE_PENDING_KEY) .ok_or_else(|| invalid("historical upgrade intent disappeared"))? .clone(); - let operation = Transaction::new(main.version, fence_operation(json, 7), None); + let operation = Transaction::new( + main.version, + fence_operation(json, intent.target_format), + None, + ); if snapshot.version().version != expected - || read_stamp(&snapshot) != Some(7) + || read_stamp(&snapshot) != Some(intent.target_format) || transaction.read_version != main.version || lance_table::format::pb::Transaction::from(&transaction).operation != lance_table::format::pb::Transaction::from(&operation).operation @@ -545,8 +621,10 @@ pub async fn historical_source(snapshot: Dataset, source_format: u32) -> Result< .checkout_version(main.version) .await .map_err(OmniError::storage)?; - if read_stamp(&source) != Some(6) { - return Err(invalid("historical upgrade fence source is not v6")); + if read_stamp(&source) != Some(intent.source_format) { + return Err(invalid( + "historical upgrade fence source format differs from the intent", + )); } Ok(source) } @@ -587,9 +665,10 @@ pub async fn set_stamp_for_test(dataset: &mut Dataset, version: u32) -> Result<( set_stamp(dataset, version).await } -/// Test-only: a stored batch of either shape laid out flat (the stamps 5 to 11 -/// shape) under `metadata`. -#[cfg(any(test, feature = "test-util"))] +/// Test and DST fixture: a stored batch of either shape laid out flat (the stamps 5 to 11 +/// shape) under `metadata`, less any `schema_contract` row, which no flat +/// manifest ever carried. +#[cfg(any(test, feature = "test-util", feature = "dst"))] pub fn flat_batch_for_test( batch: &RecordBatch, metadata: HashMap, @@ -599,6 +678,13 @@ pub fn flat_batch_for_test( } else { batch.clone() }; + let keep = arrow_array::BooleanArray::from_iter( + crate::state::string_column(&logical, "object_type")? + .iter() + .map(|object_type| Some(object_type != Some(crate::OBJECT_TYPE_SCHEMA_CONTRACT))), + ); + let logical = datafusion::arrow::compute::filter_record_batch(&logical, &keep) + .map_err(OmniError::arrow_internal)?; let schema = Arc::new(Schema::new_with_metadata( flat_manifest_schema().fields().clone(), metadata, @@ -606,10 +692,91 @@ pub fn flat_batch_for_test( flat_to_storage(&logical, &schema) } -/// Test-only: rewrite the live rows flat and stamp `version` in the same commit, +/// Validate the exact source layout without interpreting a restamped current row as legacy. +pub fn validate_schema_contract_source(dataset: &Dataset) -> Result<()> { + let expected = match read_stamp(dataset) { + Some(11) => flat_manifest_schema(), + Some(12) => packed_v12_schema(dataset.schema().metadata.clone())?, + _ => { + return Err(invalid( + "schema-contract conversion requires format 11 or 12", + )); + } + }; + let actual: Schema = dataset.schema().into(); + if actual.fields() != expected.fields() { + return Err(invalid( + "source manifest columns do not match its registered format", + )); + } + Ok(()) +} + +/// Read a converted branch without admitting its pending root for ordinary serving. +pub async fn read_upgrade_schema_contract(dataset: &Dataset) -> Result { + let state = crate::state::read_manifest_state(dataset).await?; + crate::state::read_schema_contract_row(dataset, state.schema_contract.as_ref()).await +} + +fn packed_v12_schema(metadata: HashMap) -> Result { + let current = manifest_storage_schema(metadata.clone())?; + Ok(Arc::new(Schema::new_with_metadata( + current.fields().iter().take(3).cloned().collect::>(), + metadata, + ))) +} + +/// Preserve every live source row while adding the contract in the current packed shape. +pub async fn schema_contract_conversion_batches( + source: &Dataset, + metadata: HashMap, + contract: &SchemaContractRow, +) -> Result<(SchemaRef, Vec)> { + validate_schema_contract_source(source)?; + let scan = read_publish_scan(source).await?; + if scan.schema_contract.is_some() { + return Err(invalid( + "legacy source unexpectedly carries a schema contract row", + )); + } + let schema = manifest_storage_schema(metadata)?; + let mut rows = scan.live_rows; + rows.push(entries_to_batch(&[], &HashMap::new(), &[], Some(contract))?); + let batches = rows + .iter() + .map(|batch| compact_to_storage(batch, &schema)) + .collect::>>()?; + Ok((schema, batches)) +} + +/// Test-only: remove the contract and content columns, preserving the actual v12 packed layout. +#[cfg(any(test, feature = "test-util"))] +pub async fn restamp_packed_v12_for_test(dataset: &mut Dataset) -> Result<()> { + let live_rows = read_publish_scan(dataset).await?.live_rows; + let mut metadata = dataset.schema().metadata.clone(); + metadata.insert(INTERNAL_SCHEMA_VERSION_KEY.to_string(), "12".to_string()); + let schema = packed_v12_schema(metadata.clone())?; + let current_schema = manifest_storage_schema(metadata.clone())?; + let batches = live_rows + .iter() + .map(|batch| { + let flat = flat_batch_for_test(batch, metadata.clone())?; + let packed = compact_to_storage(&flat, ¤t_schema)?; + RecordBatch::try_new( + schema.clone(), + packed.columns().iter().take(3).cloned().collect(), + ) + .map_err(OmniError::arrow_internal) + }) + .collect::>>()?; + *dataset = commit_overwrite(dataset.clone(), batches).await?; + Ok(()) +} + +/// Test and DST fixture: rewrite the live rows flat and stamp `version` in one commit, /// the `__manifest` a pre-stamp-12 binary left behind; a restamp alone leaves /// packed rows no flat reader can project. -#[cfg(any(test, feature = "test-util"))] +#[cfg(any(test, feature = "test-util", feature = "dst"))] pub async fn restamp_flat_for_test(dataset: &mut Dataset, version: u32) -> Result<()> { let live_rows = read_publish_scan(dataset).await?.live_rows; let mut metadata = dataset.schema().metadata.clone(); @@ -662,16 +829,17 @@ mod tests { /// the floor or above the ceiling. #[test] fn unsupported_guard_accepts_exactly_the_supported_range() { - assert_eq!(stamp_for_system_columns(SYSTEM_COLUMNS_LEGACY).unwrap(), 12); - assert_eq!(stamp_for_system_columns(SYSTEM_COLUMNS_V3).unwrap(), 12); + assert_eq!(stamp_for_system_columns(SYSTEM_COLUMNS_LEGACY).unwrap(), 13); + assert_eq!(stamp_for_system_columns(SYSTEM_COLUMNS_V3).unwrap(), 13); assert!(stamp_for_system_columns(omnigraph_compiler::SYSTEM_COLUMNS_META).is_err()); assert_eq!( ( MIN_SUPPORTED_INTERNAL_SCHEMA_VERSION, INTERNAL_MANIFEST_SCHEMA_VERSION ), - (11, 12) + (13, 13) ); + assert_eq!((LAST_FLAT_STAMP, PACKED_RECORD_STAMP), (11, 12)); for stamp in MIN_SUPPORTED_INTERNAL_SCHEMA_VERSION..=INTERNAL_MANIFEST_SCHEMA_VERSION { assert!( refuse_if_stamp_unsupported(stamp).is_ok(), @@ -681,12 +849,12 @@ mod tests { let below = refuse_if_stamp_unsupported(MIN_SUPPORTED_INTERNAL_SCHEMA_VERSION - 1) .expect_err("a sub-floor stamp must be refused") .to_string(); - assert!(below.contains("reads only v11 to v12"), "got: {below}"); + assert!(below.contains("reads only v13 to v13"), "got: {below}"); assert!( - below.contains("0.11.x (detached table commits)"), + below.contains("0.11.x (packed catalog record)"), "got: {below}" ); - assert!(below.contains("omnigraph upgrade"), "got: {below}"); + assert!(below.contains("omnigraph export"), "got: {below}"); let legacy = refuse_if_stamp_unsupported(7) .expect_err("a v7 stamp must be refused") .to_string(); @@ -695,12 +863,18 @@ mod tests { "got: {legacy}" ); assert!(legacy.contains("omnigraph upgrade"), "got: {legacy}"); + for stamp in 6..=12 { + let refusal = refuse_if_stamp_unsupported(stamp) + .expect_err("conversion requires explicit offline upgrade") + .to_string(); + assert!(refusal.contains("omnigraph upgrade"), "v{stamp}: {refusal}"); + } let future_stamp = INTERNAL_MANIFEST_SCHEMA_VERSION + 1; let future = refuse_if_stamp_unsupported(future_stamp) .expect_err("the first unsupported future stamp must be refused") .to_string(); - assert!(future.contains("internal schema v13"), "got: {future}"); - assert!(future.contains("reads only v11 to v12"), "got: {future}"); + assert!(future.contains("internal schema v14"), "got: {future}"); + assert!(future.contains("reads only v13 to v13"), "got: {future}"); assert!(future.contains("upgrade omnigraph"), "got: {future}"); } diff --git a/crates/omnigraph-catalog/src/publisher.rs b/crates/omnigraph-catalog/src/publisher.rs index dfcd2e144..547d55713 100644 --- a/crates/omnigraph-catalog/src/publisher.rs +++ b/crates/omnigraph-catalog/src/publisher.rs @@ -16,14 +16,18 @@ //! version publication against a managed namespace manifest. use std::collections::HashMap; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; +use arrow_array::RecordBatch; use async_trait::async_trait; use lance::Dataset; use lance::Error as LanceError; +use lance_core::deepsize::DeepSizeOf; use lance_namespace::NamespaceError; #[cfg(any(test, feature = "test-util"))] use lance_namespace::models::CreateTableVersionRequest; +use lance_table::format::Manifest; +use lance_table::io::commit::ManifestLocation; use crate::error::{OmniError, Result}; @@ -34,16 +38,18 @@ use super::layout::{ open_manifest_dataset_with_session, table_object_id, tombstone_object_id, version_object_id, }; use super::metadata::{TableVersionMetadata, parse_namespace_version_request}; -use super::migrations::guard_stamp; +use super::migrations::{INTERNAL_MANIFEST_SCHEMA_VERSION, guard_stamp, read_stamp}; use super::state::{ - GraphLineageRow, GraphLineageRowPart, ManifestState, ProjectionAccumulator, - assemble_manifest_projection, graph_head_object_id, graph_lineage_row_parts, head_lineage_row, - manifest_rows_batch, read_manifest_state, read_publish_scan, + GraphLineageRow, GraphLineageRowPart, ManifestState, ProjectionAccumulator, SchemaContractHead, + assemble_manifest_projection, decode_publish_batch, graph_head_object_id, + graph_lineage_row_parts, head_lineage_row, manifest_rows_batch, read_manifest_state, + read_publish_scan, schema_contract_metadata_json, }; use super::{ DatasetEntry, ExpectedTableVersions, MAIN_BRANCH_HEAD_KEY, ManifestChange, ManifestIncarnation, - NativeRefPin, OBJECT_TYPE_TABLE, OBJECT_TYPE_TABLE_TOMBSTONE, OBJECT_TYPE_TABLE_VERSION, - TableIdentity, TableRegistration, TableRename, TableTombstone, WinnerRef, + NativeRefPin, OBJECT_TYPE_SCHEMA_CONTRACT, OBJECT_TYPE_TABLE, OBJECT_TYPE_TABLE_TOMBSTONE, + OBJECT_TYPE_TABLE_VERSION, SCHEMA_CONTRACT_OBJECT_ID, TableIdentity, TableRegistration, + TableRename, TableTombstone, WinnerRef, }; use crate::seams::{contention, decide_seam, fail}; @@ -147,6 +153,11 @@ pub struct PublishOutcome { #[async_trait] pub trait ManifestBatchPublisher: Send + Sync { + /// Acknowledged rows only when their immutable image matches the capture. + fn cached_rows(&self, _dataset: &Dataset) -> Option { + None + } + /// Publish without a graph-head precondition. Every production writer /// calls `publish_with_precondition`; only the publisher's own tests use /// this shorthand. @@ -179,6 +190,63 @@ pub struct GraphNamespacePublisher { root_uri: String, branch: Option, control_session: Arc, + published_rows: Mutex>, +} + +pub(crate) const PUBLISHED_ROWS_CACHE_BYTES: usize = 8 * 1024 * 1024; + +#[derive(Debug)] +struct PublishedRows { + manifest: Manifest, + location: ManifestLocation, + batch: RecordBatch, +} + +impl PublishedRows { + fn new(dataset: &Dataset, batch: RecordBatch) -> Option { + let manifest = dataset.manifest(); + let location = dataset.manifest_location(); + let bytes = batch + .get_array_memory_size() + .saturating_add(manifest.deep_size_of()) + .saturating_add(location.path.as_ref().len()) + .saturating_add(location.e_tag.as_ref().map_or(0, String::capacity)) + .saturating_add(std::mem::size_of::()); + if bytes > PUBLISHED_ROWS_CACHE_BYTES + || read_stamp(dataset) != Some(INTERNAL_MANIFEST_SCHEMA_VERSION) + || manifest + .schema + .fields_pre_order() + .any(|field| field.dictionary.is_some()) + { + return None; + } + Some(Self { + manifest: manifest.clone(), + location: location.clone(), + batch, + }) + } + + fn matches(&self, dataset: &Dataset) -> bool { + manifest_image_matches(&self.manifest, &self.location, dataset) + } +} + +pub(crate) fn manifest_image_matches( + manifest: &Manifest, + location: &ManifestLocation, + dataset: &Dataset, +) -> bool { + let other = dataset.manifest_location(); + location.path == other.path + && location.version == other.version + && location.naming_scheme == other.naming_scheme + && !matches!((&location.e_tag, &other.e_tag), (Some(a), Some(b)) if a != b) + && !matches!((location.size, other.size), (Some(a), Some(b)) if a != b) + && manifest == dataset.manifest() + // Lance Schema equality compares fields but omits schema metadata. + && manifest.schema.metadata == dataset.schema().metadata } #[derive(Debug)] @@ -192,6 +260,8 @@ struct PendingVersionRow { table_version: Option, table_branch: Option, row_count: Option, + /// `(source, ir)` texts, present on the `schema_contract` row alone. + schema_content: Option<(String, String)>, } /// Everything one CAS attempt needs out of a single `__manifest` scan @@ -208,6 +278,9 @@ struct LoadedPublishState { existing_tombstones: HashMap<(TableIdentity, u64), u64>, lineage_rows: Vec, graph_heads: HashMap, + /// The live `schema_contract` row's head, carried forward by the fold + /// unless the batch replaces the row. + schema_contract: Option, /// The scanned rows, the input of the copy-on-write publish. live_rows: Vec, } @@ -288,6 +361,7 @@ impl GraphNamespacePublisher { .filter(|branch| *branch != "main") .map(ToOwned::to_owned), control_session, + published_rows: Mutex::new(None), } } @@ -307,13 +381,11 @@ impl GraphNamespacePublisher { contention(&PUBLISH_LOAD_STATE)?; let dataset = self.dataset().await?; guard_stamp(&dataset)?; - // ONE `__manifest` scan for everything the publish needs: table - // locations, version entries, tombstones, `graph_commit` lineage rows - // for parent resolution, AND exact `graph_head` rows for OCC (RFC-013 - // P2 / RFC-022). Extraction rides this pass instead of a second - // `read_graph_lineage` scan; the per-attempt re-read is preserved because - // `load_publish_state` runs once per CAS attempt. - let scan = read_publish_scan(&dataset).await?; + let cached = self.cached_rows(&dataset); + let scan = match cached { + Some(batch) => decode_publish_batch(&dataset, batch).await?, + None => read_publish_scan(&dataset).await?, + }; let existing_versions = scan .version_entries .iter() @@ -327,6 +399,7 @@ impl GraphNamespacePublisher { existing_tombstones, lineage_rows: scan.lineage_rows, graph_heads: scan.graph_heads, + schema_contract: scan.schema_contract, live_rows: scan.live_rows, }) } @@ -396,6 +469,7 @@ impl GraphNamespacePublisher { table_version: None, table_branch: None, row_count: None, + schema_content: None, }); } ManifestChange::RenameTable(TableRename { @@ -459,15 +533,39 @@ impl GraphNamespacePublisher { table_version: None, table_branch: None, row_count: None, + schema_content: None, }); } - ManifestChange::Update(_) | ManifestChange::Tombstone(_) => {} + ManifestChange::Update(_) + | ManifestChange::Tombstone(_) + | ManifestChange::SchemaContract(_) => {} } } + let mut schema_contract_claimed = false; for change in changes { match change { ManifestChange::RegisterTable(_) | ManifestChange::RenameTable(_) => {} + ManifestChange::SchemaContract(contract) => { + if std::mem::replace(&mut schema_contract_claimed, true) { + return Err(OmniError::manifest( + "the schema contract is replaced twice in one publish request" + .to_string(), + )); + } + rows.push(PendingVersionRow { + object_id: SCHEMA_CONTRACT_OBJECT_ID.to_string(), + object_type: OBJECT_TYPE_SCHEMA_CONTRACT.to_string(), + location: None, + metadata: Some(schema_contract_metadata_json(&contract.head)?), + table_key: String::new(), + identity: None, + table_version: None, + table_branch: None, + row_count: None, + schema_content: Some((contract.source.clone(), contract.ir.clone())), + }); + } ManifestChange::Update(update) => { update.identity.validate()?; let request = update.to_create_table_version_request(); @@ -520,6 +618,7 @@ impl GraphNamespacePublisher { table_version: Some(table_version), table_branch, row_count: Some(row_count), + schema_content: None, }); } ManifestChange::Tombstone(TableTombstone { @@ -572,6 +671,7 @@ impl GraphNamespacePublisher { table_version: Some(*tombstone_version), table_branch: None, row_count: None, + schema_content: None, }); } } @@ -634,6 +734,8 @@ impl GraphNamespacePublisher { let mut table_versions: Vec> = Vec::with_capacity(rows.len()); let mut table_branches = Vec::with_capacity(rows.len()); let mut row_counts: Vec> = Vec::with_capacity(rows.len()); + let mut schema_sources = Vec::with_capacity(rows.len()); + let mut schema_irs = Vec::with_capacity(rows.len()); for row in rows { object_ids.push(row.object_id); @@ -645,6 +747,9 @@ impl GraphNamespacePublisher { table_versions.push(row.table_version); table_branches.push(row.table_branch); row_counts.push(row.row_count); + let (source, ir) = row.schema_content.unzip(); + schema_sources.push(source); + schema_irs.push(ir); } manifest_rows_batch( @@ -657,6 +762,8 @@ impl GraphNamespacePublisher { table_versions, table_branches, row_counts, + schema_sources, + schema_irs, ) } @@ -965,7 +1072,7 @@ impl GraphNamespacePublisher { dataset: Dataset, rows: Vec, live_rows: Vec, - ) -> Result { + ) -> Result<(Dataset, Option)> { fail(&PUBLISH_PRE_MERGE)?; let pending = Self::pending_rows_to_batch(rows)?; let new_dataset = commit::overwrite(dataset, pending, live_rows).await?; @@ -1027,6 +1134,7 @@ fn lineage_part_to_pending(part: GraphLineageRowPart) -> PendingVersionRow { table_version: part.table_version, table_branch: part.table_branch, row_count: None, + schema_content: None, } } @@ -1070,6 +1178,15 @@ fn publish_outcome_in_doubt( #[async_trait] impl ManifestBatchPublisher for GraphNamespacePublisher { + fn cached_rows(&self, dataset: &Dataset) -> Option { + self.published_rows + .lock() + .expect("published rows cache lock poisoned") + .as_ref() + .filter(|rows| rows.matches(dataset)) + .map(|rows| rows.batch.clone()) + } + async fn publish_with_precondition( &self, changes: &[ManifestChange], @@ -1116,13 +1233,10 @@ impl ManifestBatchPublisher for GraphNamespacePublisher { existing_tombstones, lineage_rows, graph_heads, + schema_contract, live_rows, } = loaded; - // Exact logical authority is checked on EVERY attempt from this - // attempt's single manifest scan. In particular, a CAS retry after - // another writer creates or advances `graph_head:` fails - // here instead of transparently re-parenting the prepared intent. self.check_publish_precondition(&dataset, &graph_heads, precondition) .await?; let base_incarnation = self.checked_base_incarnation(&dataset, precondition); @@ -1167,6 +1281,7 @@ impl ManifestBatchPublisher for GraphNamespacePublisher { existing_versions.into_values().collect(), existing_tombstones.into_keys(), graph_heads, + schema_contract, )?; return Ok(PublishOutcome { dataset, @@ -1198,6 +1313,13 @@ impl ManifestBatchPublisher for GraphNamespacePublisher { intent.graph_commit_id.clone(), ); } + let fold_schema_contract = changes + .iter() + .find_map(|change| match change { + ManifestChange::SchemaContract(contract) => Some(contract.head.clone()), + _ => None, + }) + .or(schema_contract); // Validate the complete post-batch fold before the physical merge. // In particular, alias collisions must fail without advancing // `__manifest`; discovering one after `merge_rows` would be an @@ -1208,10 +1330,11 @@ impl ManifestBatchPublisher for GraphNamespacePublisher { fold_entries, fold_tombstones, fold_graph_heads, + fold_schema_contract, )?; match self.merge_rows(dataset.clone(), rows, live_rows).await { - Ok(new_dataset) => { + Ok((new_dataset, retained)) => { if new_dataset.version().version != new_manifest_version { return Err(OmniError::manifest_internal(format!( "manifest commit at version {} is durable but its rows carry manifest \ @@ -1220,6 +1343,11 @@ impl ManifestBatchPublisher for GraphNamespacePublisher { new_manifest_version ))); } + *self + .published_rows + .lock() + .expect("published rows cache lock poisoned") = + retained.and_then(|batch| PublishedRows::new(&new_dataset, batch)); known_state.version = new_dataset.version().version; return Ok(PublishOutcome { dataset: new_dataset, diff --git a/crates/omnigraph-catalog/src/record.rs b/crates/omnigraph-catalog/src/record.rs index 4586e0a89..21359a3b0 100644 --- a/crates/omnigraph-catalog/src/record.rs +++ b/crates/omnigraph-catalog/src/record.rs @@ -1,4 +1,4 @@ -//! Stored shape (internal-schema stamp 12). A row is three columns: `object_id`, +//! Stored shape (internal-schema stamps 12 and 13). A row is `object_id`, //! `object_type`, and `record`, a Lance packed struct holding every other field //! of [`manifest_schema`] row-major in one column, so a scan reads one column's //! pages for the record instead of one per field. Lance's packed struct has no @@ -6,10 +6,13 @@ //! child), so every child is declared non-null and the ninth child, `present`, //! carries one bit per field that is null in the logical row; the child under //! a set bit holds the filler (`""` or `0`), which a reader checks and refuses -//! anything else. [`compact_to_storage`] and [`expand_from_storage`] are the -//! two boundaries; everything above them keeps the logical [`manifest_schema`]. -//! A scan projects `record` whole: Lance 11 cannot project a fixed-width child -//! of a packed struct on its own. +//! anything else. Stamp 13 adds the two schema-contract content columns of +//! [`SCHEMA_CONTENT_COLUMNS`] beside `record`, nullable and null on every row +//! but the `schema_contract` row, so a scan that does not project them never +//! reads the contract's bytes. [`compact_to_storage`] and +//! [`expand_from_storage`] are the two boundaries; everything above them keeps +//! the logical [`manifest_schema`]. A scan projects `record` whole: Lance 11 +//! cannot project a fixed-width child of a packed struct on its own. use std::collections::HashMap; use std::sync::Arc; @@ -24,23 +27,25 @@ use arrow_schema::{DataType, Field, Fields, Schema, SchemaRef}; use crate::error::{OmniError, Result}; use crate::migrations::{ - INTERNAL_MANIFEST_SCHEMA_VERSION, MIN_SUPPORTED_INTERNAL_SCHEMA_VERSION, PACKED_RECORD_STAMP, + INTERNAL_MANIFEST_SCHEMA_VERSION, LAST_FLAT_STAMP, MIN_SUPPORTED_INTERNAL_SCHEMA_VERSION, + PACKED_RECORD_STAMP, }; /// The logical `__manifest` row schema, what every reader and writer above the /// storage boundary sees. `object_id` keeps Lance's unenforced primary-key /// marker: every existing `__manifest` carries it and Lance treats the marker /// as fixed once set, so an overwrite must present it too. The stored shape is -/// [`manifest_storage_schema`] (stamp 12: the fields after `object_type` -/// packed into one `record` struct) or [`flat_manifest_schema`] (stamps 5 to -/// 11). The publish CAS itself is the version commit of `commit::overwrite`. +/// [`manifest_storage_schema`] (stamps 12 and 13: the fields after +/// `object_type` packed into one `record` struct, the content columns beside +/// it) or [`flat_manifest_schema`] (stamps 5 to 11). The publish CAS itself is +/// the version commit of `commit::overwrite`. pub fn manifest_schema() -> SchemaRef { let object_id_metadata: HashMap = [("lance-schema:unenforced-primary-key", "true")] .into_iter() .map(|(k, v)| (k.to_string(), v.to_string())) .collect(); - Arc::new(Schema::new(vec![ + let mut fields = vec![ Field::new("object_id", DataType::Utf8, false).with_metadata(object_id_metadata), Field::new("object_type", DataType::Utf8, false), Field::new("location", DataType::Utf8, true), @@ -51,17 +56,37 @@ pub fn manifest_schema() -> SchemaRef { Field::new("table_version", DataType::UInt64, true), Field::new("table_branch", DataType::Utf8, true), Field::new("row_count", DataType::UInt64, true), - ])) + ]; + fields.extend(SCHEMA_CONTENT_COLUMNS.map(schema_content_field)); + Arc::new(Schema::new(fields)) +} + +/// The two top-level columns holding the schema contract's texts on the +/// `schema_contract` row: the `.pg` source and the IR JSON, byte-exact. Null on +/// every other row; absent from the flat shape and from stamp 12. +pub const SCHEMA_CONTENT_COLUMNS: [&str; 2] = ["schema_source", "schema_ir"]; + +fn schema_content_field(name: &str) -> Field { + Field::new(name, DataType::LargeUtf8, true) +} + +fn is_schema_content_column(name: &str) -> bool { + SCHEMA_CONTENT_COLUMNS.contains(&name) } /// The stored shape of stamps 5 to 11: the logical columns as flat columns, -/// plus `base_objects`, a list column no path ever wrote a value into or read -/// (dropped from the logical schema at stamp 12). Readers of a stamp-11 -/// manifest project the logical columns out of it; the upgrade command -/// validates a conversion source against it. +/// less the stamp-13 content columns, plus `base_objects`, a list column no +/// path ever wrote a value into or read (dropped from the logical schema at +/// stamp 12). Readers of a stamp-11 manifest project the logical columns out +/// of it; the upgrade command validates a conversion source against it. pub fn flat_manifest_schema() -> SchemaRef { let logical = manifest_schema(); - let mut fields: Vec> = logical.fields().iter().cloned().collect(); + let mut fields: Vec> = logical + .fields() + .iter() + .filter(|field| !is_schema_content_column(field.name())) + .cloned() + .collect(); fields.insert( 4, Arc::new(Field::new( @@ -73,6 +98,17 @@ pub fn flat_manifest_schema() -> SchemaRef { Arc::new(Schema::new(fields)) } +/// The logical columns a flat-shape scan projects: every field of +/// [`flat_manifest_schema`] but `base_objects`. +pub(crate) fn flat_projection() -> Vec { + flat_manifest_schema() + .fields() + .iter() + .map(|field| field.name().clone()) + .filter(|name| name != "base_objects") + .collect() +} + /// The stored column holding every record field of a row as one packed struct. pub(crate) const RECORD_COLUMN: &str = "record"; /// The bitmask child of `record`: bit `i` set means field `i` of @@ -110,29 +146,30 @@ pub(crate) fn stored_shape(stamp: Option) -> StoredShape { } } -/// The shape a publish writes at `stamp`: packed for every served stamp (so a -/// stamp-11 manifest converts on its next publish), flat below the served floor. +/// The shape a publish writes at `stamp`: packed from [`LAST_FLAT_STAMP`] up +/// (a stamp-11 manifest converts on its next publish, the storage upgrade's +/// rewrite included), flat below it. pub(crate) fn written_shape(stamp: Option) -> StoredShape { - if stamp.is_some_and(|stamp| stamp >= MIN_SUPPORTED_INTERNAL_SCHEMA_VERSION) { + if stamp.is_some_and(|stamp| stamp >= LAST_FLAT_STAMP) { StoredShape::Packed } else { StoredShape::Flat } } -/// A flat write lands only below the served floor and every served stamp reads packed from -/// [`PACKED_RECORD_STAMP`] up, so [`stored_shape`] and [`written_shape`] agree only while the -/// packed stamp sits inside the served range. +/// Every served stamp reads packed, a publish converts exactly the last flat +/// stamp, and a flat write lands only below the served floor. const _: () = assert!( - MIN_SUPPORTED_INTERNAL_SCHEMA_VERSION <= PACKED_RECORD_STAMP - && PACKED_RECORD_STAMP <= INTERNAL_MANIFEST_SCHEMA_VERSION + LAST_FLAT_STAMP + 1 == PACKED_RECORD_STAMP + && PACKED_RECORD_STAMP <= MIN_SUPPORTED_INTERNAL_SCHEMA_VERSION + && MIN_SUPPORTED_INTERNAL_SCHEMA_VERSION <= INTERNAL_MANIFEST_SCHEMA_VERSION ); -/// The stored schema at stamp 12: `object_id` (with Lance's unenforced -/// primary-key marker), `object_type`, and the packed `record` struct; the -/// dataset's schema metadata (where the stamp lives) is `metadata`. The only -/// error is a `RECORD_FIELDS` name missing from `manifest_schema`, two -/// constants of this file, so the `Result` is never `Err`; it stays for the -/// crate's no-panic rule. +/// The stored schema at stamp 13: `object_id` (with Lance's unenforced +/// primary-key marker), `object_type`, the packed `record` struct, and the two +/// nullable content columns; the dataset's schema metadata (where the stamp +/// lives) is `metadata`. The only error is a `RECORD_FIELDS` name missing +/// from `manifest_schema`, two constants of this file, so the `Result` is +/// never `Err`; it stays for the crate's no-panic rule. pub(crate) fn manifest_storage_schema(metadata: HashMap) -> Result { let logical = manifest_schema(); let mut children = Vec::with_capacity(RECORD_FIELDS.len() + 1); @@ -154,10 +191,42 @@ pub(crate) fn manifest_storage_schema(metadata: HashMap) -> Resu PACKED_STRUCT_KEY.to_string(), "true".to_string(), )])); - Ok(Arc::new(Schema::new_with_metadata( - vec![logical.field(0).clone(), logical.field(1).clone(), record], - metadata, - ))) + let mut fields = vec![logical.field(0).clone(), logical.field(1).clone(), record]; + fields.extend(SCHEMA_CONTENT_COLUMNS.map(schema_content_field)); + Ok(Arc::new(Schema::new_with_metadata(fields, metadata))) +} + +/// The stored columns a packed-shape scan projects: the record whole, plus the +/// content columns when `with_content` and the dataset carries them (a stamp-12 +/// version inside a stamp-13 graph has none; its scan expands them null). +pub(crate) fn packed_projection(dataset: &lance::Dataset, with_content: bool) -> Vec { + let mut projection: Vec = ["object_id", "object_type", RECORD_COLUMN] + .into_iter() + .map(str::to_string) + .collect(); + if with_content { + projection.extend( + SCHEMA_CONTENT_COLUMNS + .into_iter() + .filter(|name| dataset.schema().field(name).is_some()) + .map(str::to_string), + ); + } + projection +} + +/// The content column `name` of `batch` in the stored shape: the batch's own +/// column when it carries one, else all null (a logical batch read from a +/// version without the columns, or one that never projected them). +fn stored_content_column(batch: &RecordBatch, name: &str) -> Result { + match batch.column_by_name(name) { + None => Ok(new_null_array(&DataType::LargeUtf8, batch.num_rows())), + Some(column) if column.data_type() == &DataType::LargeUtf8 => Ok(column.clone()), + Some(column) => Err(OmniError::manifest_internal(format!( + "manifest column '{name}' is {}, expected LargeUtf8", + column.data_type() + ))), + } } fn record_fields_of(schema: &SchemaRef) -> Result { @@ -250,23 +319,24 @@ pub(crate) fn compact_to_storage(batch: &RecordBatch, schema: &SchemaRef) -> Res children.push(Arc::new(UInt8Array::from(present))); let record = StructArray::try_new(record_fields_of(schema)?, children, None) .map_err(OmniError::arrow_internal)?; - RecordBatch::try_new( - schema.clone(), - vec![ - named_column(batch, "object_id")?, - named_column(batch, "object_type")?, - Arc::new(record), - ], - ) - .map_err(OmniError::arrow_internal) + let mut columns = vec![ + named_column(batch, "object_id")?, + named_column(batch, "object_type")?, + Arc::new(record) as ArrayRef, + ]; + for name in SCHEMA_CONTENT_COLUMNS { + columns.push(stored_content_column(batch, name)?); + } + RecordBatch::try_new(schema.clone(), columns).map_err(OmniError::arrow_internal) } /// Spread a stored batch back into the logical [`manifest_schema`] columns, restoring each null /// from its `present` bit. Every child keeps its value buffers: a logical column is the child's /// buffers under a null bitmap built from `present`, and only the rows marked null are read, to -/// check that the filler is what the child holds there. Any other projected column (a Lance -/// row-metadata column) passes through unchanged after the logical columns, so the boundary is -/// total over projections. +/// check that the filler is what the child holds there. A content column the scan did not +/// project, or that the stored version does not carry, expands all null. Any other projected +/// column (a Lance row-metadata column) passes through unchanged after the logical columns, so +/// the boundary is total over projections. pub(crate) fn expand_from_storage(batch: &RecordBatch) -> Result { let record = named_column(batch, RECORD_COLUMN)?; let record = record @@ -344,11 +414,16 @@ pub(crate) fn expand_from_storage(batch: &RecordBatch) -> Result { }; columns.push(column); } + for name in SCHEMA_CONTENT_COLUMNS { + fields.push(Arc::new(schema_content_field(name))); + columns.push(stored_content_column(batch, name)?); + } for (field, column) in batch.schema().fields().iter().zip(batch.columns()) { if !matches!( field.name().as_str(), "object_id" | "object_type" | RECORD_COLUMN - ) { + ) && !is_schema_content_column(field.name()) + { fields.push(field.clone()); columns.push(column.clone()); } @@ -378,6 +453,16 @@ fn filler_mismatch(name: &str, row: usize) -> OmniError { /// conversion may hold open; every supported manifest converts to the packed /// shape on its next publish. pub(crate) fn flat_to_storage(batch: &RecordBatch, schema: &SchemaRef) -> Result { + for name in SCHEMA_CONTENT_COLUMNS { + if batch + .column_by_name(name) + .is_some_and(|column| column.null_count() < column.len()) + { + return Err(OmniError::manifest_internal(format!( + "manifest column '{name}' carries a value, which the flat shape cannot store" + ))); + } + } let columns = schema .fields() .iter() diff --git a/crates/omnigraph-catalog/src/retention.rs b/crates/omnigraph-catalog/src/retention.rs index f59bfee71..ccd20af82 100644 --- a/crates/omnigraph-catalog/src/retention.rs +++ b/crates/omnigraph-catalog/src/retention.rs @@ -235,6 +235,7 @@ impl ManifestCoordinator { .await { Ok(dataset) => dataset, + Err(lance::Error::DatasetNotFound { .. }) => continue, Err(error) if error.is_not_found() => continue, Err(error) => return Err(OmniError::storage(error)), }; diff --git a/crates/omnigraph-catalog/src/state.rs b/crates/omnigraph-catalog/src/state.rs index 962629f1e..180e99823 100644 --- a/crates/omnigraph-catalog/src/state.rs +++ b/crates/omnigraph-catalog/src/state.rs @@ -1,12 +1,16 @@ use std::collections::HashMap; use std::sync::Arc; -use arrow_array::{Array, RecordBatch, StringArray, UInt64Array}; -use futures::TryStreamExt; +use arrow_array::{Array, LargeStringArray, RecordBatch, StringArray, UInt64Array}; +use futures::future::BoxFuture; +use futures::{Stream, StreamExt, TryStreamExt}; use lance::Dataset; use crate::error::{OmniError, Result}; -use crate::record::{RECORD_COLUMN, StoredShape, expand_from_storage, stored_shape}; +use crate::record::{ + SCHEMA_CONTENT_COLUMNS, StoredShape, expand_from_storage, flat_projection, packed_projection, + stored_shape, +}; /// The row schemas' public path; they are defined in the private `record` module. pub use crate::record::{flat_manifest_schema, manifest_schema}; @@ -14,8 +18,9 @@ use super::layout::{manifest_version_from_object_id, table_object_id, version_ob use super::metadata::TableVersionMetadata; use super::migrations::read_stamp; use super::{ - MAIN_BRANCH_HEAD_KEY, OBJECT_TYPE_GRAPH_COMMIT, OBJECT_TYPE_GRAPH_HEAD, OBJECT_TYPE_TABLE, - OBJECT_TYPE_TABLE_TOMBSTONE, OBJECT_TYPE_TABLE_VERSION, TableIdentity, TableRegistration, + MAIN_BRANCH_HEAD_KEY, OBJECT_TYPE_GRAPH_COMMIT, OBJECT_TYPE_GRAPH_HEAD, + OBJECT_TYPE_SCHEMA_CONTRACT, OBJECT_TYPE_TABLE, OBJECT_TYPE_TABLE_TOMBSTONE, + OBJECT_TYPE_TABLE_VERSION, SCHEMA_CONTRACT_OBJECT_ID, TableIdentity, TableRegistration, }; #[derive(Debug, Clone)] @@ -55,6 +60,54 @@ pub struct ManifestState { /// manifest-only refresh from leaving coarse write authority split between /// a fresh table view and the commit graph's older derived cache. pub graph_heads: HashMap, + /// The `schema_contract` row's small fields from this SAME manifest + /// version; `None` before the row exists (a stamp-12 version read through + /// time travel). + pub schema_contract: Option, +} + +/// The small fields of the `schema_contract` row: the JSON payload of its +/// `metadata` column and the part of the contract every manifest-state read +/// folds. The texts stay in the content columns, projected for serving admission +/// and by [`read_schema_contract_row`]. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct SchemaContractHead { + pub schema_ir_hash: String, + pub schema_identity_version: u32, + pub schema_identity_domain: String, +} + +/// The whole `schema_contract` row: the `.pg` source and the IR JSON, +/// byte-exact, beside the folded head. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SchemaContractRow { + pub source: String, + pub ir: String, + pub head: SchemaContractHead, +} + +impl SchemaContractRow { + /// A contract for a catalog-only fixture: the IR text and hash of the + /// catalog's bound IR, an empty source, identity version 1. + #[cfg(any(test, feature = "test-util"))] + pub fn for_test_catalog(catalog: &omnigraph_compiler::catalog::Catalog) -> Result { + let schema_ir = catalog.bound_schema_ir().ok_or_else(|| { + OmniError::manifest_internal( + "a test schema contract requires an identity-bound catalog".to_string(), + ) + })?; + Ok(Self { + source: String::new(), + ir: omnigraph_compiler::schema_ir_pretty_json(schema_ir) + .map_err(|error| OmniError::manifest_internal(error.to_string()))?, + head: SchemaContractHead { + schema_ir_hash: omnigraph_compiler::schema_ir_hash(schema_ir) + .map_err(|error| OmniError::manifest_internal(error.to_string()))?, + schema_identity_version: 1, + schema_identity_domain: schema_ir.schema_identity_domain.as_str().to_string(), + }, + }) + } } #[derive(Debug, Clone)] @@ -115,7 +168,7 @@ pub fn graph_head_object_id(branch: Option<&str>) -> String { ) } -#[derive(Debug, Clone)] +#[derive(Debug)] struct ManifestScan { table_registrations: HashMap, version_entries: Vec, @@ -131,9 +184,12 @@ struct ManifestScan { /// `lineage_rows`, it does not grow with commit history. OCC must distinguish /// a present head from an absent one (notably on a fresh named branch). graph_heads: HashMap, + /// The `schema_contract` row's head, collected on every pass. + schema_contract: Option, + schema_contract_row: Result>, /// Every scanned row in the logical `manifest_schema()` columns (expanded - /// from the packed shape when stored so), kept only on the publish scan: - /// the copy-on-write publish rewrites them. + /// from the packed shape when stored so, the content columns included), + /// kept only on the publish scan: the copy-on-write publish rewrites them. live_rows: Vec, } @@ -145,7 +201,7 @@ pub async fn read_manifest_state_with_registration_clocks( "registration-clock conversion requires a v6 source".to_string(), )); } - let scan = read_manifest_scan_with_clocks(dataset, false, None, true, false).await?; + let scan = read_manifest_scan_with_clocks(dataset, false, None, true, false, false).await?; manifest_state_from_scan(dataset.version().version, scan) } @@ -172,6 +228,8 @@ pub async fn read_manifest_state_and_lineage( tombstones, lineage_rows, graph_heads, + schema_contract, + schema_contract_row: _, live_rows: _, } = read_manifest_scan(dataset, true).await?; let state = assemble_manifest_state( @@ -182,6 +240,7 @@ pub async fn read_manifest_state_and_lineage( .into_iter() .map(|t| (t.identity, t.manifest_version)), graph_heads, + schema_contract, )?; Ok((state, lineage_rows)) } @@ -195,6 +254,7 @@ fn manifest_state_from_scan(version: u64, scan: ManifestScan) -> Result, tombstone_map: HashMap, graph_heads: HashMap, + schema_contract: Option, } impl ProjectionAccumulator { @@ -224,6 +285,7 @@ impl ProjectionAccumulator { latest_versions: HashMap::new(), tombstone_map: HashMap::new(), graph_heads: HashMap::new(), + schema_contract: None, } } @@ -237,6 +299,7 @@ impl ProjectionAccumulator { version_entries: Vec, tombstones: impl IntoIterator, graph_heads: HashMap, + schema_contract: Option, ) -> Result<()> { for (identity, registration) in registrations { if let Some(existing) = self.registrations.insert(identity, registration.clone()) { @@ -275,6 +338,9 @@ impl ProjectionAccumulator { } } self.graph_heads.extend(graph_heads); + if let Some(head) = schema_contract { + self.schema_contract = Some(head); + } Ok(()) } @@ -307,6 +373,7 @@ impl ProjectionAccumulator { .into_iter() .map(|t| (t.identity, t.manifest_version)), scan.graph_heads, + scan.schema_contract, ) } @@ -317,6 +384,13 @@ impl ProjectionAccumulator { self.graph_heads.remove(branch_key); } + /// Drop the schema contract's head (its durable row was deleted, the + /// mutable-row half of a contract replacement; the replacement arrives via + /// the delta fragments' live rows). + pub(crate) fn remove_schema_contract(&mut self) { + self.schema_contract = None; + } + /// Reduce to the visible state at `version`. Pure and repeatable. pub(crate) fn finish(&self, version: u64) -> Result { finish_manifest_state( @@ -325,6 +399,7 @@ impl ProjectionAccumulator { self.latest_versions.values().cloned(), &self.tombstone_map, self.graph_heads.clone(), + self.schema_contract.clone(), ) } } @@ -336,7 +411,70 @@ pub(crate) async fn read_manifest_projection( dataset: &Dataset, ) -> Result<(ManifestState, ProjectionAccumulator, Vec)> { let version = dataset.version().version; - let mut scan = read_manifest_scan_fragments(dataset, true, None).await?; + let scan = read_manifest_scan_fragments(dataset, true, None).await?; + projection_from_scan(version, scan) +} + +pub(crate) async fn read_manifest_projection_with_contract( + dataset: &Dataset, +) -> Result<( + ManifestState, + ProjectionAccumulator, + Vec, + Result, +)> { + let version = dataset.version().version; + let (scan, contract) = read_manifest_scan_with_contract(dataset, true).await?; + let (state, accumulator, lineage_rows) = projection_from_scan(version, scan)?; + Ok((state, accumulator, lineage_rows, contract)) +} + +pub(crate) async fn read_manifest_state_with_contract( + dataset: &Dataset, +) -> Result<(ManifestState, Result)> { + let (scan, contract) = read_manifest_scan_with_contract(dataset, false).await?; + Ok(( + manifest_state_from_scan(dataset.version().version, scan)?, + contract, + )) +} + +async fn read_manifest_scan_with_contract( + dataset: &Dataset, + collect_lineage: bool, +) -> Result<(ManifestScan, Result)> { + let content_shape = require_schema_content(dataset); + let scan_dataset = if content_shape.is_ok() + && !dataset + .schema() + .metadata + .contains_key(crate::migrations::UPGRADE_PENDING_KEY) + { + crate::instrumentation::manifest_scan_dataset(dataset).await? + } else { + dataset.clone() + }; + let version = dataset.version().version; + let mut scan = read_manifest_scan_with_clocks( + &scan_dataset, + collect_lineage, + None, + false, + false, + content_shape.is_ok(), + ) + .await?; + let captured = std::mem::replace(&mut scan.schema_contract_row, Ok(None)); + let contract = content_shape + .and(captured) + .and_then(|row| finish_schema_contract_row(row, scan.schema_contract.as_ref(), version)); + Ok((scan, contract)) +} + +fn projection_from_scan( + version: u64, + mut scan: ManifestScan, +) -> Result<(ManifestState, ProjectionAccumulator, Vec)> { let lineage_rows = std::mem::take(&mut scan.lineage_rows); let mut accumulator = ProjectionAccumulator::empty(); accumulator.fold_scan(scan)?; @@ -441,6 +579,7 @@ pub(crate) fn assemble_manifest_state( version_entries: Vec, tombstones: impl IntoIterator, graph_heads: HashMap, + schema_contract: Option, ) -> Result { assemble_manifest_projection( version, @@ -448,6 +587,7 @@ pub(crate) fn assemble_manifest_state( version_entries, tombstones, graph_heads, + schema_contract, ) .map(|(state, _)| state) } @@ -461,9 +601,16 @@ pub(crate) fn assemble_manifest_projection( version_entries: Vec, tombstones: impl IntoIterator, graph_heads: HashMap, + schema_contract: Option, ) -> Result<(ManifestState, ProjectionAccumulator)> { let mut accumulator = ProjectionAccumulator::empty(); - accumulator.fold_parts(registrations, version_entries, tombstones, graph_heads)?; + accumulator.fold_parts( + registrations, + version_entries, + tombstones, + graph_heads, + schema_contract, + )?; Ok((accumulator.finish(version)?, accumulator)) } @@ -476,6 +623,7 @@ fn finish_manifest_state( latest_versions: impl Iterator, tombstone_map: &HashMap, graph_heads: HashMap, + schema_contract: Option, ) -> Result { let mut entries: Vec = latest_versions .filter(|entry| { @@ -513,6 +661,7 @@ fn finish_manifest_state( version, entries, graph_heads, + schema_contract, }) } @@ -552,6 +701,8 @@ pub(crate) struct PublishScan { /// Exact `graph_head:` rows keyed by the branch suffix (`main` for /// main). Absence is meaningful and is preserved by a missing map entry. pub(crate) graph_heads: HashMap, + /// The `schema_contract` row's head; absent before the row exists. + pub(crate) schema_contract: Option, /// The scanned rows themselves, the input of `commit::overwrite`. pub(crate) live_rows: Vec, } @@ -570,8 +721,21 @@ pub(crate) async fn read_manifest_table_registrations( /// always on here (the publisher resolves a parent), so the lineage JSON decode /// rides the same pass as the table-state assembly instead of a second scan. pub(crate) async fn read_publish_scan(dataset: &Dataset) -> Result { - let scan = read_manifest_scan_with_clocks(dataset, true, None, false, true).await?; - Ok(PublishScan { + let scan = read_manifest_scan_with_clocks(dataset, true, None, false, true, false).await?; + Ok(publish_scan_from_rows(scan)) +} + +pub(crate) async fn decode_publish_batch( + dataset: &Dataset, + batch: RecordBatch, +) -> Result { + let batches = futures::stream::iter([expand_from_storage(&batch)]); + let scan = reduce_manifest_batches(dataset, batches, true, false, false, true, false).await?; + Ok(publish_scan_from_rows(scan)) +} + +fn publish_scan_from_rows(scan: ManifestScan) -> PublishScan { + PublishScan { table_registrations: scan.table_registrations, version_entries: scan.version_entries, tombstones: scan @@ -586,10 +750,172 @@ pub(crate) async fn read_publish_scan(dataset: &Dataset) -> Result .collect(), lineage_rows: scan.lineage_rows, graph_heads: scan.graph_heads, + schema_contract: scan.schema_contract, live_rows: scan.live_rows, + } +} + +/// Decode one `schema_contract` row's head out of its `metadata` column. +/// Shared by the state scan and the content read, so the two cannot drift. +fn decode_schema_contract_row( + object_ids: &StringArray, + metadata: &StringArray, + row: usize, +) -> Result { + require_object_id( + object_ids, + row, + SCHEMA_CONTRACT_OBJECT_ID, + OBJECT_TYPE_SCHEMA_CONTRACT, + )?; + if metadata.is_null(row) { + return Err(OmniError::manifest_internal( + "manifest schema_contract row missing metadata".to_string(), + )); + } + serde_json::from_str(metadata.value(row)).map_err(|e| { + OmniError::manifest_internal(format!("failed to decode schema_contract metadata: {e}")) }) } +/// Read the complete contract at the pinned version and verify its folded head. +/// AllEarly prevents Lance 11's heuristic from splitting the packed record +/// into incompatible child projections during the filtered scan. +pub(crate) async fn read_schema_contract_row( + dataset: &Dataset, + expected: Option<&SchemaContractHead>, +) -> Result { + require_schema_content(dataset)?; + crate::instrumentation::record_manifest_scan(); + let mut scanner = dataset.scan(); + scanner + .project(&packed_projection(dataset, true)) + .map_err(OmniError::storage)?; + scanner.filter_expr( + datafusion::prelude::col("object_id") + .eq(datafusion::prelude::lit(SCHEMA_CONTRACT_OBJECT_ID)), + ); + scanner.materialization_style(lance::dataset::scanner::MaterializationStyle::AllEarly); + let mut batches = scanner + .try_into_stream() + .await + .map_err(OmniError::storage)?; + let mut found: Option = None; + while let Some(batch) = batches.try_next().await.map_err(OmniError::storage)? { + let batch = expand_from_storage(&batch)?; + fold_schema_contract_batch(&batch, &mut found)?; + } + finish_schema_contract_row(found, expected, dataset.version().version) +} + +pub(crate) fn schema_contract_from_batch( + dataset: &Dataset, + batch: &RecordBatch, + expected: Option<&SchemaContractHead>, +) -> Result { + require_schema_content(dataset)?; + let batch = expand_from_storage(batch)?; + let mut found = None; + fold_schema_contract_batch(&batch, &mut found)?; + finish_schema_contract_row(found, expected, dataset.version().version) +} + +fn require_schema_content(dataset: &Dataset) -> Result<()> { + let stamp = read_stamp(dataset); + if stored_shape(stamp) != StoredShape::Packed + || SCHEMA_CONTENT_COLUMNS + .iter() + .any(|name| dataset.schema().field(name).is_none()) + { + return Err(OmniError::manifest_internal(format!( + "__manifest version {} carries no schema_contract row: it was written at internal \ + schema v{}, before the contract lived in the manifest", + dataset.version().version, + stamp.map_or("?".to_string(), |stamp| stamp.to_string()) + ))); + } + for name in SCHEMA_CONTENT_COLUMNS { + let field = dataset + .schema() + .field(name) + .expect("content field checked above"); + if field.data_type() != arrow_schema::DataType::LargeUtf8 { + return Err(OmniError::manifest_internal(format!( + "manifest column '{name}' is {}, expected LargeUtf8", + field.data_type() + ))); + } + } + Ok(()) +} + +fn fold_schema_contract_batch( + batch: &RecordBatch, + found: &mut Option, +) -> Result<()> { + let object_ids = string_column(batch, "object_id")?; + let object_types = string_column(batch, "object_type")?; + let metadata = string_column(batch, "metadata")?; + let sources = large_string_column(batch, SCHEMA_CONTENT_COLUMNS[0])?; + let irs = large_string_column(batch, SCHEMA_CONTENT_COLUMNS[1])?; + for row in 0..batch.num_rows() { + if object_ids.value(row) != SCHEMA_CONTRACT_OBJECT_ID { + continue; + } + if object_types.value(row) != OBJECT_TYPE_SCHEMA_CONTRACT { + return Err(OmniError::manifest_internal(format!( + "manifest row '{SCHEMA_CONTRACT_OBJECT_ID}' has object_type '{}'", + object_types.value(row) + ))); + } + let head = decode_schema_contract_row(object_ids, metadata, row)?; + if sources.is_null(row) || irs.is_null(row) { + return Err(OmniError::manifest_internal( + "manifest schema_contract row is missing its source or IR text".to_string(), + )); + } + let contract = SchemaContractRow { + source: sources.value(row).to_string(), + ir: irs.value(row).to_string(), + head, + }; + if found.replace(contract).is_some() { + return Err(OmniError::manifest_internal( + "manifest has two schema_contract rows".to_string(), + )); + } + } + Ok(()) +} + +fn finish_schema_contract_row( + found: Option, + expected: Option<&SchemaContractHead>, + version: u64, +) -> Result { + let contract = found.ok_or_else(|| { + OmniError::manifest_internal(format!( + "__manifest version {} has no schema_contract row", + version + )) + })?; + if let Some(expected) = expected + && *expected != contract.head + { + return Err(OmniError::manifest_internal(format!( + "manifest schema_contract row disagrees with the folded state: the row carries IR \ + hash {} (identity v{} of domain {}), the state {} (identity v{} of domain {})", + contract.head.schema_ir_hash, + contract.head.schema_identity_version, + contract.head.schema_identity_domain, + expected.schema_ir_hash, + expected.schema_identity_version, + expected.schema_identity_domain, + ))); + } + Ok(contract) +} + /// Decode one `graph_commit` row (`object_type == OBJECT_TYPE_GRAPH_COMMIT`) into /// a [`GraphLineageRow`]. The single decode for both lineage readers — the /// dedicated `read_graph_lineage` scan and the folded `collect_lineage` branch of @@ -705,79 +1031,107 @@ async fn read_manifest_scan_fragments( collect_lineage: bool, fragments: Option>, ) -> Result { - read_manifest_scan_with_clocks(dataset, collect_lineage, fragments, false, false).await + read_manifest_scan_with_clocks(dataset, collect_lineage, fragments, false, false, false).await } -/// The columns a `__manifest` scan projects: the three stored columns with `record` whole -/// when packed (the `record` module doc: a fixed-width child cannot be projected alone), -/// otherwise exactly the logical columns. `expand_from_storage` turns the packed rows logical. -fn manifest_projection(shape: StoredShape) -> Vec { +/// The columns a `__manifest` scan projects: `record` whole when packed, the content +/// columns only when `with_content` (publication or cold admission), the flat +/// logical columns otherwise. +fn manifest_projection(dataset: &Dataset, shape: StoredShape, with_content: bool) -> Vec { match shape { - StoredShape::Packed => ["object_id", "object_type", RECORD_COLUMN] - .into_iter() - .map(str::to_string) - .collect(), - StoredShape::Flat => manifest_schema() - .fields() - .iter() - .map(|field| field.name().clone()) - .collect(), + StoredShape::Packed => packed_projection(dataset, with_content), + StoredShape::Flat => flat_projection(), } } -async fn read_manifest_scan_with_clocks( +fn read_manifest_scan_with_clocks( dataset: &Dataset, collect_lineage: bool, fragments: Option>, use_row_update_versions: bool, retain_live_rows: bool, -) -> Result { - let historical; - let dataset = if super::migrations::read_stamp(dataset) == Some(7) - && dataset - .schema() - .metadata - .contains_key(crate::migrations::UPGRADE_PENDING_KEY) - { - historical = Box::pin(crate::migrations::historical_source(dataset.clone(), 7)).await?; - &historical - } else { - dataset - }; - let stamp = read_stamp(dataset); - let legacy = stamp == Some(6); - let shape = stored_shape(stamp); - crate::instrumentation::record_manifest_scan(); - let mut projection = manifest_projection(shape); - if use_row_update_versions { - projection.push("_row_last_updated_at_version".to_string()); - } - let is_delta_scan = fragments.is_some(); - let mut scanner = dataset.scan(); - scanner.project(&projection).map_err(OmniError::storage)?; - if let Some(fragments) = fragments { - // Lance semantics this fold depends on: `with_fragments(vec![])` - // scans ZERO fragments (it does not fall back to all) — an empty - // delta (a DV-only refresh window) must read nothing. - scanner.with_fragments(fragments); - } - let mut batches = scanner - .try_into_stream() + read_contract: bool, +) -> BoxFuture<'_, Result> { + Box::pin(async move { + let historical; + let dataset = if let Some(stamp @ (7 | 13)) = super::migrations::read_stamp(dataset) + && dataset + .schema() + .metadata + .contains_key(crate::migrations::UPGRADE_PENDING_KEY) + { + historical = + Box::pin(crate::migrations::historical_source(dataset.clone(), stamp)).await?; + &historical + } else { + dataset + }; + let stamp = read_stamp(dataset); + let shape = stored_shape(stamp); + crate::instrumentation::record_manifest_scan(); + let mut projection = manifest_projection(dataset, shape, retain_live_rows || read_contract); + if use_row_update_versions { + projection.push("_row_last_updated_at_version".to_string()); + } + let is_delta_scan = fragments.is_some(); + let mut scanner = dataset.scan(); + scanner.project(&projection).map_err(OmniError::storage)?; + if let Some(fragments) = fragments { + // Lance semantics this fold depends on: `with_fragments(vec![])` + // scans ZERO fragments (it does not fall back to all) — an empty + // delta (a DV-only refresh window) must read nothing. + scanner.with_fragments(fragments); + } + let batches = scanner + .try_into_stream() + .await + .map_err(OmniError::storage)? + .map(move |batch| { + let batch = batch.map_err(OmniError::storage)?; + match shape { + StoredShape::Packed => expand_from_storage(&batch), + StoredShape::Flat => Ok(batch), + } + }); + reduce_manifest_batches( + dataset, + batches, + collect_lineage, + use_row_update_versions, + is_delta_scan, + retain_live_rows, + read_contract, + ) .await - .map_err(OmniError::storage)?; + }) +} +async fn reduce_manifest_batches( + dataset: &Dataset, + mut batches: impl Stream> + Unpin, + collect_lineage: bool, + use_row_update_versions: bool, + is_delta_scan: bool, + retain_live_rows: bool, + read_contract: bool, +) -> Result { + let legacy = read_stamp(dataset) == Some(6); let mut table_registrations = HashMap::new(); let mut version_entries = Vec::new(); let mut tombstones = Vec::new(); let mut lineage_rows = Vec::new(); let mut graph_heads = HashMap::new(); + let mut schema_contract = None; + let mut schema_contract_row = Ok(None); let mut live_rows = Vec::new(); - while let Some(batch) = batches.try_next().await.map_err(OmniError::storage)? { - let batch = match shape { - StoredShape::Packed => expand_from_storage(&batch)?, - StoredShape::Flat => batch, - }; + while let Some(batch) = batches.try_next().await? { + if read_contract + && let Ok(found) = &mut schema_contract_row + && let Err(error) = fold_schema_contract_batch(&batch, found) + { + schema_contract_row = Err(error); + } if retain_live_rows { live_rows.push(batch.clone()); } @@ -804,6 +1158,14 @@ async fn read_manifest_scan_with_clocks( for row in 0..batch.num_rows() { let table_key = table_keys.value(row).to_string(); + if object_ids.value(row) == SCHEMA_CONTRACT_OBJECT_ID + && object_types.value(row) != OBJECT_TYPE_SCHEMA_CONTRACT + { + return Err(OmniError::manifest_internal(format!( + "manifest row '{SCHEMA_CONTRACT_OBJECT_ID}' has object_type '{}'", + object_types.value(row) + ))); + } match object_types.value(row) { OBJECT_TYPE_TABLE => { let identity = required_table_identity( @@ -949,6 +1311,20 @@ async fn read_manifest_scan_with_clocks( decode_graph_head_row(object_ids, metadata, row)?; graph_heads.insert(branch_key, head_commit_id); } + OBJECT_TYPE_SCHEMA_CONTRACT => { + require_null_table_identity( + stable_table_ids, + table_incarnation_ids, + row, + OBJECT_TYPE_SCHEMA_CONTRACT, + )?; + let head = decode_schema_contract_row(object_ids, metadata, row)?; + if schema_contract.replace(head).is_some() { + return Err(OmniError::manifest_internal( + "manifest has two schema_contract rows".to_string(), + )); + } + } // Commit rows are skipped on the table-state path; unknown future // object types are skipped on every path. _ => {} @@ -1006,6 +1382,8 @@ async fn read_manifest_scan_with_clocks( tombstones, lineage_rows, graph_heads, + schema_contract, + schema_contract_row, live_rows, }) } @@ -1027,7 +1405,7 @@ pub async fn read_graph_lineage( let shape = stored_shape(read_stamp(dataset)); let mut scanner = dataset.scan(); scanner - .project(&manifest_projection(shape)) + .project(&manifest_projection(dataset, shape, false)) .map_err(OmniError::storage)?; let mut batches = scanner .try_into_stream() @@ -1159,12 +1537,20 @@ pub fn graph_lineage_row_parts( ]) } +/// The `metadata` JSON of a `schema_contract` row carrying `head`. +pub(crate) fn schema_contract_metadata_json(head: &SchemaContractHead) -> Result { + serde_json::to_string(head).map_err(|e| { + OmniError::manifest_internal(format!("failed to encode schema_contract metadata: {e}")) + }) +} + pub(crate) fn entries_to_batch( entries: &[DatasetEntry], version_metadata: &HashMap, genesis_lineage: &[GraphLineageRowPart], + schema_contract: Option<&SchemaContractRow>, ) -> Result { - let cap = entries.len() * 2 + genesis_lineage.len(); + let cap = entries.len() * 2 + genesis_lineage.len() + usize::from(schema_contract.is_some()); let mut object_ids = Vec::with_capacity(cap); let mut object_types = Vec::with_capacity(cap); let mut locations = Vec::with_capacity(cap); @@ -1174,6 +1560,8 @@ pub(crate) fn entries_to_batch( let mut table_versions = Vec::with_capacity(cap); let mut table_branches = Vec::with_capacity(cap); let mut row_counts = Vec::with_capacity(cap); + let mut schema_sources = Vec::with_capacity(cap); + let mut schema_irs = Vec::with_capacity(cap); for entry in entries { object_ids.push(table_object_id(entry.identity)); @@ -1185,6 +1573,8 @@ pub(crate) fn entries_to_batch( table_versions.push(None); table_branches.push(None); row_counts.push(None); + schema_sources.push(None); + schema_irs.push(None); object_ids.push(version_object_id(entry.identity, entry.manifest_version)); object_types.push(OBJECT_TYPE_TABLE_VERSION.to_string()); @@ -1205,6 +1595,8 @@ pub(crate) fn entries_to_batch( table_versions.push(Some(entry.published_dataset_version)); table_branches.push(entry.native_dataset_branch.clone()); row_counts.push(Some(entry.entity_count)); + schema_sources.push(None); + schema_irs.push(None); } // Genesis graph-lineage rows ride the init write so a fresh graph carries @@ -1222,6 +1614,22 @@ pub(crate) fn entries_to_batch( table_versions.push(part.table_version); table_branches.push(part.table_branch.clone()); row_counts.push(None); + schema_sources.push(None); + schema_irs.push(None); + } + + if let Some(contract) = schema_contract { + object_ids.push(SCHEMA_CONTRACT_OBJECT_ID.to_string()); + object_types.push(OBJECT_TYPE_SCHEMA_CONTRACT.to_string()); + locations.push(None); + metadata.push(Some(schema_contract_metadata_json(&contract.head)?)); + table_keys.push(String::new()); + table_identities.push(None); + table_versions.push(None); + table_branches.push(None); + row_counts.push(None); + schema_sources.push(Some(contract.source.clone())); + schema_irs.push(Some(contract.ir.clone())); } manifest_rows_batch( @@ -1234,6 +1642,8 @@ pub(crate) fn entries_to_batch( table_versions, table_branches, row_counts, + schema_sources, + schema_irs, ) } @@ -1247,6 +1657,8 @@ pub(crate) fn manifest_rows_batch( table_versions: Vec>, table_branches: Vec>, row_counts: Vec>, + schema_sources: Vec>, + schema_irs: Vec>, ) -> Result { let len = object_ids.len(); if table_identities.len() != len { @@ -1256,9 +1668,27 @@ pub(crate) fn manifest_rows_batch( table_identities.len() ))); } + if schema_sources.len() != len || schema_irs.len() != len { + return Err(OmniError::manifest_internal(format!( + "manifest batch has {len} object rows but {} schema sources and {} schema IRs", + schema_sources.len(), + schema_irs.len() + ))); + } for (row, (object_type, identity)) in object_types.iter().zip(table_identities.iter()).enumerate() { + let carries_content = schema_sources[row].is_some() || schema_irs[row].is_some(); + if carries_content != (object_type == OBJECT_TYPE_SCHEMA_CONTRACT) { + return Err(OmniError::manifest_internal(format!( + "manifest {object_type} row at index {row} {} schema contract content", + if carries_content { + "must not carry" + } else { + "is missing its" + } + ))); + } match object_type.as_str() { OBJECT_TYPE_TABLE | OBJECT_TYPE_TABLE_VERSION | OBJECT_TYPE_TABLE_TOMBSTONE => { let identity = identity.ok_or_else(|| { @@ -1291,11 +1721,26 @@ pub(crate) fn manifest_rows_batch( } } } - OBJECT_TYPE_GRAPH_COMMIT | OBJECT_TYPE_GRAPH_HEAD if identity.is_some() => { + OBJECT_TYPE_GRAPH_COMMIT | OBJECT_TYPE_GRAPH_HEAD | OBJECT_TYPE_SCHEMA_CONTRACT + if identity.is_some() => + { return Err(OmniError::manifest_internal(format!( "manifest {object_type} row at index {row} must not carry table identity" ))); } + OBJECT_TYPE_SCHEMA_CONTRACT => { + if object_ids[row] != SCHEMA_CONTRACT_OBJECT_ID { + return Err(OmniError::manifest_internal(format!( + "manifest {object_type} row at index {row} has object_id {:?}, expected '{SCHEMA_CONTRACT_OBJECT_ID}'", + object_ids[row] + ))); + } + if schema_sources[row].is_none() || schema_irs[row].is_none() { + return Err(OmniError::manifest_internal(format!( + "manifest {object_type} row at index {row} is missing its source or IR text" + ))); + } + } _ => {} } } @@ -1320,11 +1765,26 @@ pub(crate) fn manifest_rows_batch( Arc::new(UInt64Array::from(table_versions)), Arc::new(StringArray::from(table_branches)), Arc::new(UInt64Array::from(row_counts)), + Arc::new(LargeStringArray::from(schema_sources)), + Arc::new(LargeStringArray::from(schema_irs)), ], ) .map_err(OmniError::arrow_internal) } +fn large_string_column<'a>(batch: &'a RecordBatch, name: &str) -> Result<&'a LargeStringArray> { + batch + .column_by_name(name) + .ok_or_else(|| { + OmniError::manifest_internal(format!("manifest batch missing '{name}' column")) + })? + .as_any() + .downcast_ref::() + .ok_or_else(|| { + OmniError::manifest_internal(format!("manifest column '{name}' is not LargeUtf8")) + }) +} + pub(crate) fn string_column<'a>(batch: &'a RecordBatch, name: &str) -> Result<&'a StringArray> { batch .column_by_name(name) diff --git a/crates/omnigraph-catalog/src/tests.rs b/crates/omnigraph-catalog/src/tests.rs index aed1610e9..9ced5f0bd 100644 --- a/crates/omnigraph-catalog/src/tests.rs +++ b/crates/omnigraph-catalog/src/tests.rs @@ -323,10 +323,17 @@ async fn exact_genesis_probe_rejects_another_initialization_attempt() { assert_eq!(catalog.system_columns, system_columns); let control_session = crate::lance_access::control_session(); let committed_attempt = GenesisManifestAttempt::mint(catalog.system_columns).unwrap(); + let contract = SchemaContractRow::for_test_catalog(&catalog).unwrap(); - ManifestCoordinator::init_commit(uri, &catalog, &control_session, &committed_attempt) - .await - .unwrap(); + ManifestCoordinator::init_commit( + uri, + &catalog, + &contract, + &control_session, + &committed_attempt, + ) + .await + .unwrap(); ManifestCoordinator::open_exact_genesis_with_lineage( uri, &committed_attempt, @@ -1738,7 +1745,8 @@ impl ManifestBatchPublisher for RecordingPublisher { ManifestChange::Update(update) => Some(update.to_create_table_version_request()), ManifestChange::RegisterTable(_) | ManifestChange::RenameTable(_) - | ManifestChange::Tombstone(_) => None, + | ManifestChange::Tombstone(_) + | ManifestChange::SchemaContract(_) => None, }) .collect(); self.requests.lock().await.extend_from_slice(&requests); @@ -2683,6 +2691,19 @@ async fn exact_publish_rejects_named_branch_delete_recreate_aba() { let mut mc = ManifestCoordinator::init(uri, &catalog).await.unwrap(); mc.create_branch("feature").await.unwrap(); + let publisher = GraphNamespacePublisher::new(uri, Some("feature")); + let contract = mc.read_schema_contract().await.unwrap(); + publisher + .publish( + &[ManifestChange::SchemaContract(contract.clone())], + &HashMap::new(), + None, + ) + .await + .unwrap(); + mc.commit_changes(&[ManifestChange::SchemaContract(contract)]) + .await + .unwrap(); let old_branch = open_manifest_dataset(uri, Some("feature")).await.unwrap(); let old_identifier = old_branch.branch_identifier().await.unwrap(); assert!( @@ -2729,13 +2750,14 @@ async fn exact_publish_rejects_named_branch_delete_recreate_aba() { let recreated = open_manifest_dataset(uri, Some("feature")).await.unwrap(); let recreated_identifier = recreated.branch_identifier().await.unwrap(); assert_ne!(old_identifier, recreated_identifier); + assert_eq!(old_branch.version().version, recreated.version().version); let precondition = PublishPrecondition::ExactGraphHead(GraphHeadExpectation::new( Some("feature"), old_identifier, None, )); - let err = GraphNamespacePublisher::new(uri, Some("feature")) + let err = publisher .publish_with_precondition(&[], &HashMap::new(), None, &precondition) .await .expect_err("recreated branch must reject the old incarnation token"); @@ -3152,7 +3174,7 @@ async fn legacy_manifest_fixture( entry.identity, entry.version_metadata.to_json_string().unwrap(), )]); - let batch = super::state::entries_to_batch(&[entry], &metadata, &[]).unwrap(); + let batch = super::state::entries_to_batch(&[entry], &metadata, &[], None).unwrap(); let batch = if matches!(mode, lance::dataset::WriteMode::Append) { batch.slice(1, 1) } else { @@ -3265,7 +3287,7 @@ async fn legacy_manifest_decoder_preserves_equal_version_tombstone() { source.identity, source.version_metadata.to_json_string().unwrap(), )]); - let batch = super::state::entries_to_batch(std::slice::from_ref(&source), &metadata, &[]) + let batch = super::state::entries_to_batch(std::slice::from_ref(&source), &metadata, &[], None) .unwrap() .slice(1, 1); let mut columns = batch.columns().to_vec(); @@ -3356,6 +3378,8 @@ fn record_states_batch() -> RecordBatch { vec![None, Some(9), Some(0)], vec![None, Some("main".into()), Some(String::new())], vec![None, None, Some(0)], + vec![None, None, None], + vec![None, None, None], ) .unwrap() } @@ -3365,8 +3389,26 @@ fn packed_record_round_trips_nulls_through_present_bits() { let logical = record_states_batch(); let schema = super::record::manifest_storage_schema(HashMap::new()).unwrap(); let names: Vec<_> = schema.fields().iter().map(|f| f.name().as_str()).collect(); - assert_eq!(names, ["object_id", "object_type", "record"]); + assert_eq!( + names, + [ + "object_id", + "object_type", + "record", + "schema_source", + "schema_ir" + ] + ); let stored = super::record::compact_to_storage(&logical, &schema).unwrap(); + assert!( + stored + .column_by_name("schema_source") + .is_some_and(|column| column.null_count() == 3) + && stored + .column_by_name("schema_ir") + .is_some_and(|column| column.null_count() == 3), + "rows without a contract store null content" + ); let record = stored .column_by_name("record") .unwrap() @@ -3398,9 +3440,21 @@ fn packed_record_round_trips_nulls_through_present_bits() { let expanded = super::record::expand_from_storage(&stored).unwrap(); assert_eq!(expanded, logical); + let without_content = stored.project(&[0, 1, 2]).unwrap(); + assert_eq!( + super::record::expand_from_storage(&without_content).unwrap(), + logical, + "a scan that never projected the content columns expands them null" + ); + let flat = super::state::flat_manifest_schema(); assert_eq!(flat.fields().len(), 11); assert_eq!(flat.field(4).name(), "base_objects"); + assert!( + flat.field_with_name("schema_source").is_err() + && flat.field_with_name("schema_ir").is_err(), + "the flat shape never carried the content columns" + ); } /// `stored` with its `present` column replaced by `present`, the rows a @@ -3420,15 +3474,20 @@ fn with_present_bits(stored: &RecordBatch, present: Vec) -> RecordBatch { .unwrap(); children[present_index] = Arc::new(arrow_array::UInt8Array::from(present)); let tampered = arrow_array::StructArray::new(record.fields().clone(), children, None); - RecordBatch::try_new( - stored.schema(), - vec![ - stored.column(0).clone(), - stored.column(1).clone(), - Arc::new(tampered), - ], - ) - .unwrap() + let columns = stored + .schema() + .fields() + .iter() + .zip(stored.columns()) + .map(|(field, column)| -> arrow_array::ArrayRef { + if field.name() == "record" { + Arc::new(tampered.clone()) + } else { + column.clone() + } + }) + .collect(); + RecordBatch::try_new(stored.schema(), columns).unwrap() } #[test] @@ -3456,9 +3515,9 @@ fn packed_record_refuses_a_null_bit_beside_a_value() { ); } -/// A manifest still stored flat at stamp 11 opens and reads as it is, and its -/// next publish rewrites it packed at stamp 12 with the same state; the -/// pre-conversion version keeps its flat shape for time travel. +/// A flat stamp-11 manifest reads as it is; a publish over it rewrites it packed at stamp 13 +/// with the same table state (no `schema_contract` row: the storage upgrade route adds that), +/// and the pre-conversion version keeps its flat shape for time travel. #[tokio::test] async fn stamp_11_manifest_converts_on_its_next_publish() { let dir = tempfile::tempdir().unwrap(); @@ -3467,7 +3526,7 @@ async fn stamp_11_manifest_converts_on_its_next_publish() { .await .unwrap(); let mut born = open_manifest_dataset(uri, None).await.unwrap(); - assert_eq!(super::migrations::read_stamp(&born), Some(12)); + assert_eq!(super::migrations::read_stamp(&born), Some(13)); let at_birth = logical_view(&born).await; super::migrations::restamp_flat_for_test(&mut born, 11) @@ -3480,13 +3539,21 @@ async fn stamp_11_manifest_converts_on_its_next_publish() { assert!(flat.schema().field("base_objects").is_some()); let flat_version = flat.version().version; assert_eq!(logical_view(&flat).await, at_birth); + assert!( + super::state::read_manifest_state(&flat) + .await + .unwrap() + .schema_contract + .is_none(), + "a flat manifest has no schema_contract row" + ); let live_rows = read_publish_scan(&flat).await.unwrap().live_rows; let empty_pending = live_rows[0].slice(0, 0); - let converted = super::commit::overwrite(flat, empty_pending, live_rows) + let (converted, _) = super::commit::overwrite(flat, empty_pending, live_rows) .await .unwrap(); - assert_eq!(super::migrations::read_stamp(&converted), Some(12)); + assert_eq!(super::migrations::read_stamp(&converted), Some(13)); assert!( converted.manifest().uses_stable_row_ids(), "the conversion overwrite must keep the stable row ids genesis enables" @@ -3497,13 +3564,22 @@ async fn stamp_11_manifest_converts_on_its_next_publish() { .iter() .map(|f| f.name.clone()) .collect(); - assert_eq!(names, ["object_id", "object_type", "record"]); + assert_eq!( + names, + [ + "object_id", + "object_type", + "record", + "schema_source", + "schema_ir" + ] + ); let fragments = converted.get_fragments(); let file = &fragments[0].metadata().files[0]; assert_eq!( file.column_indices.len(), - 3, - "the packed record is one physical column beside `object_id` and `object_type`: {:?}", + 5, + "the packed record is one physical column beside `object_id`, `object_type` and the two content columns: {:?}", file.column_indices ); assert_eq!(logical_view(&converted).await, at_birth); @@ -3514,6 +3590,464 @@ async fn stamp_11_manifest_converts_on_its_next_publish() { assert_eq!(logical_view(&historical).await, at_birth); } +fn replacement_contract() -> SchemaContractRow { + SchemaContractRow { + source: "node Person {\n name: String\n nickname: String?\n}\n".to_string(), + ir: "{\n \"ir_version\": 5,\n \"nodes\": []\n}\n".to_string(), + head: SchemaContractHead { + schema_ir_hash: "sha256:replacement".to_string(), + schema_identity_version: 2, + schema_identity_domain: "01ARZ3NDEKTSV4RRFFQ69G5FAV".to_string(), + }, + } +} + +async fn schema_contract_row_count(dataset: &Dataset) -> usize { + let mut scanner = dataset.scan(); + scanner.filter_expr( + datafusion::prelude::col("object_id") + .eq(datafusion::prelude::lit(SCHEMA_CONTRACT_OBJECT_ID)), + ); + scanner.count_rows().await.unwrap() as usize +} + +/// Genesis writes the one `schema_contract` row in the Create commit: the head is folded into +/// the state and the snapshot, the texts come back byte-exact through both reads. +#[tokio::test] +async fn genesis_writes_the_schema_contract_row_and_reads_it_back() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let catalog = build_test_catalog(); + let expected = SchemaContractRow::for_test_catalog(&catalog).unwrap(); + let mc = ManifestCoordinator::init(uri, &catalog).await.unwrap(); + + assert_eq!( + mc.known_state.schema_contract.as_ref(), + Some(&expected.head) + ); + assert_eq!(mc.snapshot().schema_contract(), Some(&expected.head)); + assert_eq!(mc.read_schema_contract().await.unwrap(), expected); + assert_eq!( + ManifestCoordinator::read_schema_contract_at(uri, None, 1) + .await + .unwrap(), + expected + ); + let ds = open_manifest_dataset(uri, None).await.unwrap(); + assert_eq!(schema_contract_row_count(&ds).await, 1); + let batch = &read_publish_scan(&ds).await.unwrap().live_rows[0]; + let names: Vec<_> = batch + .schema() + .fields() + .iter() + .map(|field| field.name().clone()) + .collect(); + assert_eq!( + names, + super::state::manifest_schema() + .fields() + .iter() + .map(|field| field.name().clone()) + .collect::>(), + "the publish scan carries every logical column, the content columns included" + ); +} + +/// `ManifestChange::SchemaContract` replaces the live row in the same commit as its table rows; +/// a publish without one carries the row forward; the pre-replacement version still answers +/// with the old contract; two replacements in one batch are refused. +#[tokio::test] +async fn publish_replaces_the_schema_contract_row_and_carries_it_forward() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let catalog = build_test_catalog(); + let genesis_contract = SchemaContractRow::for_test_catalog(&catalog).unwrap(); + let mut mc = ManifestCoordinator::init(uri, &catalog).await.unwrap(); + let genesis_version = mc.version(); + let person_entry = mc.snapshot().dataset("node:Person").unwrap().clone(); + let person_update = append_person_and_make_update(uri, &person_entry, "Ann").await; + + let replacement = replacement_contract(); + let replaced_at = mc + .commit_changes(&[ + ManifestChange::Update(person_update), + ManifestChange::SchemaContract(replacement.clone()), + ]) + .await + .unwrap(); + assert_eq!( + mc.known_state.schema_contract.as_ref(), + Some(&replacement.head), + "the publish fold reflects the replaced row without a re-scan" + ); + assert_eq!(mc.read_schema_contract().await.unwrap(), replacement); + assert_eq!( + mc.snapshot().dataset("node:Person").unwrap().entity_count, + 1, + "the table row and the contract land in one commit" + ); + + let carried = append_person_and_make_update(uri, &person_entry, "Bob").await; + let carried_at = mc + .commit_changes(&[ManifestChange::Update(carried)]) + .await + .unwrap(); + assert!(carried_at > replaced_at); + assert_eq!( + mc.known_state.schema_contract.as_ref(), + Some(&replacement.head) + ); + assert_eq!(mc.read_schema_contract().await.unwrap(), replacement); + let ds = open_manifest_dataset(uri, None).await.unwrap(); + assert_eq!(schema_contract_row_count(&ds).await, 1); + + assert_eq!( + ManifestCoordinator::read_schema_contract_at(uri, None, genesis_version) + .await + .unwrap(), + genesis_contract, + "time travel below the replacement reads the contract of that version" + ); + + let reopened = ManifestCoordinator::open(uri).await.unwrap(); + assert_eq!( + reopened.known_state.schema_contract.as_ref(), + Some(&replacement.head) + ); + + let twice = mc + .commit_changes(&[ + ManifestChange::SchemaContract(replacement.clone()), + ManifestChange::SchemaContract(genesis_contract.clone()), + ]) + .await + .expect_err("two contract replacements in one batch must be refused") + .to_string(); + assert!(twice.contains("replaced twice"), "{twice}"); + assert_eq!(mc.version(), carried_at, "a refused batch advances nothing"); +} + +#[tokio::test] +async fn publisher_reuses_only_its_exact_bounded_published_rows() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let catalog = build_test_catalog(); + let coordinator = ManifestCoordinator::init(uri, &catalog).await.unwrap(); + let publisher = GraphNamespacePublisher::new(uri, None); + let mut contract = coordinator.read_schema_contract().await.unwrap(); + for (round, expected_scans) in [1, 0, 1, 0, 1, 1].into_iter().enumerate() { + if round == 2 { + contract.source = format!("\n{}\n", contract.source); + contract.ir = format!("\n{}\n", contract.ir); + let foreign = GraphNamespacePublisher::new(uri, None); + foreign + .publish( + &[ManifestChange::SchemaContract(contract.clone())], + &HashMap::new(), + None, + ) + .await + .unwrap(); + } + if round == 4 { + contract.source = format!( + "{}{}", + " ".repeat(super::publisher::PUBLISHED_ROWS_CACHE_BYTES + 1024), + contract.source + ); + publisher + .publish( + &[ManifestChange::SchemaContract(contract.clone())], + &HashMap::new(), + None, + ) + .await + .unwrap(); + } + let intent = LineageIntent { + graph_commit_id: ulid::Ulid::new().to_string(), + branch: None, + actor_id: None, + merged_parent_commit_id: None, + created_at: 0, + }; + let probes = crate::instrumentation::QueryIoProbes::default(); + let scans = probes.manifest_scan_count.clone(); + let outcome = crate::instrumentation::with_query_io_probes( + probes, + publisher.publish(&[], &HashMap::new(), Some(&intent)), + ) + .await + .unwrap(); + assert_eq!( + scans.load(std::sync::atomic::Ordering::Relaxed), + expected_scans, + "round {round}: cold, warm, foreign replacement, warm, oversized, oversized" + ); + let scan = read_publish_scan(&outcome.dataset).await.unwrap(); + assert_eq!(scan.lineage_rows.len(), round + 2); + assert_eq!(scan.graph_heads.get("main"), Some(&intent.graph_commit_id)); + let reopened = ManifestCoordinator::open(uri).await.unwrap(); + assert_eq!(reopened.read_schema_contract().await.unwrap(), contract); + assert_eq!(schema_contract_row_count(&outcome.dataset).await, 1); + } +} + +/// The content read checks the row against the head the caller's state +/// folded from the same version; a disagreement is an error, never a silent +/// pick of one side. +#[tokio::test] +async fn schema_contract_read_refuses_a_row_that_disagrees_with_the_fold() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let catalog = build_test_catalog(); + let mut mc = ManifestCoordinator::init(uri, &catalog).await.unwrap(); + mc.known_state.schema_contract = Some(replacement_contract().head); + let error = mc + .read_schema_contract() + .await + .expect_err("a folded head that differs from the row must be refused") + .to_string(); + assert!(error.contains("disagrees with the folded state"), "{error}"); +} + +/// A live-read refresh after another handle's schema apply observes the +/// replaced contract, and the refresh's projection folds it. +#[tokio::test] +async fn refresh_observes_a_schema_contract_replaced_by_another_handle() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let catalog = build_test_catalog(); + let genesis_contract = SchemaContractRow::for_test_catalog(&catalog).unwrap(); + ManifestCoordinator::init(uri, &catalog).await.unwrap(); + let control_session = crate::lance_access::control_session(); + let (mut reader, _) = ManifestCoordinator::open_with_lineage(uri, None, &control_session) + .await + .unwrap(); + assert_eq!( + reader.known_state.schema_contract.as_ref(), + Some(&genesis_contract.head) + ); + + let mut writer = ManifestCoordinator::open(uri).await.unwrap(); + let replacement = replacement_contract(); + writer + .commit_changes(&[ManifestChange::SchemaContract(replacement.clone())]) + .await + .unwrap(); + + reader.refresh_with_lineage().await.unwrap(); + assert_eq!( + reader.known_state.schema_contract.as_ref(), + Some(&replacement.head) + ); + assert_eq!(reader.read_schema_contract().await.unwrap(), replacement); + let refreshed = reader.refresh_for_live_read(|_| true).await.unwrap(); + assert!(refreshed.is_none()); + assert_eq!( + reader.known_state.schema_contract.as_ref(), + Some(&replacement.head) + ); +} + +#[tokio::test] +async fn cold_contract_capture_scans_once_and_refuses_reserved_id_aliases() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let catalog = build_test_catalog(); + let mut writer = ManifestCoordinator::init(uri, &catalog).await.unwrap(); + let mut expected = writer.read_schema_contract().await.unwrap(); + expected.source = format!("\n{}\n", expected.source); + expected.ir = format!("\n{}\n", expected.ir); + writer + .commit_changes(&[ManifestChange::SchemaContract(expected.clone())]) + .await + .unwrap(); + let session = crate::lance_access::control_session(); + let probes = crate::instrumentation::QueryIoProbes::default(); + let scans = probes.manifest_scan_count.clone(); + let (reader, lineage, contract) = crate::instrumentation::with_query_io_probes( + probes, + ManifestCoordinator::open_with_lineage_and_contract(uri, None, &session), + ) + .await + .unwrap(); + assert_eq!(scans.load(std::sync::atomic::Ordering::Relaxed), 1); + assert_eq!(contract.unwrap(), expected); + assert_eq!(reader.snapshot().schema_contract(), Some(&expected.head)); + assert_eq!(reader.version(), writer.version()); + let dataset = open_manifest_dataset(uri, None).await.unwrap(); + assert_eq!(lineage, read_graph_lineage(&dataset).await.unwrap().0); + + let rows = read_publish_scan(&dataset).await.unwrap().live_rows; + let alias = relabelled_manifest_row(&rows, SCHEMA_CONTRACT_OBJECT_ID); + let mut columns = alias.columns().to_vec(); + columns[alias.schema().index_of("object_type").unwrap()] = + Arc::new(StringArray::from(vec!["unknown_extension"])); + let alias = RecordBatch::try_new(alias.schema(), columns).unwrap(); + let schema = super::record::manifest_storage_schema(dataset.schema().metadata.clone()).unwrap(); + let stored = super::record::compact_to_storage(&alias, &schema).unwrap(); + InsertBuilder::new(Arc::new(dataset)) + .with_params(&WriteParams { + mode: WriteMode::Append, + skip_auto_cleanup: true, + ..Default::default() + }) + .execute(vec![stored]) + .await + .unwrap(); + let error = ManifestCoordinator::open_with_lineage_and_contract(uri, None, &session) + .await + .err() + .expect("reserved schema-contract id must prevent catalog construction"); + let OmniError::Manifest(error) = error else { + panic!("expected a manifest integrity error, got: {error}"); + }; + assert_eq!(error.kind, crate::error::ManifestErrorKind::Internal); + assert_eq!( + error.message, + "manifest row 'schema_contract' has object_type 'unknown_extension'" + ); +} + +#[tokio::test] +async fn prepared_contract_capture_retries_only_replaced_unreadable_data() { + for replace in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + let writer = ManifestCoordinator::init(root, &build_test_catalog()) + .await + .unwrap(); + let expected = writer.read_schema_contract().await.unwrap(); + let session = crate::lance_access::control_session(); + let prepared = ManifestCoordinator::prepare_open_with_contract(root, &session) + .await + .unwrap(); + let dataset = open_manifest_dataset(root, None).await.unwrap(); + let old_version = dataset.version().version; + let files = dataset + .get_fragments() + .iter() + .flat_map(|fragment| { + fragment + .metadata() + .files + .iter() + .map(|file| file.path.clone()) + }) + .collect::>(); + if replace { + let rows = read_publish_scan(&dataset).await.unwrap().live_rows; + let schema = + super::record::manifest_storage_schema(dataset.schema().metadata.clone()).unwrap(); + let stored = rows + .iter() + .map(|row| super::record::compact_to_storage(row, &schema).unwrap()) + .collect::>(); + let replacement = InsertBuilder::new(Arc::new(dataset)) + .with_params(&WriteParams { + mode: WriteMode::Overwrite, + skip_auto_cleanup: true, + ..Default::default() + }) + .execute(stored) + .await + .unwrap(); + assert!(replacement.version().version > old_version); + assert!(replacement.get_fragments().iter().all(|fragment| { + fragment + .metadata() + .files + .iter() + .all(|file| !files.contains(&file.path)) + })); + } + for file in files { + std::fs::remove_file(dir.path().join("__manifest/data").join(file)).unwrap(); + } + let probes = crate::instrumentation::QueryIoProbes::default(); + let opens = probes.internal_open_count.clone(); + let scans = probes.manifest_scan_count.clone(); + let result = crate::instrumentation::with_query_io_probes( + probes, + ManifestCoordinator::open_prepared_with_lineage_and_contract(root, prepared), + ) + .await; + assert_eq!(opens.load(std::sync::atomic::Ordering::Relaxed), 1); + assert_eq!( + scans.load(std::sync::atomic::Ordering::Relaxed), + if replace { 2 } else { 1 } + ); + if replace { + let (reader, _, contract) = result.unwrap(); + assert!(reader.version() > old_version); + assert_eq!(contract.unwrap(), expected); + } else { + assert!( + result.is_err(), + "unchanged unreadable data must remain an error" + ); + } + } +} + +#[tokio::test] +async fn prepared_contract_capture_rejects_another_root_before_scanning() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + ManifestCoordinator::init(root, &build_test_catalog()) + .await + .unwrap(); + let prepared = ManifestCoordinator::prepare_open_with_contract( + root, + &crate::lance_access::control_session(), + ) + .await + .unwrap(); + let probes = crate::instrumentation::QueryIoProbes::default(); + let scans = probes.manifest_scan_count.clone(); + let result = crate::instrumentation::with_query_io_probes( + probes, + ManifestCoordinator::open_prepared_with_lineage_and_contract("/another-root", prepared), + ) + .await; + assert!( + result + .err() + .unwrap() + .to_string() + .contains("different graph root") + ); + assert_eq!(scans.load(std::sync::atomic::Ordering::Relaxed), 0); +} + +/// Routine state scans leave contract content unread; cold admission projects it once. +#[tokio::test] +async fn state_scan_projection_leaves_the_content_columns_unread() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + ManifestCoordinator::init(uri, &build_test_catalog()) + .await + .unwrap(); + let ds = open_manifest_dataset(uri, None).await.unwrap(); + assert_eq!( + super::record::packed_projection(&ds, false), + ["object_id", "object_type", "record"] + ); + assert_eq!( + super::record::packed_projection(&ds, true), + [ + "object_id", + "object_type", + "record", + "schema_source", + "schema_ir" + ] + ); + let state = super::state::read_manifest_state(&ds).await.unwrap(); + assert!(state.schema_contract.is_some()); +} + /// What a reader projects out of a `__manifest` version through either stored shape: the /// `ManifestState` entries (as `Debug` text: `DatasetEntry` has no `PartialEq`) and heads, and /// the lineage scan's commits and heads; not the dataset version, which every rewrite changes. @@ -3534,3 +4068,611 @@ async fn logical_view( heads, ) } + +/// A retired same-name candidate can lack the wanted version. Continue to the +/// exact owner, but never accept a different commit or a missing owning manifest. +#[tokio::test] +async fn pinned_graph_commit_skips_absent_wrong_incarnation() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let mut mc = ManifestCoordinator::init(uri, &build_test_catalog()) + .await + .unwrap(); + let mut main = open_manifest_dataset(uri, None).await.unwrap(); + let base_version = main.version().version; + let old_native = "b1.00000000000000000000000001"; + let live_native = "b1.00000000000000000000000002"; + let old = crate::lance_clone::create_branch(&mut main, old_native, base_version) + .await + .unwrap(); + mc.delete_branch("b1").await.unwrap(); + drop(old); + crate::lance_clone::create_branch(&mut main, live_native, base_version) + .await + .unwrap(); + let mut live = ManifestCoordinator::open_at_branch(uri, "b1") + .await + .unwrap(); + let intent = LineageIntent { + graph_commit_id: "00000000000000000000000003".into(), + branch: Some("b1".into()), + actor_id: None, + merged_parent_commit_id: None, + created_at: 1, + }; + live.commit_changes_with_lineage(&[], &HashMap::new(), Some(&intent)) + .await + .unwrap(); + let graph = crate::commit_graph::CommitGraph::open_at_branch(uri, "b1") + .await + .unwrap(); + let commit = graph.get_commit(&intent.graph_commit_id).unwrap(); + assert!(commit.graph_manifest_version > base_version); + let absent_candidate = main + .checkout_version(lance::dataset::refs::Ref::Version( + Some(old_native.to_string()), + Some(commit.graph_manifest_version), + )) + .await + .expect_err("the retired candidate never published the requested version"); + assert!( + matches!(&absent_candidate, lance::Error::DatasetNotFound { .. }), + "{absent_candidate:?}" + ); + let pinned = ManifestCoordinator::pinned_graph_commit(uri, &commit) + .await + .expect("the retired candidate lacks this version; the live owner has it"); + assert_eq!( + pinned.dataset.manifest().branch.as_deref(), + Some(live_native) + ); + assert_eq!( + pinned.dataset.version().version, + commit.graph_manifest_version + ); + + let wrong = crate::commit_graph::GraphCommit { + graph_commit_id: "00000000000000000000000004".into(), + ..commit.clone() + }; + let error = ManifestCoordinator::pinned_graph_commit(uri, &wrong) + .await + .err() + .expect("a same-version manifest must match the full requested graph commit"); + assert!( + matches!( + &error, + OmniError::Manifest(ManifestError { + kind: crate::error::ManifestErrorKind::NotFound, + .. + }) + ), + "{error:?}" + ); + assert!( + error + .to_string() + .contains("no matching retained native manifest") + ); + + let store = pinned.dataset.object_store(None).await.unwrap(); + store + .delete(&pinned.dataset.manifest_location().path) + .await + .unwrap(); + let error = ManifestCoordinator::pinned_graph_commit(uri, &commit) + .await + .err() + .expect("the true retained manifest is missing; no candidate may substitute"); + assert!( + matches!( + &error, + OmniError::Manifest(ManifestError { + kind: crate::error::ManifestErrorKind::NotFound, + .. + }) + ), + "{error:?}" + ); + assert!( + error + .to_string() + .contains("no matching retained native manifest") + ); +} + +#[derive(Debug)] +struct SmallScanProbeMarker; + +impl lance::io::WrappingObjectStore for SmallScanProbeMarker { + fn wrap( + &self, + _: &str, + original: Arc, + ) -> Arc { + original + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct SmallScanIo { + requests: u64, + bytes: u64, + writes: u64, +} + +fn drain_small_scan_io(stores: &crate::instrumentation::ProbedStores) -> SmallScanIo { + let mut total = SmallScanIo { + requests: 0, + bytes: 0, + writes: 0, + }; + for store in stores.stores() { + let stats = store.io_stats_incremental(); + total.requests += stats.read_iops; + total.bytes += stats.read_bytes; + total.writes += stats.write_iops; + } + total +} + +fn schema_comment_noise(lines: usize) -> String { + let mut random = 0xa076_1d64_78bd_642fu64; + let mut text = String::new(); + for _ in 0..lines { + random ^= random << 13; + random ^= random >> 7; + random ^= random << 17; + use std::fmt::Write; + writeln!(&mut text, "// {random:016x}").unwrap(); + } + text +} + +async fn small_scan_catalog_fixture(root: &str, large: bool) -> (SchemaContractRow, u64) { + let catalog = build_test_catalog(); + let mut contract = SchemaContractRow::for_test_catalog(&catalog).unwrap(); + contract.source = format!("{}\n", test_schema_source()); + contract + .source + .push_str(&schema_comment_noise(if large { 32_768 } else { 256 })); + let attempt = GenesisManifestAttempt::mint(catalog.system_columns).unwrap(); + let dataset = ManifestCoordinator::init_commit( + root, + &catalog, + &contract, + &crate::lance_access::control_session(), + &attempt, + ) + .await + .unwrap(); + assert!( + dataset + .manifest() + .fragments + .iter() + .all(|fragment| fragment.deletion_file.is_none() && fragment.overlays.is_empty()) + ); + let total = dataset + .manifest() + .fragments + .iter() + .flat_map(|f| &f.files) + .map(|file| file.file_size_bytes.get().unwrap().get()) + .sum::(); + if large { + assert!( + total > 65_536, + "fixture did not cross fallback boundary: {total}" + ); + } else { + assert!( + (4097..=65_536).contains(&total), + "fixture missed small-read window: {total}" + ); + } + (contract, total) +} + +async fn measured_catalog_scan( + root: &str, + expected: &SchemaContractRow, + optimized: bool, +) -> (SmallScanIo, Vec) { + use std::sync::atomic::Ordering; + let probes = crate::instrumentation::QueryIoProbes { + manifest_wrapper: Some(Arc::new(SmallScanProbeMarker)), + ..Default::default() + }; + let stores = probes.manifest_stores.clone(); + let opens = probes.internal_open_count.clone(); + let scans = probes.manifest_scan_count.clone(); + crate::instrumentation::with_query_io_probes(probes, async { + let session = crate::lance_access::control_session(); + let dataset = super::layout::open_manifest_dataset_with_session(root, None, &session) + .await + .unwrap(); + assert_eq!(dataset.object_store(None).await.unwrap().block_size(), 4096); + let original = dataset.object_store(None).await.unwrap(); + let _ = drain_small_scan_io(&stores); + opens.store(0, Ordering::Relaxed); + scans.store(0, Ordering::Relaxed); + + if optimized { + let (state, _, lineage, row) = + super::state::read_manifest_projection_with_contract(&dataset) + .await + .unwrap(); + assert_eq!(row.unwrap(), *expected); + assert_eq!(state.schema_contract.as_ref(), Some(&expected.head)); + assert_eq!(state.version, dataset.version().version); + assert!(!lineage.is_empty()); + } else { + let scan = super::state::read_publish_scan(&dataset).await.unwrap(); + assert_eq!(scan.schema_contract.as_ref(), Some(&expected.head)); + assert!(!scan.lineage_rows.is_empty()); + let mut rows = 0; + for batch in scan.live_rows { + use arrow_array::Array; + let ids = batch + .column_by_name("object_id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let sources = batch + .column_by_name("schema_source") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let irs = batch + .column_by_name("schema_ir") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for row in 0..batch.num_rows() { + if ids.value(row) == SCHEMA_CONTRACT_OBJECT_ID { + assert!(!sources.is_null(row) && !irs.is_null(row)); + assert_eq!(sources.value(row), expected.source); + assert_eq!(irs.value(row), expected.ir); + rows += 1; + } + } + } + assert_eq!(rows, 1); + } + assert_eq!( + opens.load(Ordering::Relaxed), + 0, + "scan helper must not reopen __manifest" + ); + assert_eq!(scans.load(Ordering::Relaxed), 1); + assert!(Arc::ptr_eq( + &original, + &dataset.object_store(None).await.unwrap() + )); + assert_eq!( + original.block_size(), + 4096, + "the held control dataset keeps its policy" + ); + let io = drain_small_scan_io(&stores); + let mut blocks = stores + .stores() + .iter() + .map(|store| store.block_size()) + .collect::>(); + blocks.sort_unstable(); + blocks.dedup(); + (io, blocks) + }) + .await +} + +#[tokio::test] +async fn cold_catalog_small_scan_reduces_real_reads_and_large_scan_is_unchanged() { + for large in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + let (expected, encoded_bytes) = small_scan_catalog_fixture(root, large).await; + let (baseline, baseline_blocks) = measured_catalog_scan(root, &expected, false).await; + let (actual, blocks) = measured_catalog_scan(root, &expected, true).await; + let (again, again_blocks) = measured_catalog_scan(root, &expected, true).await; + assert_eq!(baseline_blocks, [4096]); + assert_eq!(baseline.writes, 0); + assert_eq!(actual.writes, 0); + assert!( + actual.requests > 0, + "rebound store IO must remain in probe totals" + ); + assert_eq!( + actual, again, + "a new cold scan must read again, with no persistent byte reuse" + ); + assert_eq!(blocks, again_blocks); + if large { + assert_eq!( + blocks, + [4096], + "large file must retain the original tail/gap policy" + ); + assert_eq!( + actual, baseline, + "large fallback must preserve physical requests AND bytes" + ); + } else { + assert_eq!( + blocks, + [4096, 65_536], + "both original and rebound stores must be accounted" + ); + assert!( + actual.requests < baseline.requests, + "small physical reads did not fall: original={baseline:?}, optimized={actual:?}" + ); + assert_eq!( + actual.bytes, encoded_bytes, + "eligible encoded data should be fetched once, without overlapping ranges" + ); + } + } +} + +#[tokio::test] +async fn cold_catalog_small_scan_failure_does_not_poison_a_fresh_retry() { + use object_store::ObjectStoreExt; + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + let (expected, _) = small_scan_catalog_fixture(root, false).await; + let probes = crate::instrumentation::QueryIoProbes { + manifest_wrapper: Some(Arc::new(SmallScanProbeMarker)), + ..Default::default() + }; + let stores = probes.manifest_stores.clone(); + crate::instrumentation::with_query_io_probes(probes, async { + let dataset = super::layout::open_manifest_dataset_with_session( + root, + None, + &crate::lance_access::control_session(), + ) + .await + .unwrap(); + let files = dataset + .manifest() + .fragments + .iter() + .flat_map(|f| &f.files) + .collect::>(); + assert_eq!( + files.len(), + 1, + "fault fixture should have exactly one physical data file" + ); + let file_path = object_store::path::Path::from_filesystem_path( + dir.path() + .join("__manifest") + .join("data") + .join(&files[0].path), + ) + .unwrap(); + let store = dataset.object_store(None).await.unwrap(); + let saved = store + .inner + .get(&file_path) + .await + .unwrap() + .bytes() + .await + .unwrap(); + store + .inner + .put(&file_path, saved.slice(..saved.len() - 1).into()) + .await + .unwrap(); + let _ = drain_small_scan_io(&stores); + let failure = super::state::read_manifest_projection_with_contract(&dataset).await; + assert!(failure.is_err(), "captured-size mismatch must fail closed"); + let failed_io = drain_small_scan_io(&stores); + assert!( + failed_io.requests > 0, + "failure must reach the tracked physical backend" + ); + store.inner.put(&file_path, saved.into()).await.unwrap(); + let _ = drain_small_scan_io(&stores); + let (_, _, _, contract) = super::state::read_manifest_projection_with_contract(&dataset) + .await + .unwrap(); + assert_eq!(contract.unwrap(), expected); + let retry_io = drain_small_scan_io(&stores); + assert!( + retry_io.requests > 0, + "the retry must issue a fresh physical read" + ); + assert_eq!(retry_io.writes, 0); + assert_eq!(dataset.object_store(None).await.unwrap().block_size(), 4096); + }) + .await; +} + +async fn small_read_fixture(uri: &str, fragments: usize) -> Dataset { + use arrow_array::{ArrayRef, Int32Array, RecordBatch, RecordBatchIterator}; + use lance::dataset::{WriteMode, WriteParams}; + for fragment in 0..fragments { + let batch = RecordBatch::try_from_iter([( + "value", + Arc::new(Int32Array::from(vec![1, 2, 3, 4])) as ArrayRef, + )]) + .unwrap(); + let schema = batch.schema(); + Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + uri, + Some(WriteParams { + mode: if fragment == 0 { + WriteMode::Create + } else { + WriteMode::Append + }, + skip_auto_cleanup: true, + ..Default::default() + }), + ) + .await + .unwrap(); + } + lance::dataset::builder::DatasetBuilder::from_uri(uri) + .with_store_params(Default::default()) + .with_session(Arc::new(lance::session::Session::new( + 0, + 0, + Arc::new(lance::io::ObjectStoreRegistry::default()), + ))) + .load() + .await + .unwrap() +} + +#[tokio::test] +async fn manifest_scan_read_budget_is_inclusive_and_covers_all_files() { + for (sizes, eligible) in [ + (vec![65_536u64], true), + (vec![65_537u64], false), + (vec![32_768u64, 32_768], true), + (vec![32_768u64, 32_769], false), + ] { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("__manifest"); + let dataset = small_read_fixture(uri.to_str().unwrap(), sizes.len()).await; + let files = dataset + .manifest() + .fragments + .iter() + .flat_map(|fragment| &fragment.files) + .collect::>(); + assert_eq!( + files.len(), + sizes.len(), + "fixture must have distinct physical files" + ); + for (file, size) in files.iter().zip(&sizes) { + file.file_size_bytes + .set(std::num::NonZeroU64::new(*size).unwrap()); + } + let original = dataset.object_store(None).await.unwrap(); + assert_eq!(original.block_size(), 4096); + let _ = original.io_stats_incremental(); + let scan = crate::instrumentation::manifest_scan_dataset(&dataset) + .await + .unwrap(); + let scan_store = scan.object_store(None).await.unwrap(); + assert_eq!( + Arc::ptr_eq(&original, &scan_store), + !eligible, + "sizes={sizes:?}" + ); + assert_eq!( + scan_store.block_size(), + if eligible { 65_536 } else { 4096 } + ); + assert_eq!(dataset.object_store(None).await.unwrap().block_size(), 4096); + assert_eq!(scan.version().version, dataset.version().version); + assert_eq!(original.io_stats_incremental().read_iops, 0); + if eligible { + assert_eq!( + scan_store.io_stats_incremental().read_iops, + 0, + "store rebinding must not reopen a dataset or read a file" + ); + } + } +} + +#[tokio::test] +async fn manifest_scan_preserves_existing_large_block_and_real_deletion_layout() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("__manifest"); + let uri = uri.to_str().unwrap(); + let mut dataset = small_read_fixture(uri, 1).await; + for block_size in [65_536usize, 131_072] { + let configured = lance::dataset::builder::DatasetBuilder::from_uri(uri) + .with_store_params(lance::io::ObjectStoreParams { + block_size: Some(block_size), + ..Default::default() + }) + .with_session(crate::lance_access::control_session()) + .load() + .await + .unwrap(); + let before = configured.object_store(None).await.unwrap(); + let after = crate::instrumentation::manifest_scan_dataset(&configured) + .await + .unwrap(); + assert!(Arc::ptr_eq( + &before, + &after.object_store(None).await.unwrap() + )); + assert_eq!( + after.object_store(None).await.unwrap().block_size(), + block_size + ); + } + + dataset.delete("value = 1").await.unwrap(); + assert!( + dataset + .manifest() + .fragments + .iter() + .any(|f| f.deletion_file.is_some()) + ); + let before = dataset.object_store(None).await.unwrap(); + let scan = crate::instrumentation::manifest_scan_dataset(&dataset) + .await + .unwrap(); + assert!(Arc::ptr_eq( + &before, + &scan.object_store(None).await.unwrap() + )); + assert_eq!(scan.scan().try_into_batch().await.unwrap().num_rows(), 3); +} + +#[tokio::test] +async fn manifest_scan_keeps_raw_memory_binding_readable() { + use arrow_array::{ArrayRef, Int32Array, RecordBatch, RecordBatchIterator}; + for scheme in ["memory://", "memory:/", "MeMoRy://"] { + let batch = RecordBatch::try_from_iter([( + "value", + Arc::new(Int32Array::from(vec![31, 47])) as ArrayRef, + )]) + .unwrap(); + let schema = batch.schema(); + let dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &format!("{scheme}raw-memory-{}", ulid::Ulid::new()), + Some(lance::dataset::WriteParams { + store_params: Some(Default::default()), + ..Default::default() + }), + ) + .await + .unwrap(); + let before = dataset.object_store(None).await.unwrap(); + let scan = crate::instrumentation::manifest_scan_dataset(&dataset) + .await + .unwrap(); + assert!(Arc::ptr_eq( + &before, + &scan.object_store(None).await.unwrap() + )); + let actual = scan.scan().try_into_batch().await.unwrap(); + assert_eq!(actual.num_rows(), 2); + let values = actual + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(values.values().as_ref(), &[31, 47]); + } +} diff --git a/crates/omnigraph-cli/tests/cli_schema_config.rs b/crates/omnigraph-cli/tests/cli_schema_config.rs index d70042962..f6d524a4d 100644 --- a/crates/omnigraph-cli/tests/cli_schema_config.rs +++ b/crates/omnigraph-cli/tests/cli_schema_config.rs @@ -130,7 +130,7 @@ fn init_creates_graph_successfully_on_missing_local_directory() { let stdout = stdout_string(&output); assert!(stdout.contains("initialized")); - assert!(graph.join("_schema.pg").exists()); + assert!(!graph.join("_schema.pg").exists()); assert!(graph.join("__manifest").exists()); // RFC-008 stage 3: init no longer scaffolds the legacy config file. assert!(!temp.path().join("omnigraph.yaml").exists()); diff --git a/crates/omnigraph-cli/tests/crossversion_upgrade.rs b/crates/omnigraph-cli/tests/crossversion_upgrade.rs index 1fc1b40aa..91e3e05e1 100644 --- a/crates/omnigraph-cli/tests/crossversion_upgrade.rs +++ b/crates/omnigraph-cli/tests/crossversion_upgrade.rs @@ -821,7 +821,7 @@ fn current_v10_refuses_and_rebuilds_genuine_v6_and_v6_refuses_v10() { /// The guide's own preflight spelling and the default both report /// already_current on a graph this binary created. #[test] -fn current_binary_reports_already_current_on_a_fresh_graph() { +fn storage_upgrade_current_binary_reports_already_current_on_a_fresh_graph() { let temp = tempdir().unwrap(); let graph = temp.path().join("fresh-current.omni"); let uri = graph.to_str().unwrap(); @@ -832,14 +832,14 @@ fn current_binary_reports_already_current_on_a_fresh_graph() { for args in [ vec!["upgrade", uri, "--check", "--json"], vec!["upgrade", uri, "--json"], - vec!["upgrade", uri, "--check", "--to-format", "11", "--json"], + vec!["upgrade", uri, "--check", "--to-format", "13", "--json"], ] { let report = support::parse_stdout_json(&output_success(cli().args(&args))); assert_eq!(report["outcome"], "already_current", "{args:?}"); - assert_eq!(report["observed_format"], 12, "{args:?}"); + assert_eq!(report["observed_format"], 13, "{args:?}"); } - for lower in ["7", "8", "10"] { + for lower in ["7", "8", "10", "11"] { let refused = support::parse_stdout_json(&output_failure(cli().args([ "upgrade", uri, @@ -1296,7 +1296,17 @@ query revise($body: String) { update Doc set { body: $body } where slug = "dl-ba } fn migration_bin(variable: &str, version: &str) -> Option { - let Some(path) = std::env::var_os(variable).map(PathBuf::from) else { + let fixtures = + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/storage-upgrade-binaries"); + let local = fixtures.join(match variable { + "OMNIGRAPH_V09_BIN" => "v0.9.0/omnigraph", + "OMNIGRAPH_V6_BIN" => "v0.10.0/omnigraph", + _ => panic!("unknown migration predecessor {variable}"), + }); + let selected = std::env::var_os(variable) + .map(PathBuf::from) + .or_else(|| fixtures.exists().then_some(local)); + let Some(path) = selected else { assert!( std::env::var_os("OMNIGRAPH_REQUIRE_STORAGE_UPGRADE_TESTS").is_none(), "required storage migration predecessor {variable} is unset" @@ -1309,6 +1319,10 @@ fn migration_bin(variable: &str, version: &str) -> Option { "{variable} is not a binary: {}", path.display() ); + eprintln!( + "storage migration predecessor {version}: {}", + path.display() + ); let output = run_old(&path, &["version"]); assert_ok("migration predecessor version", &output); assert_eq!( @@ -1568,6 +1582,22 @@ query vectors($q: Vector(4)) { payloads(&before), "storage migration must preserve every table object without adding table objects" ); + let immutable_payloads = |files: &std::collections::BTreeMap>| { + payloads(files) + .into_iter() + .filter(|(path, _)| { + !path.components().any(|part| { + ["_versions", "_transactions", "_refs"] + .iter() + .any(|name| part.as_os_str() == std::ffi::OsStr::new(name)) + }) + }) + .collect::>() + }; + let original_payloads = immutable_payloads(&before); + for kind in ["nodes", "edges"] { + assert!(original_payloads.keys().any(|path| path.starts_with(kind))); + } for check_mode in [true, false] { let mut command = cli(); command.args(["upgrade", uri, "--to-format", "8", "--json"]); @@ -1583,25 +1613,32 @@ query vectors($q: Vector(4)) { default_route.arg("--check"); } let completed = support::parse_stdout_json(&output_success(&mut default_route)); - assert_eq!(completed["target_format"], 11); + assert_eq!(completed["target_format"], 13); assert_eq!(completed["target_defaulted"], true); if check_mode { assert_eq!( completed["outcome"], "check_passed", - "the default route reaches v11 through the live branches: {completed}" + "the default route reaches v13 through the live branches: {completed}" ); assert_eq!( graph_files(&graph), after, - "the v11 check must be effect-free" + "the v13 check must be effect-free" ); } else { assert_eq!( completed["outcome"], "completed", - "the default route takes a branched graph to v11: {completed}" + "the default route takes a branched graph to v13: {completed}" ); - assert_eq!(completed["completed_handlers"].as_array().unwrap().len(), 2); + assert_eq!(completed["completed_handlers"].as_array().unwrap().len(), 3); after = graph_files(&graph); + assert_eq!( + immutable_payloads(&after), + original_payloads, + "the complete v6-to-v13 route must preserve every immutable table object" + ); + assert_eq!(completed["work"]["payload_bytes_copied"], 0); + assert_eq!(completed["work"]["payload_bytes_rewritten"], 0); } let mut downgrade = cli(); downgrade.args(["upgrade", uri, "--to-format", "7", "--json"]); @@ -1616,6 +1653,25 @@ query vectors($q: Vector(4)) { "downgrade refusal must be effect-free" ); } + for name in ["_schema.pg", "_schema.ir.json", "__schema_state.json"] { + assert!(!graph.join(name).exists(), "converted graph retains {name}"); + } + tokio::runtime::Runtime::new().unwrap().block_on(async { + let db = Omnigraph::open_read_only(uri).await.unwrap(); + for branch in ["main", "review"] { + db.sync_branch(branch).await.unwrap(); + assert_eq!( + db.schema_source().as_bytes(), + before[Path::new("_schema.pg")] + ); + let expected_ir: serde_json::Value = + serde_json::from_slice(&before[Path::new("_schema.ir.json")]).unwrap(); + assert_eq!( + serde_json::to_value(db.catalog().bound_schema_ir().unwrap()).unwrap(), + expected_ir + ); + } + }); let refused = support::parse_stdout_json(&output_failure(cli().args([ "upgrade", uri, @@ -1631,7 +1687,7 @@ query vectors($q: Vector(4)) { .unwrap() .iter() .any(|finding| finding["code"] == "target_below_stamp"), - "v8 is below the served floor once the graph is at v11: {refused}" + "v8 is below the served floor once the graph is at v13: {refused}" ); assert_eq!( graph_files(&graph), @@ -1835,10 +1891,10 @@ query vectors($q: Vector(4)) { }; assert_eq!( stamps(), - [11, 11], - "the upgrade stops at v11 on every live branch" + [13, 13], + "the upgrade converts every live branch to v13" ); - for (branch, after_rebuild) in [("main", [12, 11]), ("review", [12, 12])] { + for (branch, after_rebuild) in [("main", [13, 13]), ("review", [13, 13])] { output_success(cli().args([ "rebuild-full-text-indexes", uri, @@ -1849,7 +1905,7 @@ query vectors($q: Vector(4)) { assert_eq!( stamps(), after_rebuild, - "the rebuild is {branch}'s first publish, so {branch} alone converts to v12" + "the rebuild preserves stamp 13 on {branch} and every other live branch" ); assert_eq!( current_query("--branch", branch, "terms"), diff --git a/crates/omnigraph-cluster/src/tests.rs b/crates/omnigraph-cluster/src/tests.rs index 951424a97..53af28f3b 100644 --- a/crates/omnigraph-cluster/src/tests.rs +++ b/crates/omnigraph-cluster/src/tests.rs @@ -2977,11 +2977,9 @@ async fn apply_creates_graph_and_unblocks_dependents() { async fn apply_create_failure_blocks_dependents_and_keeps_sidecar() { let dir = fixture(); write_state_resources(dir.path(), &[]); - // Make the init fail its strict preflight: a junk _schema.pg already - // sits at the derived root (the engine refuses to overwrite it). let root = dir.path().join(CLUSTER_GRAPHS_DIR).join("knowledge.omni"); fs::create_dir_all(&root).unwrap(); - fs::write(root.join("_schema.pg"), "junk").unwrap(); + fs::write(root.join("__init_claim.json"), "incomplete initializer").unwrap(); let out = apply_config_dir(dir.path()).await; assert!(!out.ok); diff --git a/crates/omnigraph-cluster/tests/failpoints.rs b/crates/omnigraph-cluster/tests/failpoints.rs index 29a1c5dcb..0c3c6a5ee 100644 --- a/crates/omnigraph-cluster/tests/failpoints.rs +++ b/crates/omnigraph-cluster/tests/failpoints.rs @@ -13,7 +13,6 @@ use std::fs; use std::path::{Path, PathBuf}; use omnigraph::db::Omnigraph; -use omnigraph::error::OmniError; use omnigraph_cluster::seams::FailScenario; use omnigraph_cluster::{ ApplyOptions, apply_config_dir, apply_config_dir_with_options, approve_config_dir, @@ -477,7 +476,8 @@ async fn schema_apply_error_before_graph_movement_removes_sidecar() { fs::write(dir.path().join("people.pg"), SCHEMA_V2).unwrap(); { - let _failpoint = omnigraph::seams::catalog::SCHEMA_APPLY_BEFORE_STAGING_WRITE.fire_always(); + let _failpoint = + omnigraph::seams::catalog::GRAPH_PUBLISH_BEFORE_COMMIT_APPEND.fire_always(); let out = Box::pin(apply_config_dir(dir.path())).await; assert!(!out.ok); assert!( @@ -512,7 +512,6 @@ async fn schema_apply_error_after_graph_movement_keeps_sidecar() { let dir = fixture(); converge_with_live_graph(dir.path()).await; let uri = dir.path().join("graphs/knowledge.omni"); - let pre_schema_source = fs::read_to_string(uri.join("_schema.pg")).unwrap(); fs::write(dir.path().join("people.pg"), SCHEMA_V2).unwrap(); let desired = validate_config_dir(dir.path()); let v2_digest = desired.resource_digests["schema.knowledge"].clone(); @@ -529,28 +528,10 @@ async fn schema_apply_error_after_graph_movement_keeps_sidecar() { "{:?}", out.diagnostics ); - // Read-only opens cannot repair the torn manifest/schema authority and - // must refuse it. Inspect the physical source directly to prove that - // recovery has not rewritten it behind the read-only boundary. - let read_only_error = match Omnigraph::open_read_only(uri.to_string_lossy().as_ref()).await - { - Ok(_) => panic!("read-only open must refuse incomplete SchemaApply recovery"), - Err(error) => error, - }; - match read_only_error { - OmniError::RecoveryRequired { reason, .. } => { - assert!( - reason.contains("read-only open found SchemaApply manifest outcome") - && reason.contains("run a read-write open"), - "unexpected read-only recovery refusal: {reason}" - ); - } - other => panic!("expected RecoveryRequired from read-only open, got: {other}"), - } - assert_eq!( - fs::read_to_string(uri.join("_schema.pg")).unwrap(), - pre_schema_source - ); + let read_only = Omnigraph::open_read_only(uri.to_string_lossy().as_ref()) + .await + .unwrap(); + assert_eq!(read_only.schema_source().as_str(), SCHEMA_V2); let sidecars = recovery_sidecars(dir.path()); assert_eq!(sidecars.len(), 1, "{sidecars:?}"); let sidecar: serde_json::Value = diff --git a/crates/omnigraph-core/src/branch_control.rs b/crates/omnigraph-core/src/branch_control.rs index acbaad393..20f2bece4 100644 --- a/crates/omnigraph-core/src/branch_control.rs +++ b/crates/omnigraph-core/src/branch_control.rs @@ -110,12 +110,6 @@ pub async fn resolve_live_native_branch( crate::branch_names::resolve_native_branch(live.iter().map(String::as_str), logical) } -pub async fn schema_apply_locked(dataset: &Dataset) -> Result { - live_manifest_branches_matching(dataset, crate::branch_names::is_schema_apply_lock_branch) - .await - .map(|branches| !branches.is_empty()) -} - async fn live_manifest_branches_matching( dataset: &Dataset, matches: impl Fn(&str) -> bool, @@ -1467,30 +1461,18 @@ mod tests { assert_eq!(vanish.injected_failures(), 1); } - /// The scoped sentinel check is a writer fence: unrelated damaged refs - /// must not block it, but a damaged sentinel cannot mean "unlocked". Keep - /// legacy and generated lifetimes, retirement, and read races in one fixture. + /// Relevant damaged refs fail authority reads; unrelated refs do not. #[tokio::test] - async fn schema_lock_lookup_preserves_authority_while_ignoring_unrelated_refs() { + async fn live_branch_lookup_preserves_authority_while_ignoring_unrelated_refs() { let dir = tempfile::tempdir().unwrap(); let mut dataset = test_dataset(&dir).await; let version = dataset.version().version; - let sentinel = crate::branch_names::SCHEMA_APPLY_LOCK_BRANCH; + let logical = "guarded"; let generated = crate::branch_names::native_branch_name( - sentinel, + logical, &crate::branch_names::mint_incarnation(), ); - for logical in [sentinel.to_string(), format!("/{sentinel}")] { - let native = crate::branch_names::native_branch_name( - &logical, - &crate::branch_names::mint_incarnation(), - ); - assert!(crate::branch_names::is_schema_apply_lock_branch( - crate::branch_names::logical_branch_name(&native) - )); - } - dataset .create_branch("unrelated", version, None) .await @@ -1499,16 +1481,18 @@ mod tests { let store = dataset.object_store(None).await.unwrap(); let unrelated = lance::dataset::refs::branch_contents_path(&root, "unrelated"); store.put(&unrelated, b"{").await.unwrap(); - assert!( - !schema_apply_locked(&dataset).await.unwrap(), - "an unrelated malformed ref must not turn an idle schema into a lock error" + assert_eq!( + resolve_live_native_branch(&dataset, logical).await.unwrap(), + None, + "an unrelated malformed ref must not turn an absent branch into a lookup error" ); - for native in [sentinel.to_string(), generated] { + for native in [logical.to_string(), generated] { dataset.create_branch(&native, version, None).await.unwrap(); - assert!( - schema_apply_locked(&dataset).await.unwrap(), - "a live sentinel must block writers: {native}" + assert_eq!( + resolve_live_native_branch(&dataset, logical).await.unwrap(), + Some(native.clone()), + "a live ref must resolve: {native}" ); let transient = Arc::new(VanishingBranchRefFault::stale_head_size_once(&format!( @@ -1517,9 +1501,10 @@ mod tests { let wrapped = dataset.with_object_store_wrappers(vec![ Arc::clone(&transient) as Arc ]); - assert!( - schema_apply_locked(&wrapped).await.unwrap(), - "a torn sentinel read must retry without admitting a writer" + assert_eq!( + resolve_live_native_branch(&wrapped, logical).await.unwrap(), + Some(native.clone()), + "a torn ref read must retry without reporting the branch absent" ); assert_eq!(transient.injected_failures(), 1); @@ -1527,9 +1512,10 @@ mod tests { retire_branch_recoverably(&dataset, &native, &identity) .await .unwrap(); - assert!( - !schema_apply_locked(&dataset).await.unwrap(), - "a validated retired sentinel must no longer block writers: {native}" + assert_eq!( + resolve_live_native_branch(&dataset, logical).await.unwrap(), + None, + "a validated retired ref must no longer resolve: {native}" ); let retired = archived_manifest_branch(&dataset, &native) @@ -1556,9 +1542,9 @@ mod tests { .replace_metadata(&native, metadata) .await .unwrap(); - let error = schema_apply_locked(&dataset) + let error = resolve_live_native_branch(&dataset, logical) .await - .expect_err("malformed or mismatched sentinel retirement must fail closed"); + .expect_err("malformed or mismatched retirement must fail closed"); assert!(error.to_string().contains("retirement metadata"), "{error}"); dataset .branches() @@ -1575,13 +1561,14 @@ mod tests { let wrapped = dataset.with_object_store_wrappers(vec![ Arc::clone(&persistent) as Arc ]); - schema_apply_locked(&wrapped) + resolve_live_native_branch(&wrapped, logical) .await - .expect_err("an unreadable relevant sentinel must never be treated as absent"); + .expect_err("an unreadable relevant ref must never be treated as absent"); assert!(persistent.injected_failures() >= BRANCH_REF_READ_MAX_ATTEMPTS); assert_eq!(persistent.branch_lists(), 1); - assert!( - !schema_apply_locked(&dataset).await.unwrap(), + assert_eq!( + resolve_live_native_branch(&dataset, logical).await.unwrap(), + None, "the read-only failures must leave the validated retirement intact" ); } diff --git a/crates/omnigraph-core/src/branch_names.rs b/crates/omnigraph-core/src/branch_names.rs index 64e5ae7a0..36f58fdb0 100644 --- a/crates/omnigraph-core/src/branch_names.rs +++ b/crates/omnigraph-core/src/branch_names.rs @@ -177,12 +177,6 @@ pub fn is_merge_input_tag(name: &str) -> bool { matches!(merge_input_owner(name), Ok(Some(_))) } -pub const SCHEMA_APPLY_LOCK_BRANCH: &str = "__schema_apply_lock__"; - -pub fn is_schema_apply_lock_branch(name: &str) -> bool { - name.trim_start_matches('/') == SCHEMA_APPLY_LOCK_BRANCH -} - /// Resolve a logical branch to its single live native ref. /// /// `Ok(None)` means no incarnation exists. More than one live incarnation is a diff --git a/crates/omnigraph-core/src/instrumentation.rs b/crates/omnigraph-core/src/instrumentation.rs index 43f08414b..54fa18434 100644 --- a/crates/omnigraph-core/src/instrumentation.rs +++ b/crates/omnigraph-core/src/instrumentation.rs @@ -9,12 +9,12 @@ //! a task-local ([`QueryIoProbes`]) set by the test; production leaves it unset, //! so the open helpers attach nothing (one unset-`Option` check per open). //! - **omnigraph `StorageAdapter`** — [`CountingStorageAdapter`], a decorator that -//! counts per-method calls (the schema-contract reads on the query path). +//! counts per-method calls at the adapter boundary. //! - **branch merge** — [`MergeWriteProbes`] reports structural route counters //! and completed timing intervals without reading the clock when unset. //! //! The probes themselves only observe, and the decorator delegates every call. -//! The shared dataset opener also supplies the process control session when a +//! The shared dataset opener also supplies a zero-cache control session when a //! caller has no graph-scoped data session, so detached opens still reuse the //! process object-store registry without caching mutable metadata. `IOTracker` //! (the concrete counter) lives in tests via the `lance-io` dev-dependency; this @@ -1717,6 +1717,9 @@ pub enum VersionResolution { At(u64), } +mod small_manifest_reads; +pub use small_manifest_reads::manifest_scan_dataset; + /// Open a table pin (RFC 0067): a pin without a staged version opens its /// target, a staged pin above `last_linear_version` opens its detached /// version, and any older pin keeps the v10 twin rule for historical rows. @@ -1812,7 +1815,7 @@ pub async fn open_pinned_dataset( /// store). No wrapper (production) adds nothing. /// 3. A caller-provided graph data `Session` warms Lance's metadata/index /// caches across data-table opens. When absent (for example a detached -/// historical snapshot), the process-wide zero-cache +/// historical snapshot), a fresh zero-cache /// control session is attached instead. Every open therefore reuses the /// shared object-store registry/client pool without letting mutable control /// metadata become stale in a session cache. diff --git a/crates/omnigraph-core/src/instrumentation/small_manifest_reads.rs b/crates/omnigraph-core/src/instrumentation/small_manifest_reads.rs new file mode 100644 index 000000000..0afb27cf3 --- /dev/null +++ b/crates/omnigraph-core/src/instrumentation/small_manifest_reads.rs @@ -0,0 +1,427 @@ +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; + +use async_trait::async_trait; +use futures::stream::BoxStream; +use futures::{StreamExt, TryStreamExt}; +use lance::Dataset; +use lance_table::format::{RowDatasetVersionMeta, RowIdMeta}; +use object_store::path::Path; +use object_store::{ + CopyOptions, GetOptions, GetRange, GetResult, GetResultPayload, ListResult, MultipartUpload, + ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, +}; + +use crate::error::{OmniError, Result}; + +const MAX_SCAN_FILE_BYTES: u64 = 64 * 1024; + +/// A scan-local store for small manifest data files; larger or indirect layouts retain their reader. +pub async fn manifest_scan_dataset(dataset: &Dataset) -> Result { + let Some(files) = small_files(dataset) else { + return Ok(dataset.clone()); + }; + let original = dataset + .object_store(None) + .await + .map_err(OmniError::storage)?; + if original.block_size() >= MAX_SCAN_FILE_BYTES as usize { + return Ok(dataset.clone()); + } + let Some(mut params) = dataset.store_params().cloned() else { + return Ok(dataset.clone()); + }; + params.block_size = Some(MAX_SCAN_FILE_BYTES as usize); + let (store, _) = lance::io::ObjectStore::from_uri_and_params( + dataset.session().store_registry(), + dataset.uri(), + ¶ms, + ) + .await + .map_err(OmniError::storage)?; + if let Some(wrapper) = ¶ms.object_store_wrapper { + super::record_probed_store(wrapper, Arc::clone(&store)); + } + let data_path = dataset.branch_location().path.clone().join("data"); + let sizes = files + .into_iter() + .map(|(path, size)| (data_path.clone().join(path), size)) + .collect(); + let mut bounded = store.as_ref().clone(); + bounded.inner = Arc::new(BoundedManifestStore { + inner: Arc::clone(&store.inner), + sizes, + }); + Ok(dataset.with_object_store(Arc::new(bounded), Some(params))) +} + +fn small_files(dataset: &Dataset) -> Option> { + if url::Url::parse(dataset.uri()).is_ok_and(|uri| uri.scheme() == "memory") + || !dataset.manifest().base_paths.is_empty() + { + return None; + } + let mut files = HashMap::new(); + let mut total = 0u64; + for fragment in dataset.manifest().fragments.iter() { + if !fragment.overlays.is_empty() + || fragment.deletion_file.is_some() + || matches!(fragment.row_id_meta, Some(RowIdMeta::External(_))) + || matches!( + fragment.last_updated_at_version_meta, + Some(RowDatasetVersionMeta::External(_)) + ) + || matches!( + fragment.created_at_version_meta, + Some(RowDatasetVersionMeta::External(_)) + ) + { + return None; + } + for file in &fragment.files { + if file.base_id.is_some() || files.contains_key(&file.path) { + return None; + } + let size = file.file_size_bytes.get()?.get(); + total = total.checked_add(size)?; + if total > MAX_SCAN_FILE_BYTES { + return None; + } + files.insert(file.path.clone(), size); + } + } + (!files.is_empty()).then_some(files) +} + +#[derive(Debug)] +struct BoundedManifestStore { + inner: Arc, + sizes: HashMap, +} + +impl fmt::Display for BoundedManifestStore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "bounded manifest reads") + } +} + +fn invalid_data(message: &'static str) -> object_store::Error { + object_store::Error::Generic { + store: "bounded manifest reads", + source: Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + message, + )), + } +} + +#[async_trait] +impl ObjectStore for BoundedManifestStore { + async fn get_opts( + &self, + path: &Path, + mut options: GetOptions, + ) -> object_store::Result { + let Some(&size) = self.sizes.get(path) else { + return self.inner.get_opts(path, options).await; + }; + if options.range.is_some() || options.head { + return self.inner.get_opts(path, options).await; + } + options.range = Some(GetRange::Bounded(0..size)); + let result = self.inner.get_opts(path, options).await?; + if result.meta.size != size || result.range != (0..size) { + return Err(invalid_data( + "manifest data size differs from its captured metadata", + )); + } + let meta = result.meta.clone(); + let attributes = result.attributes.clone(); + let stream = futures::stream::try_unfold( + (result.into_stream(), size), + |(mut stream, remaining)| async move { + match stream.try_next().await? { + Some(chunk) if chunk.len() as u64 > remaining => Err(invalid_data( + "manifest data response exceeds its bounded read", + )), + Some(chunk) => { + let remaining = remaining - chunk.len() as u64; + Ok(Some((chunk, (stream, remaining)))) + } + None if remaining != 0 => Err(invalid_data( + "manifest data response is shorter than its captured size", + )), + None => Ok(None), + } + }, + ); + Ok(GetResult { + payload: GetResultPayload::Stream(stream.boxed()), + meta, + range: 0..size, + attributes, + }) + } + + async fn put_opts( + &self, + _: &Path, + _: PutPayload, + _: PutOptions, + ) -> object_store::Result { + Err(invalid_data("manifest scan store is read-only")) + } + + async fn put_multipart_opts( + &self, + _: &Path, + _: PutMultipartOptions, + ) -> object_store::Result> { + Err(invalid_data("manifest scan store is read-only")) + } + + fn delete_stream( + &self, + _: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + futures::stream::once(async { Err(invalid_data("manifest scan store is read-only")) }) + .boxed() + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts(&self, _: &Path, _: &Path, _: CopyOptions) -> object_store::Result<()> { + Err(invalid_data("manifest scan store is read-only")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + use object_store::ObjectStoreExt; + + #[tokio::test] + async fn bounded_manifest_reads_validate_object_size_and_keep_other_paths_readable() { + let inner = Arc::new(object_store::memory::InMemory::new()); + let path = Path::from("main/data/file.lance"); + let metadata = Path::from("main/_versions/1.manifest"); + inner + .put(&path, Bytes::from(vec![7; 5542]).into()) + .await + .unwrap(); + inner + .put(&metadata, Bytes::from_static(b"metadata").into()) + .await + .unwrap(); + for declared in [5541, 5542, 5543] { + let store = BoundedManifestStore { + inner: inner.clone(), + sizes: HashMap::from([(path.clone(), declared)]), + }; + let result = store.get(&path).await; + if declared == 5542 { + assert_eq!( + result.unwrap().bytes().await.unwrap(), + Bytes::from(vec![7; 5542]) + ); + } else { + assert!(result.is_err(), "declared={declared}"); + } + assert_eq!( + store.get(&metadata).await.unwrap().bytes().await.unwrap(), + "metadata" + ); + assert!( + store + .put(&path, Bytes::from_static(b"rewrite").into()) + .await + .is_err() + ); + } + } + + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[derive(Clone, Copy, Debug)] + enum BodyCase { + Exact, + Short, + Overflow, + Fault, + } + + #[derive(Debug)] + struct BodyStore { + inner: Arc, + case: BodyCase, + requests: Arc, + polls: Arc, + } + + impl fmt::Display for BodyStore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "manifest body test store") + } + } + + #[async_trait] + impl ObjectStore for BodyStore { + async fn get_opts( + &self, + path: &Path, + options: GetOptions, + ) -> object_store::Result { + self.requests.fetch_add(1, Ordering::SeqCst); + assert_eq!(options.range, Some(GetRange::Bounded(0..6))); + let mut result = self.inner.get_opts(path, options).await?; + let mut chunks = vec![Ok(Bytes::from_static(b"ab"))]; + match self.case { + BodyCase::Exact => chunks.push(Ok(Bytes::from_static(b"cdef"))), + BodyCase::Short => {} + BodyCase::Overflow => chunks.push(Ok(Bytes::from_static(b"cdefg"))), + BodyCase::Fault => chunks.push(Err(object_store::Error::Generic { + store: "manifest body fault", + source: Box::new(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "injected manifest body fault", + )), + })), + } + let polls = Arc::clone(&self.polls); + result.payload = GetResultPayload::Stream( + futures::stream::iter(chunks) + .inspect(move |_| { + polls.fetch_add(1, Ordering::SeqCst); + }) + .boxed(), + ); + Ok(result) + } + + async fn put_opts( + &self, + _: &Path, + _: PutPayload, + _: PutOptions, + ) -> object_store::Result { + Err(invalid_data("unexpected write through body test store")) + } + + async fn put_multipart_opts( + &self, + _: &Path, + _: PutMultipartOptions, + ) -> object_store::Result> { + Err(invalid_data("unexpected write through body test store")) + } + + fn delete_stream( + &self, + _: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + futures::stream::once(async { + Err(invalid_data("unexpected delete through body test store")) + }) + .boxed() + } + + fn list( + &self, + prefix: Option<&Path>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + + async fn list_with_delimiter( + &self, + prefix: Option<&Path>, + ) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts(&self, _: &Path, _: &Path, _: CopyOptions) -> object_store::Result<()> { + Err(invalid_data("unexpected copy through body test store")) + } + } + + #[tokio::test] + async fn bounded_manifest_body_stays_lazy_and_preserves_stream_failures() { + let inner = Arc::new(object_store::memory::InMemory::new()); + let path = Path::from("main/data/file.lance"); + inner + .put(&path, Bytes::from_static(b"abcdef").into()) + .await + .unwrap(); + for case in [ + BodyCase::Exact, + BodyCase::Short, + BodyCase::Overflow, + BodyCase::Fault, + ] { + let requests = Arc::new(AtomicUsize::new(0)); + let polls = Arc::new(AtomicUsize::new(0)); + let store = BoundedManifestStore { + inner: Arc::new(BodyStore { + inner: inner.clone(), + case, + requests: Arc::clone(&requests), + polls: Arc::clone(&polls), + }), + sizes: HashMap::from([(path.clone(), 6)]), + }; + let result = store.get(&path).await.unwrap(); + assert_eq!(requests.load(Ordering::SeqCst), 1, "case={case:?}"); + assert_eq!( + polls.load(Ordering::SeqCst), + 0, + "get_opts consumed body: {case:?}" + ); + assert_eq!(result.meta.size, 6); + assert_eq!(result.range, 0..6); + let bytes = result.bytes().await; + match case { + BodyCase::Exact => assert_eq!(bytes.unwrap(), Bytes::from_static(b"abcdef")), + BodyCase::Short | BodyCase::Overflow => { + let object_store::Error::Generic { store, source } = bytes.unwrap_err() else { + panic!( + "bounded body mismatch must retain its invalid-data diagnosis: {case:?}" + ); + }; + assert_eq!(store, "bounded manifest reads"); + assert_eq!( + source.downcast_ref::().unwrap().kind(), + std::io::ErrorKind::InvalidData, + ); + } + BodyCase::Fault => { + let object_store::Error::Generic { store, source } = bytes.unwrap_err() else { + panic!("body fault must propagate as the original object-store error"); + }; + assert_eq!(store, "manifest body fault"); + let source = source.downcast_ref::().unwrap(); + assert_eq!(source.kind(), std::io::ErrorKind::ConnectionReset); + assert_eq!(source.to_string(), "injected manifest body fault"); + } + } + let expected_polls = if matches!(case, BodyCase::Short) { + 1 + } else { + 2 + }; + assert_eq!( + polls.load(Ordering::SeqCst), + expected_polls, + "case={case:?}" + ); + assert_eq!(requests.load(Ordering::SeqCst), 1, "case={case:?}"); + } + } +} diff --git a/crates/omnigraph-core/src/lance_access.rs b/crates/omnigraph-core/src/lance_access.rs index 51dfca771..7ba8ba364 100644 --- a/crates/omnigraph-core/src/lance_access.rs +++ b/crates/omnigraph-core/src/lance_access.rs @@ -16,14 +16,6 @@ static STORE_REGISTRY: LazyLock> = LazyLock::new(|| { Arc::new(registry) }); -/// Control-plane session for `__manifest` and other mutable-tip metadata. -/// -/// Its caches are deliberately disabled. Control paths still share the -/// process-wide object-store clients, but they cannot retain mutable-tip -/// metadata across branch-recreation boundaries. -static CONTROL_SESSION: LazyLock> = - LazyLock::new(|| Arc::new(Session::new(0, 0, Arc::clone(&STORE_REGISTRY)))); - /// The split Lance access context for one graph handle. /// /// Data tables use a graph-scoped cached session. Control-plane metadata uses a @@ -48,7 +40,7 @@ impl LanceAccessContext { DEFAULT_METADATA_CACHE_SIZE, Arc::clone(&STORE_REGISTRY), )), - control_session: Arc::clone(&CONTROL_SESSION), + control_session: control_session(), } } @@ -61,8 +53,14 @@ impl LanceAccessContext { } } +/// Control-plane session for `__manifest` and other mutable-tip metadata. +/// +/// Zero-capacity Lance caches still coordinate in-flight loads. Keep the session +/// local to its owner so separate handles do not wait on each other's metadata +/// loaders; their object-store clients remain process-wide. No mutable-tip +/// metadata is retained across branch recreation. pub fn control_session() -> Arc { - Arc::clone(&CONTROL_SESSION) + Arc::new(Session::new(0, 0, Arc::clone(&STORE_REGISTRY))) } /// DST seam (Lance-realm listing): the process-wide Lance object-store @@ -283,6 +281,8 @@ mod tests { assert!(!Arc::ptr_eq(&first_data, &control)); assert!(!Arc::ptr_eq(&first_data, &second_data)); + assert!(!Arc::ptr_eq(&control, &second.control_session())); + assert!(!Arc::ptr_eq(&control_session(), &control_session())); assert!(Arc::ptr_eq( &first_data.store_registry(), &control.store_registry() diff --git a/crates/omnigraph-dst/README.md b/crates/omnigraph-dst/README.md index 137ec4717..eb41b0937 100644 --- a/crates/omnigraph-dst/README.md +++ b/crates/omnigraph-dst/README.md @@ -97,12 +97,12 @@ The five GROUPS below are the reader's map; the enforced census counts Three distinct instruments, complementary — none subsumes another: -1. **Fault plans** (weather): seeded per-call fault rates in BOTH storage - realms (the `StorageAdapter` seam and Lance's interposed provider) — - clean errors and latency, read-path corruption (bit rot, truncation, - latent sector errors), write-side weather (corrupted, lost, misdirected - writes), ack-loss, and bounded staleness. Unaimed — discovers handling - bugs statistically across the whole run. +1. **Fault plans** (weather): seeded errors, latency and lost acknowledgements + in the adapter realm and, when enabled, Lance's provider. The adapter also + models text corruption, lost or misdirected writes, and bounded staleness; + each verb needs a workload that reaches its supported methods. Seeded + engine-effect failures use existing pre-publication decision seams. + Lance-call weather retains the semantic oracles but is outside strict replay. 2. **Crash-window hunt**: a named failpoint (72 in `catalog::CRASH_WINDOWS`) armed at a seeded op index. Aimed — guarantees a specific dangerous moment is exercised. @@ -123,16 +123,26 @@ and `tests/lane_b.rs`. failpoints (`catalog::CRASH_WINDOWS`) at a seeded op index. The ignored `dst_hunt_crash_window_sweep` test sweeps the whole catalog. - **Injected storage faults**: `Scenario::faults` installs a seeded `FaultPlan` - over both realms — marked errors + latency charged in VIRTUAL - time (`start_paused` clock, so faulty universes still run in milliseconds), - plus the corruption/ack-loss/staleness axes listed under Failure models. + on the adapter and optionally Lance. Latency uses virtual time. Lance supports + errors, latency and lost acknowledgements; adapter text corruption and + staleness do not extend to Lance or binary graph-index artifacts. +- **Engine-effect failures**: `FaultPlan::engine_effect_error_pct` selects an + existing pre-publication seam from a separate seeded stream. The atomicity + and live-handle owners require counted delivery and full report replay; + the latter also requires later model-changing writes without reopening. + These process-wide decision seams require serialized scenario ownership. +- **Legacy contract read weather**: `run_legacy_upgrade_read_weather` exercises + offline upgrade admission on flat-v11 fixtures. Stale source, bit rot, + truncation and latent errors must reach validation; refused attempts preserve + stored objects, and a clean retry preserves data and graph lineage. + Current-format serving reads its schema contract through Lance. ## Known limits Racing-writer determinism is blocked on the upstream Lance deterministic-mode proposal (4 flag-gated items) — the -`#[ignore]`d instruments flip on when it lands. Crash-mode universes cannot -yet run in parallel threads (failpoint-registry threading is a planned follow-up). +`#[ignore]`d instruments flip on when it lands. Crash-mode universes and engine-effect weather use process-wide decision +seams and must be serialized within a test process. ## Deferred work (#527 review) diff --git a/crates/omnigraph-dst/cost_table.txt b/crates/omnigraph-dst/cost_table.txt index 9cc205000..34e859f39 100644 --- a/crates/omnigraph-dst/cost_table.txt +++ b/crates/omnigraph-dst/cost_table.txt @@ -1,97 +1,66 @@ -AddFriend a.exists calls=16 -AddFriend a.get calls=28 -AddFriend l.get calls=135 -AddFriend l.list calls=43 +AddFriend l.get calls=101 +AddFriend l.list calls=16 AddFriend l.put calls=24 -BranchCreate a.exists calls=12 -BranchCreate a.get calls=18 BranchCreate l.get calls=4 -BranchCreate l.list calls=12 +BranchCreate l.list calls=8 BranchCreate l.put calls=6 -BranchDelete a.exists calls=6 -BranchDelete a.get calls=9 BranchDelete l.delete calls=1 -BranchDelete l.get calls=29 -BranchDelete l.list calls=8 +BranchDelete l.get calls=31 +BranchDelete l.list calls=6 BranchDelete l.put calls=2 -Cleanup a.exists calls=24 -Cleanup a.get calls=38 Cleanup l.delete calls=16 -Cleanup l.get calls=178 -Cleanup l.list calls=188 -DeletePerson a.exists calls=6 -DeletePerson a.get calls=12 +Cleanup l.get calls=170 +Cleanup l.list calls=172 DeletePerson l.get calls=3 -DeletePerson l.list calls=12 -EnsureIndices a.exists calls=4 -EnsureIndices a.get calls=7 -EnsureIndices l.get calls=9 -EnsureIndices l.list calls=8 -InsertLegacy a.exists calls=12 -InsertLegacy a.get calls=21 -InsertLegacy l.get calls=53 -InsertLegacy l.list calls=31 +DeletePerson l.list calls=3 +EnsureIndices l.get calls=4 +EnsureIndices l.list calls=2 +InsertLegacy l.get calls=18 +InsertLegacy l.list calls=11 InsertLegacy l.put calls=18 -InsertV a.exists calls=4 -InsertV a.get calls=7 -InsertV l.get calls=9 -InsertV l.list calls=9 +InsertV l.get calls=2 +InsertV l.list calls=3 InsertV l.put calls=6 -Optimize a.exists calls=28 -Optimize a.get calls=42 +Optimize a.exists calls=2 Optimize a.put calls=1 -Optimize l.get calls=72 -Optimize l.list calls=37 +Optimize l.get calls=55 +Optimize l.list calls=17 Optimize l.put calls=50 -RemoveFriendshipsFrom a.exists calls=14 -RemoveFriendshipsFrom a.get calls=27 -RemoveFriendshipsFrom l.get calls=35 -RemoveFriendshipsFrom l.list calls=31 +RemoveFriendshipsFrom l.get calls=14 +RemoveFriendshipsFrom l.list calls=8 RemoveFriendshipsFrom l.put calls=5 -UpdateV a.exists calls=8 -UpdateV a.get calls=16 UpdateV l.get calls=4 -UpdateV l.list calls=16 +UpdateV l.list calls=4 _audit a.delete calls=1 -_audit a.exists calls=81 -_audit a.get calls=122 +_audit a.get calls=2 _audit a.list calls=2 _audit a.put_if_absent calls=1 -_audit l.get calls=843 +_audit l.get calls=822 _audit l.list calls=127 -_close a.exists calls=6 -_close a.get calls=10 -_close l.get calls=17 -_close l.list calls=9 -_history a.exists calls=60 -_history a.get calls=90 -_history l.get calls=240 +_close l.list calls=3 _history l.list calls=30 _setup a.delete calls=3 -_setup a.exists calls=23 -_setup a.get calls=21 +_setup a.exists calls=2 _setup a.list calls=1 -_setup a.put calls=2 -_setup a.put_if_absent calls=4 -_setup l.get calls=46 -_setup l.list calls=29 +_setup a.put_if_absent calls=3 +_setup l.get calls=45 +_setup l.list calls=18 _setup l.put calls=26 -_verify a.exists calls=413 -_verify a.get calls=637 -_verify l.get calls=1638 -_verify l.list calls=458 +_verify a.get calls=10 +_verify l.get calls=1357 +_verify l.list calls=437 MergeTargetHistory2 l.delete calls=3 -MergeTargetHistory2 l.get calls=94 -MergeTargetHistory2 l.list calls=20 +MergeTargetHistory2 l.get calls=97 +MergeTargetHistory2 l.list calls=12 MergeTargetHistory2 l.put calls=11 MergeTargetHistory32 l.delete calls=3 -MergeTargetHistory32 l.get calls=94 -MergeTargetHistory32 l.list calls=20 +MergeTargetHistory32 l.get calls=97 +MergeTargetHistory32 l.list calls=12 MergeTargetHistory32 l.put calls=11 MergeBackAfterThreeWay l.delete calls=3 -MergeBackAfterThreeWay l.get calls=112 -MergeBackAfterThreeWay l.list calls=25 +MergeBackAfterThreeWay l.get calls=115 +MergeBackAfterThreeWay l.list calls=17 MergeBackAfterThreeWay l.put calls=11 -BranchWrite l.get calls=40 -BranchWrite l.list calls=13 +BranchWrite l.get calls=19 +BranchWrite l.list calls=5 BranchWrite l.put calls=6 diff --git a/crates/omnigraph-dst/src/catalog.rs b/crates/omnigraph-dst/src/catalog.rs index 344dfe852..7f3966ab1 100644 --- a/crates/omnigraph-dst/src/catalog.rs +++ b/crates/omnigraph-dst/src/catalog.rs @@ -1,5 +1,5 @@ //! The full crash-window catalog for the hunt -//! (`dst_hunt_crash_window_sweep`): 56 of the engine's decision seams +//! (`dst_hunt_crash_window_sweep`): 52 of the engine's decision seams //! (`omnigraph::seams::catalog`) at the pinned engine version. A seam added //! to the engine enters here as never-reached until its workload exists. //! @@ -7,7 +7,7 @@ //! be a name the engine catalog declares, so a typo'd or renamed-away window //! fails the suite instead of compiling and silently never firing. -pub const CRASH_WINDOWS: [&str; 56] = [ +pub const CRASH_WINDOWS: [&str; 52] = [ "blob_read.post_capture", "branch_control.pre_gates", "branch_create.post_native", @@ -35,8 +35,6 @@ pub const CRASH_WINDOWS: [&str; 56] = [ "graph_publish.after_manifest_commit", "graph_publish.before_commit_append", "init.after_coordinator_init", - "init.after_schema_contract_written", - "init.after_schema_pg_written", "init.post_manifest_create", "load.between_table_stages", "load.post_branch_create_pre_stage", @@ -53,8 +51,6 @@ pub const CRASH_WINDOWS: [&str; 56] = [ "publish.load_state", "read.refresh_post_state_pre_lineage", "schema_apply.after_manifest_commit", - "schema_apply.after_staging_write", - "schema_apply.before_staging_write", "schema_apply.post_lock_pre_effect", "schema_apply.post_table_commit", "schema_reload.before_contract_read", diff --git a/crates/omnigraph-dst/src/cost.rs b/crates/omnigraph-dst/src/cost.rs index 919965028..ca9ebdd4e 100644 --- a/crates/omnigraph-dst/src/cost.rs +++ b/crates/omnigraph-dst/src/cost.rs @@ -8,8 +8,8 @@ //! //! Slot-armed and default-None: zero draws, zero behavior change, near //! zero cost for every other test. Verbs are prefixed by realm — -//! `a.` = adapter (manifest/refs/sidecars via `StorageAdapter`), -//! `l.` = Lance (table data/manifests via the provider shim). Labels are +//! `a.` = adapter (control objects via `StorageAdapter`), +//! `l.` = Lance (graph manifest and table datasets via the provider shim). Labels are //! the universe's op kinds plus harness phases (`_setup`, `_verify`, //! `_history`, `_close`, `_audit`) so oracle reads never pollute op costs. diff --git a/crates/omnigraph-dst/src/harness.rs b/crates/omnigraph-dst/src/harness.rs index 1a99b2be1..fbba3e6bf 100644 --- a/crates/omnigraph-dst/src/harness.rs +++ b/crates/omnigraph-dst/src/harness.rs @@ -146,6 +146,12 @@ pub(crate) const LATENT_MARKER: &str = "latent sector error (dst)"; pub struct FaultPlan { pub seed: u64, pub error_pct: u64, + /// Seeded failures at current engine pre-publication effect boundaries. + /// Unlike Lance IO weather, selection follows the sequential workload, + /// not internal call order, so these universes retain strict replay. + /// Requires failpoints and a serialized `FailScenario`; zero leaves + /// the existing schedule unchanged. + pub engine_effect_error_pct: u64, /// Slatedb-derived doctrine (probe/list failpoints): faults on READ-class /// calls too — reads and listings (`read_fault` call sites) — one seam knob instead of /// per-point instrumentation. @@ -275,6 +281,7 @@ impl FaultPlan { Self { seed: 0, error_pct: 0, + engine_effect_error_pct: 0, read_error_pct: 0, latency_pct: 0, max_latency_ms: 1, @@ -1063,6 +1070,10 @@ pub struct UniverseReport { /// model-predicted merge conflict, keep-serving deferral refusals — /// the watched streak and the maintenance-barrier spelling). pub legal_rejections: usize, + /// Counted engine effect seams that fired and returned their exact error. + pub engine_effects_injected: usize, + /// Successful model-changing data writes after at least one such failure. + pub writes_after_engine_effect: usize, /// errors the Lance-realm injector actually delivered — /// evidence the table realm saw weather (0 in clean universes). pub lance_realm_injected: usize, @@ -5253,6 +5264,14 @@ pub fn run_universe(root: &str, scenario: &Scenario) -> UniverseReport { /// Retain detector panic payloads while the shared executor finalizes resources. pub fn run_universe_caught(root: &str, sc: &Scenario) -> std::thread::Result { + assert!( + cfg!(feature = "failpoints") + || sc + .faults + .as_ref() + .is_none_or(|plan| plan.engine_effect_error_pct == 0), + "engine effect weather requires --features failpoints" + ); assert!( !(sc.keep_handle && (sc.crash_at.is_some() @@ -5308,6 +5327,302 @@ pub fn run_universe_caught(root: &str, sc: &Scenario) -> std::thread::Result, + pub missing_ir_refused: bool, + pub final_rows: Vec<(String, i64, i64)>, + pub commit_ids: Vec, +} + +struct LegacyUpgradeReadScenario { + plan: FaultPlan, + attempts: usize, + execute_under_weather: bool, +} + +/// Exercise the actual legacy-file consumer in a seeded universe. The fixture +/// becomes genuine flat-v11 rows before weather starts. Every weathered attempt +/// must leave all stored objects unchanged; execution attempts are refusal-only. +pub fn run_legacy_upgrade_read_weather( + root: &str, + seed: u64, + plan: FaultPlan, + attempts: usize, + execute_under_weather: bool, +) -> LegacyUpgradeReadReport { + assert!( + !plan.lance_realm, + "legacy adapter weather requires strict replay" + ); + assert!(attempts > 0); + crate::env_knobs::require_pool_env(); + let environment = MemoryEnvironment::new(root, seed, UniverseProcess::Shared); + let scenario = LegacyUpgradeReadScenario { + plan, + attempts, + execute_under_weather, + }; + let run = crate::environment::run_universe(&environment, &scenario); + run.cleanup + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) + .expect("legacy upgrade universe cleanup"); + run.result + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) + .expect("legacy upgrade universe setup") +} + +async fn legacy_upgrade_stored_objects(resources: &MemoryStorage) -> BTreeMap> { + use futures::TryStreamExt; + use lance_io::object_store::providers::shared_memory::SharedMemoryStoreProvider; + use lance_io::object_store::{ObjectStoreParams, ObjectStoreProvider}; + use object_store::ObjectStoreExt; + + let mut objects = BTreeMap::new(); + for uri in resources.adapter.list_dir(&resources.root).await.unwrap() { + let text = resources.adapter.read_text(&uri).await.unwrap(); + objects.insert(format!("adapter:{uri}"), text.into_bytes()); + } + let url = url::Url::parse(&resources.root).unwrap(); + let provider = SharedMemoryStoreProvider::default(); + let prefix = provider.extract_path(&url).unwrap(); + let store = provider + .new_store(url, &ObjectStoreParams::default()) + .await + .unwrap() + .inner; + let mut listing = store.list(Some(&prefix)); + while let Some(object) = listing.try_next().await.unwrap() { + let bytes = store + .get(&object.location) + .await + .unwrap() + .bytes() + .await + .unwrap(); + objects.insert(format!("lance:{}", object.location), bytes.to_vec()); + } + objects +} + +impl UniverseScenario for LegacyUpgradeReadScenario { + type Output = LegacyUpgradeReadReport; + + async fn run<'a>( + &'a self, + resources: &'a mut MemoryStorage, + _workload_seed: u64, + ) -> LegacyUpgradeReadReport { + use omnigraph::db::{UpgradeOptions, UpgradeOutcome, upgrade_storage}; + + let root = resources.root.as_str(); + let db = Session::from_defaults( + Arc::new( + Omnigraph::init_with_storage( + root, + TEST_SCHEMA, + resources.adapter.clone(), + InitOptions::default(), + ) + .await + .unwrap(), + ), + SessionSettings::default(), + ); + db.load_jsonl(TEST_DATA, LoadMode::Overwrite).await.unwrap(); + let expected_export = db.export_jsonl("main", &[]).await.unwrap(); + let expected_rows = person_rows(&db).await; + let expected_edges = knows_pairs(&db).await; + let expected_commits = db.list_commits(None).await.unwrap(); + drop(db); + let (source, ir, state) = omnigraph::db::dst_prepare_legacy_upgrade_fixture(root) + .await + .unwrap(); + for (name, text) in [ + ("_schema.pg", source.as_str()), + ("_schema.ir.json", ir.as_str()), + ("__schema_state.json", state.as_str()), + ] { + resources + .adapter + .write_text(&format!("{root}/{name}"), text) + .await + .unwrap(); + } + + let source_uri = format!("{root}/_schema.pg"); + if self.plan.stale_read_pct > 0 { + resources + .adapter + .write_text( + &source_uri, + &format!("{}\nnode FormerFixture {{ value: String }}\n", source), + ) + .await + .unwrap(); + } + let failing = Arc::new(FailingStorage::new( + resources.adapter.clone(), + self.plan.clone(), + None, + None, + )); + if self.plan.stale_read_pct > 0 { + failing.write_text(&source_uri, &source).await.unwrap(); + } + let _decoration = + omnigraph::storage::STORAGE.install(Arc::new(FailingStorageDecorator(failing.clone()))); + let before = legacy_upgrade_stored_objects(resources).await; + + let clean_check = upgrade_storage( + root, + UpgradeOptions { + check: true, + to_format: Some(13), + }, + ) + .await + .unwrap(); + assert_eq!(clean_check.outcome, UpgradeOutcome::CheckPassed); + assert_eq!(legacy_upgrade_stored_objects(resources).await, before); + + let mut report = LegacyUpgradeReadReport { + checks_passed: 0, + checks_failed: 0, + reads_corrupted: 0, + reads_truncated: 0, + latent_errors: 0, + stale_reads_served: 0, + detections: Vec::new(), + missing_ir_refused: false, + final_rows: Vec::new(), + commit_ids: Vec::new(), + }; + failing.enable(); + for attempt in 0..self.attempts { + let damage_before = failing.damage_events() + failing.stale_reads_count(); + let result = upgrade_storage( + root, + UpgradeOptions { + check: !self.execute_under_weather, + to_format: Some(13), + }, + ) + .await + .expect("upgrade exposes preflight refusals as a typed report"); + assert!( + result.last_durable_completed_boundary.is_none(), + "{result:?}" + ); + assert!(result.recovery.is_none(), "{result:?}"); + match result.outcome { + UpgradeOutcome::CheckPassed if !self.execute_under_weather => { + report.checks_passed += 1; + } + UpgradeOutcome::CheckFailed => { + report.checks_failed += 1; + assert!( + failing.damage_events() + failing.stale_reads_count() > damage_before, + "refusal without a delivered read fault: {result:?}" + ); + assert!( + result + .findings + .iter() + .any(|finding| finding.code == "preflight_failed"), + "{result:?}" + ); + for finding in result.findings { + report.detections.push(format!( + "attempt={attempt} {}: {}", + finding.code, + finding.message.replace(root, ""), + )); + } + } + other => panic!("unexpected legacy admission outcome {other:?}: {result:?}"), + } + assert_eq!(legacy_upgrade_stored_objects(resources).await, before); + } + failing.suspend(); + report.reads_corrupted = failing.reads_corrupted(); + report.reads_truncated = failing.reads_truncated(); + report.latent_errors = failing.latent_errors(); + report.stale_reads_served = failing.stale_reads_count(); + + let ir_uri = format!("{root}/_schema.ir.json"); + resources.adapter.delete(&ir_uri).await.unwrap(); + let missing_before = legacy_upgrade_stored_objects(resources).await; + let missing = upgrade_storage( + root, + UpgradeOptions { + check: false, + to_format: Some(13), + }, + ) + .await + .unwrap(); + assert_eq!(missing.outcome, UpgradeOutcome::CheckFailed); + assert!(missing.last_durable_completed_boundary.is_none()); + assert!(missing.recovery.is_none()); + assert!( + missing + .findings + .iter() + .any(|finding| finding.code == "preflight_failed") + ); + assert_eq!( + legacy_upgrade_stored_objects(resources).await, + missing_before + ); + resources.adapter.write_text(&ir_uri, &ir).await.unwrap(); + report.missing_ir_refused = true; + assert_eq!(legacy_upgrade_stored_objects(resources).await, before); + + let converted = upgrade_storage(root, UpgradeOptions::default()) + .await + .unwrap(); + assert_eq!( + converted.outcome, + UpgradeOutcome::Completed, + "{converted:?}" + ); + let db = Session::from_defaults( + Arc::new( + Omnigraph::open_with_storage(root, resources.adapter.clone()) + .await + .unwrap(), + ), + SessionSettings::default(), + ); + assert_eq!(db.export_jsonl("main", &[]).await.unwrap(), expected_export); + report.final_rows = person_rows(&db).await; + assert_eq!(report.final_rows, expected_rows); + assert_eq!(knows_pairs(&db).await, expected_edges); + let commits = db.list_commits(None).await.unwrap(); + assert_eq!( + commits, expected_commits, + "storage conversion preserves graph lineage" + ); + report.commit_ids = commits + .into_iter() + .map(|commit| commit.graph_commit_id) + .collect(); + assert!(!report.commit_ids.is_empty()); + assert!(!report.final_rows.is_empty()); + assert!(resources.adapter.list_dir(root).await.unwrap().is_empty()); + report + } +} + impl UniverseScenario for Scenario { type Output = UniverseReport; @@ -5391,6 +5706,14 @@ impl UniverseScenario for Scenario { let mut crashes = 0usize; let mut verified = 0usize; let mut legal_rejections = 0usize; + #[cfg(feature = "failpoints")] + let mut engine_effects_injected = 0usize; + #[cfg(not(feature = "failpoints"))] + let engine_effects_injected = 0usize; + let mut writes_after_engine_effect = 0usize; + #[cfg(feature = "failpoints")] + let mut effect_rng = + SplitMix64(sc.faults.as_ref().map_or(0, |plan| plan.seed) ^ 0x4546_4645_4354_4453); // Reopens performed while judging failures (reconcile / crash // recovery). A `keep_handle` universe must end with zero. let mut reopens = 0usize; @@ -5708,7 +6031,49 @@ impl UniverseScenario for Scenario { } _ => None, }; - exec_world_op(&db, &wop).await + let effect_window = sc.faults.as_ref().and_then(|plan| { + if plan.engine_effect_error_pct == 0 + || sc.probe_window.is_some() + || (sc.probe_only && crash_now.is_some()) + || !matches!(&wop, WorldOp::Data { op, .. } if is_mutation_op(op) || is_load_op(op)) + || effect_rng.below(100) >= plan.engine_effect_error_pct + { + return None; + } + let windows = [ + "mutation.post_table_commit", + "graph_publish.before_commit_append", + ]; + Some(windows[effect_rng.below(windows.len() as u64) as usize]) + }); + let effect_guard = effect_window.map(|window| { + omnigraph::seams::catalog::decide(window) + .expect("engine effect weather names a catalog seam") + .count_and_fire_at(1) + }); + let result = exec_world_op(&db, &wop).await; + let fired = effect_guard + .as_ref() + .is_some_and(|(_, count)| count.fired()); + drop(effect_guard); + if fired { + let window = effect_window.expect("a counted injection has a selected seam"); + let error = result.expect_err("pre-publication effect failure was absorbed"); + assert!( + matches!(&error, OmniError::Manifest(detail) + if detail.kind == omnigraph::error::ManifestErrorKind::BadRequest + && detail.details.is_none() + && !detail.publication_in_doubt + && detail.message == format!("injected failpoint triggered: {window}")), + "counted effect fired but an unrelated error escaped: {error}" + ); + engine_effects_injected += 1; + Err(OmniError::manifest(format!( + "{FAULT_MARKER}: engine effect {window}: {error}" + ))) + } else { + result + } }; #[cfg(not(feature = "failpoints"))] let exec_result = exec_world_op(&db, &wop).await; @@ -5862,7 +6227,13 @@ impl UniverseScenario for Scenario { f.resume(); } } + let prior_data = (engine_effects_injected > 0 + && matches!(&wop, WorldOp::Data { op, .. } if is_mutation_op(op) || is_load_op(op))) + .then(|| world.render()); apply_world(&mut world, &wop); + if prior_data.is_some_and(|before| before != world.render()) { + writes_after_engine_effect += 1; + } } Err(err) => { // One damage snapshot for the WHOLE failure handling: @@ -6421,6 +6792,8 @@ impl UniverseScenario for Scenario { bystander_trail, verified, legal_rejections, + engine_effects_injected, + writes_after_engine_effect, lance_realm_injected, writes_observed, crash_state_hit, diff --git a/crates/omnigraph-dst/src/write_census.rs b/crates/omnigraph-dst/src/write_census.rs index c4ccfdb63..98abaa147 100644 --- a/crates/omnigraph-dst/src/write_census.rs +++ b/crates/omnigraph-dst/src/write_census.rs @@ -290,11 +290,5 @@ pub fn classify(uri: &str) -> &'static str { if uri.contains("_refs/") || uri.contains("_branches/") { return "branch-ref"; } - // Exact shapes only: a broad substring here would silently swallow - // the UNKNOWN red for a future artifact whose name merely contains - // the word (under-matching is safe, over-matching defeats the map). - if uri.contains("_schema.pg") || uri.contains("_schema.ir") || uri.contains("__schema_state") { - return "schema-artifact"; - } "UNKNOWN" } diff --git a/crates/omnigraph-dst/tests/scenarios.rs b/crates/omnigraph-dst/tests/scenarios.rs index fb3b25dee..f3cec2c75 100644 --- a/crates/omnigraph-dst/tests/scenarios.rs +++ b/crates/omnigraph-dst/tests/scenarios.rs @@ -7,8 +7,8 @@ //! //! Answers the harness's store-injection question: can the //! engine init, load, query, and REOPEN against (a) an injected in-memory -//! `StorageAdapter` for the manifest/write-queue realm and (b) Lance's -//! `shared-memory://` provider for the table realm — with no local filesystem +//! `StorageAdapter` for control objects and (b) Lance's `shared-memory://` +//! provider for the graph manifest and table datasets — with no local filesystem //! involved anywhere? //! //! The two realms are SEPARATE stores under one root URI (omnigraph-storage's @@ -344,9 +344,6 @@ fn dst_schema_add_property_after_mutation_preserves_traversal() { /// window and judge the store against the birth contract (open cleanly, fail /// truthfully, or re-init cleanly on the same root). Expected map pins /// today's measured truth: -/// - `after_schema_pg_written` / `after_schema_contract_written` → -/// DiedThenReinitRecovers (`cleanup_failed_init` removes the schema files; -/// nothing else exists yet — cleanup is CORRECT early). /// - `post_manifest_create` → DiedThenOpensClean: since #487 the manifest's /// whole birth (entries, lineage, stamp) rides the Create commit, and the /// #495 fix keeps error-return cleanup on the pre-commit side, so a @@ -364,17 +361,7 @@ fn dst_birth_contract_sweep() { let _scenario = omnigraph::seams::FailScenario::setup(); #[allow(clippy::type_complexity)] - let cases: [(&'static str, &dyn Fn(&BirthOutcome) -> bool, &str); 4] = [ - ( - omnigraph::seams::catalog::INIT_AFTER_SCHEMA_PG_WRITTEN.name(), - &|o| *o == BirthOutcome::DiedThenReinitRecovers, - "DiedThenReinitRecovers", - ), - ( - omnigraph::seams::catalog::INIT_AFTER_SCHEMA_CONTRACT_WRITTEN.name(), - &|o| *o == BirthOutcome::DiedThenReinitRecovers, - "DiedThenReinitRecovers", - ), + let cases: [(&'static str, &dyn Fn(&BirthOutcome) -> bool, &str); 2] = [ ( omnigraph::seams::catalog::INIT_POST_MANIFEST_CREATE.name(), &|o| *o == BirthOutcome::DiedThenOpensClean, @@ -594,25 +581,28 @@ fn dst_v16_physical_oracle_pins_ghost_delta() { ); } -/// WIDE WORKLOAD: schema evolution (additive optional -/// props), mid-life bulk loads (merge / append / fork-from-base via the -/// implicit-fork path), and refresh/sync join the sampler. Every oracle must -/// hold and the wide universe must replay identically — the meta-test -/// covering the new families for free. +/// Wide workloads preserve every oracle and strict replay, including cleanup +/// after deleting and recreating a logical branch name. #[test] #[serial] fn dst_v14_wide_workload_replays() { - let sc = Scenario { - seed: 4040, - ops: 36, - wide: true, - ..Default::default() - }; - let a = run_universe("shared-memory://dst-wide-a", &sc); - let b = run_universe("shared-memory://dst-wide-b", &sc); - omnigraph_dst::harness::assert_strict_replay(&a, &b, "wide universe must replay identically"); - assert!(a.verified > 0); - assert!(!a.commit_ids.is_empty()); + for (seed, ops) in [(4040, 36), (10009, 30)] { + let sc = Scenario { + seed, + ops, + wide: true, + ..Default::default() + }; + let a = run_universe(&format!("shared-memory://dst-wide-{seed}-a"), &sc); + let b = run_universe(&format!("shared-memory://dst-wide-{seed}-b"), &sc); + omnigraph_dst::harness::assert_strict_replay( + &a, + &b, + "wide universe must replay identically", + ); + assert!(a.verified > 0); + assert!(!a.commit_ids.is_empty()); + } } /// TRACE-DIFF HUNT — the instrument that NAMES the residual concurrent leak: @@ -1055,26 +1045,24 @@ async fn dst_memory_graph_end_to_end() { ); } -/// FAULT-INJECTION ATOMICITY RUN: seeded write-faults on the storage -/// seam; every failed op must be invisible (continuous model checks), the -/// final world must equal the model, and the whole faulty universe must -/// REPLAY identically — determinism under injected storage errors + virtual -/// latency. +/// Seeded failures after detached effects or before manifest publication. +/// Full model checks, atomicity arbitration and strict replay stay enabled; +/// Lance-call weather has a separate owner outside strict replay. +#[cfg(feature = "failpoints")] #[test] #[serial] fn dst_v11_fault_injection_atomicity_and_replay() { + let _scenario = omnigraph::seams::FailScenario::setup(); let sc = Scenario { seed: 71, ops: 30, faults: Some(omnigraph_dst::harness::FaultPlan { seed: 7100, error_pct: 15, + engine_effect_error_pct: 35, read_error_pct: 10, latency_pct: 30, max_latency_ms: 7, - // Adapter realm only: this test is THE fully-deterministic - // faulty-universe replay pin (lance_realm moves a universe - // outside the replay envelope — see `FaultPlan::lance_realm`'s doc). lance_realm: false, ack_loss_pct: 0, client_retry: false, @@ -1090,68 +1078,98 @@ fn dst_v11_fault_injection_atomicity_and_replay() { "faulty universes must replay identically", ); assert!( - a.legal_rejections > 0, - "injected faults should actually bite" + a.engine_effects_injected > 0 && a.legal_rejections >= a.engine_effects_injected, + "counted engine effects must fail workload operations: {a:?}" ); assert!(a.verified > 0); + + for (window, fault_seed) in [ + ("mutation.post_table_commit", 0), + ("graph_publish.before_commit_append", 1), + ] { + for persistent in [false, true] { + let probe = Scenario { + seed: 71, + ops: 1, + reach_target: Some(window), + crash_on_match: (!persistent).then_some((window, 0)), + probe_only: !persistent, + probe_window: persistent.then_some(window), + faults: Some(omnigraph_dst::harness::FaultPlan { + seed: fault_seed, + engine_effect_error_pct: 100, + ..Default::default() + }), + ..Default::default() + }; + let first = run_universe( + &format!("shared-memory://dst-effect-probe-{window}-{persistent}-a"), + &probe, + ); + assert!( + first.crossed, + "effect probe {window}: crossing was suppressed" + ); + assert_eq!(first.engine_effects_injected, 0); + assert_eq!(first.crashes, 0); + assert!( + first.end_state.iter().any(|(name, _, _)| name == "mm1"), + "effect probe {window}: the observed insert did not publish" + ); + let second = run_universe( + &format!("shared-memory://dst-effect-probe-{window}-{persistent}-b"), + &probe, + ); + omnigraph_dst::harness::assert_strict_replay( + &first, + &second, + "effect weather must preserve crossing-probe replay", + ); + } + } } -/// BOUNDED STALENESS, first-contact pin: the adapter realm -/// serves seeded as-of-old-tick reads and listings (values true, recency a -/// lie; zombies and stale absences included; CAS strict at head) and the -/// universe must (1) BITE, (2) REPLAY identically (staleness draws ride -/// the plan rng; adapter realm = strict-replay envelope), (3) keep EVERY -/// oracle green on the settled truth, and (4) make PROGRESS at lag k -/// (rationale in the assert below). +/// Legacy upgrade is the remaining consumer of schema-file text. Serve an +/// actual older source value during execution, require refusal before a fence, +/// replay every counter/result, then upgrade successfully after weather stops. #[test] #[serial] -fn dst_staleness_bite_and_replay() { - // Seed 278 since RFC 0067 moved the storage-action schedule three times - // (the detached writers, the detached index writer, then the removal of - // the per-write `__recovery/` listing): each earlier pin then met a stale - // absence of a schema contract file, which the engine refuses as manual - // coordination. `dst_staleness_seed_search` lists the green seeds. - let sc = Scenario { - seed: 278, - ops: 30, - faults: Some(omnigraph_dst::harness::FaultPlan { - seed: 27_800, - stale_read_pct: 15, - stale_list_pct: 15, - max_lag_ticks: 4, - ..Default::default() - }), +fn dst_legacy_upgrade_staleness_bite_and_replay() { + use omnigraph_dst::harness::{FaultPlan, run_legacy_upgrade_read_weather}; + + let plan = FaultPlan { + seed: 27_800, + stale_read_pct: 100, + max_lag_ticks: 1, ..Default::default() }; - let a = run_universe("shared-memory://dst-s25-a", &sc); - let b = run_universe("shared-memory://dst-s25-b", &sc); - omnigraph_dst::harness::assert_strict_replay(&a, &b, "stale universes must replay identically"); - assert!( - a.stale_reads_served + a.stale_lists_served > 0, - "staleness should actually bite (reads={} lists={})", - a.stale_reads_served, - a.stale_lists_served + let a = run_legacy_upgrade_read_weather( + "shared-memory://dst-upgrade-stale-a", + 278, + plan.clone(), + 4, + true, ); + let b = + run_legacy_upgrade_read_weather("shared-memory://dst-upgrade-stale-b", 278, plan, 4, true); + assert_eq!(a, b, "legacy upgrade stale reads must replay identically"); assert!( - a.commit_ids.len() >= 6, - "bounded staleness froze the world: only {} commits — progress is \ - part of the correct-up-to-lag-k claim", - a.commit_ids.len() + a.stale_reads_served > 0, + "stale source bytes must reach preflight" ); - println!( - "dst staleness first contact: stale_reads={} stale_lists={} commits={} \ - legal_rejections={}", - a.stale_reads_served, - a.stale_lists_served, - a.commit_ids.len(), - a.legal_rejections + assert_eq!(a.checks_failed, 4); + assert_eq!(a.checks_passed, 0); + assert!(!a.detections.is_empty()); + assert!(a.missing_ir_refused); + assert!( + !a.final_rows.is_empty(), + "clean upgrade must preserve readable data" ); } -/// SEARCH INSTRUMENT for the bounded-staleness pin above: enumerate seeds -/// and print which universes bite, replay identically, keep every oracle -/// green and make progress, so the pin can be re-chosen when the engine's -/// storage-action schedule shifts. Not part of the suite. +/// Historical instrument for adapter-read schedules, excluded from the suite. +/// Current serving no longer reads schema files; the legacy upgrade pin above +/// owns schema-file staleness coverage. #[test] #[serial] #[ignore = "instrument: bounded-staleness seed search — run explicitly"] @@ -1445,16 +1463,14 @@ fn dst_lance_realm_ack_loss_bites_and_oracles_hold() { assert!(r.verified > 0); } -/// GENERALIZED LIVENESS (the successor of the retired RecoveryRequired- -/// specific keep-serving checks): one live handle rides out an entire -/// adapter-realm fault storm — clean errors, lost acknowledgements, client -/// retries — and is NEVER reopened (`keep_handle`; `reopens == 0` is the -/// proof). Every failed op's aftermath is judged through the same handle a -/// real client would keep, ops keep landing between faults, and the full -/// oracle stack holds. Adapter realm only ⇒ strict replay identity. +/// One live handle handles seeded effect failures without reopening. +/// Counted failures and later model-changing writes must both occur; +/// the full oracle stack and strict replay stay enabled. +#[cfg(feature = "failpoints")] #[test] #[serial] fn dst_fault_storm_on_one_live_handle_keeps_writing() { + let _scenario = omnigraph::seams::FailScenario::setup(); let sc = Scenario { seed: 103, ops: 30, @@ -1462,6 +1478,7 @@ fn dst_fault_storm_on_one_live_handle_keeps_writing() { faults: Some(omnigraph_dst::harness::FaultPlan { seed: 10300, error_pct: 12, + engine_effect_error_pct: 35, read_error_pct: 6, latency_pct: 10, max_latency_ms: 3, @@ -1474,8 +1491,14 @@ fn dst_fault_storm_on_one_live_handle_keeps_writing() { let a = run_universe("shared-memory://dst-live-handle-a", &sc); let b = run_universe("shared-memory://dst-live-handle-b", &sc); println!( - "dst live handle: {} reopens, {} legal rejections, {} acks lost, {} client retries, {} checks", - a.reopens, a.legal_rejections, a.acks_lost, a.client_retries, a.verified + "dst live handle: {} reopens, {} legal rejections, {} engine effects, {} later writes, {} acks lost, {} client retries, {} checks", + a.reopens, + a.legal_rejections, + a.engine_effects_injected, + a.writes_after_engine_effect, + a.acks_lost, + a.client_retries, + a.verified ); omnigraph_dst::harness::assert_strict_replay( &a, @@ -1487,134 +1510,123 @@ fn dst_fault_storm_on_one_live_handle_keeps_writing() { "the whole storm must run on one never-reopened handle" ); assert!( - a.legal_rejections > 0, - "the storm should actually fail ops on the live handle (legal_rejections={})", - a.legal_rejections + a.engine_effects_injected > 0 && a.legal_rejections >= a.engine_effects_injected, + "the storm must deliver counted failures on the live handle: {a:?}" + ); + assert!( + a.writes_after_engine_effect > 0, + "the same handle must publish model-changing data after a delivered fault: {a:?}" ); assert!(a.verified > 0); } -/// CORRUPTION AXIS (read tier): the store LIES (read-time bit rot, -/// truncated reads) and grows LATENT SECTOR ERRORS (persistent, -/// location-indexed poison), adapter realm, moderate weather. Contract: -/// detected-or-harmless — an op failure whose reads crossed the damage -/// ledger is an attributed detection (legal, recorded); silent model -/// divergence anywhere panics the universe, so a green run IS the -/// no-silent-wrong-answer verdict. Read-path verbs leave stored bytes -/// true, so oracle suspension keeps every judged read clean and STRICT -/// replay identity holds (all draws ride the plan's seeded stream). +/// Read-time bit rot and truncation reach real legacy upgrade validation. +/// Moderate weather must admit clean checks alongside attributed refusals; +/// full object, data and lineage oracles remain independent of the decorator. #[test] #[serial] -fn dst_read_corruption_bite_and_replay() { - let sc = Scenario { - seed: 83, - ops: 30, - // NO latent verb here (08-13 double-check): one latent poisoning of - // the per-op schema read converts the whole universe into a refusal - // storm (seed 83 first cut: 30/30 ops refused, world frozen at - // fixtures — green oracles judged almost nothing). This pin is the - // MODERATE regime: ops must keep succeeding ALONGSIDE delivered - // lies, or the silent tier is never even probed. Latent's refusal - // behavior is pinned by the storm test below. - faults: Some(omnigraph_dst::harness::FaultPlan { - seed: 8300, - corrupt_read_pct: 8, - truncate_read_pct: 5, - ..Default::default() - }), +fn dst_legacy_upgrade_read_corruption_bite_and_replay() { + use omnigraph_dst::harness::{FaultPlan, run_legacy_upgrade_read_weather}; + + let plan = FaultPlan { + seed: 8300, + corrupt_read_pct: 8, + truncate_read_pct: 5, ..Default::default() }; - let a = run_universe("shared-memory://dst-s11-a", &sc); - let b = run_universe("shared-memory://dst-s11-b", &sc); - println!( - "dst corruption: {} rotted, {} truncated, {} latent, {} attributed detections, {} legal rejections", - a.reads_corrupted, - a.reads_truncated, - a.latent_errors, - a.corruption_detections.len(), - a.legal_rejections + let a = run_legacy_upgrade_read_weather( + "shared-memory://dst-upgrade-corrupt-a", + 83, + plan.clone(), + 30, + false, ); - for row in &a.corruption_detections { - println!("dst corruption detection: {row}"); - } - omnigraph_dst::harness::assert_strict_replay( - &a, - &b, - "corrupted universes must replay identically", + let b = run_legacy_upgrade_read_weather( + "shared-memory://dst-upgrade-corrupt-b", + 83, + plan, + 30, + false, ); + assert_eq!(a, b, "legacy upgrade corruption must replay identically"); assert!( - a.reads_corrupted + a.reads_truncated + a.latent_errors > 0, - "corruption weather should actually deliver lies (rotted={} truncated={} latent={})", - a.reads_corrupted, - a.reads_truncated, - a.latent_errors + a.reads_corrupted + a.reads_truncated > 0, + "weather must deliver damage" ); - assert!(a.verified > 0); + assert!( + a.checks_failed > 0, + "damage must flip at least one admission result" + ); + assert!( + a.checks_passed > 0, + "moderate weather must also allow clean admission" + ); + assert!(!a.detections.is_empty()); + assert!(a.missing_ir_refused); + assert!(!a.final_rows.is_empty()); } -/// Per-verb NON-VACUITY (the vacuity rule's universe-level half; -/// the char-level classifier tests live in `harness.rs`): under heavy -/// read-corruption weather every verb must DELIVER, and the attribution -/// window must actually convert engine-born failures into recorded -/// detections — proof the axis flips op outcomes rather than injecting -/// damage nothing consumes. The printed rows are the typed-vs-raw -/// triage worklist: expected shape today is RAW parse/format errors -/// (issue-candidate class, the retention-horizon precedent), not typed -/// integrity errors. +/// Fresh fixtures keep latent errors from hiding other verbs; each must deliver +/// and cause typed refusal. Truncation and latent cells require execution to +/// refuse before a fence; bit-rot checks permit harmless changed text. #[test] #[serial] -fn dst_corruption_detections_attributed() { - let sc = Scenario { - seed: 89, - ops: 30, - faults: Some(omnigraph_dst::harness::FaultPlan { +fn dst_legacy_upgrade_corruption_detections_attributed() { + use omnigraph_dst::harness::{FaultPlan, run_legacy_upgrade_read_weather}; + + for verb in ["bitrot", "truncate", "latent"] { + let plan = FaultPlan { seed: 8900, - corrupt_read_pct: 25, - truncate_read_pct: 10, - latent_read_pct: 4, + corrupt_read_pct: if verb == "bitrot" { 100 } else { 0 }, + truncate_read_pct: if verb == "truncate" { 100 } else { 0 }, + latent_read_pct: if verb == "latent" { 100 } else { 0 }, ..Default::default() - }), - ..Default::default() - }; - let a = run_universe("shared-memory://dst-s11-storm", &sc); - println!( - "dst corruption storm: {} rotted, {} truncated, {} latent, {} attributed detections", - a.reads_corrupted, - a.reads_truncated, - a.latent_errors, - a.corruption_detections.len() - ); - for row in &a.corruption_detections { - println!("dst corruption detection: {row}"); + }; + let execute = verb != "bitrot"; + let a = run_legacy_upgrade_read_weather( + &format!("shared-memory://dst-upgrade-{verb}-a"), + 89, + plan.clone(), + 4, + execute, + ); + let b = run_legacy_upgrade_read_weather( + &format!("shared-memory://dst-upgrade-{verb}-b"), + 89, + plan, + 4, + execute, + ); + assert_eq!( + a, b, + "{verb} legacy upgrade refusals must replay identically" + ); + let delivered = match verb { + "bitrot" => a.reads_corrupted, + "truncate" => a.reads_truncated, + "latent" => a.latent_errors, + _ => unreachable!(), + }; + assert!(delivered > 0, "{verb} must deliver"); + assert!( + a.checks_failed > 0, + "{verb} must alter an engine admission result" + ); + assert!( + !a.detections.is_empty(), + "{verb} must have attributed refusals" + ); + assert!(a.missing_ir_refused); + assert!( + !a.final_rows.is_empty(), + "clean retry must finish conversion" + ); } - assert!(a.reads_corrupted > 0, "bit rot must deliver under 25%"); - assert!(a.reads_truncated > 0, "truncation must deliver under 10%"); - assert!( - !a.corruption_detections.is_empty(), - "heavy corruption should provably flip at least one op outcome into \ - an attributed detection (otherwise the axis injects damage nothing \ - consumes)" - ); } -// The sidecar-weather pin (`dst_sidecar_weather_lost_and_misdirected`) -// retired with the recovery sidecars (RFC 0067 step 5): the lost-write and -// misdirected-write verbs act on adapter-realm content writes, which in a -// standard universe were exactly the `__recovery/` sidecars, so they could -// no longer bite. The `schema_ops` workload writes control objects again -// (the schema contract files), and `dst_schema_weather_persisted_write_verbs` -// below is that pin's successor. - -/// SCHEMA-OP REQUALIFICATION (the deferred half of the roll-12 quarantine): -/// the sampler's schema face live under clean storage. Monotone additive -/// applies interleave the standard mix on seeded schedules; the model -/// ignores them by construction, so every oracle judges the surrounding -/// workload across real staged-contract applies (RFC 0067's schema -/// staging). The face's first seed scan discovered the engine's mono-branch -/// restriction ("schema apply requires a graph with only main"), now a -/// model-predicted legal refusal — so branchy schedules validate the typed -/// refusal while branchless intervals run real applies. Clean IO in both -/// realms ⇒ strict replay identity is asserted. Seed 7 emits three applies. +/// Additive schema applies interleave the seeded workload and publish their +/// contract atomically. Branchy schedules require the mono-branch refusal; +/// the full oracle stack and strict replay check accepted applies. #[test] #[serial] fn dst_schema_ops_randomized_oracles_hold_and_replay() { @@ -1644,7 +1656,7 @@ fn dst_schema_ops_randomized_oracles_hold_and_replay() { } /// SCHEMA-APPLY CRASH WINDOWS under the randomized workload: a failure at the -/// detached table commit or the staged contract write, judged by reconcile, +/// effect gate or detached table commit, judged by reconcile, /// recovery reopen and the full oracle stack (bite-and-replay). #[cfg(feature = "failpoints")] #[test] @@ -1653,7 +1665,7 @@ fn dst_schema_apply_crash_windows_bite_and_replay() { let _s = omnigraph::seams::FailScenario::setup(); let cells: [(&str, u64, usize, bool); 2] = [ ("schema_apply.post_table_commit", 7, 30, true), - ("schema_apply.after_staging_write", 7, 30, true), + ("schema_apply.post_lock_pre_effect", 7, 30, true), ]; for (window, seed, ops, dies) in cells { // Crossing proof first: same seed, record-only callback on the seam @@ -1694,19 +1706,9 @@ fn dst_schema_apply_crash_windows_bite_and_replay() { } } -/// PERSISTED-TIER WRITE VERBS, revived (the successor of the retired -/// sidecar-weather pin above): with the schema face on, the adapter realm's -/// content writes are the schema contract files, and the write-time -/// corruption / lost-write / misdirected-write verbs have live subjects -/// again. These verbs are STORE LIES (success acknowledged, wrong or no -/// bytes durable), so the contract is the violation tier's -/// detected-or-harmless: either the damage only ever grazes writes whose -/// loss the apply absorbs (universe completes, oracles green), or the -/// engine detects the lie on the schema control plane and refuses LOUDLY, -/// which fails the universe's own observation reads — a caught, typed, -/// deterministic death, never silent acceptance. Seed 101 empirically takes -/// the refusal arm (a misdirected `__schema_state.json` install); both arms -/// assert pairwise determinism. +/// Persisted adapter lies during schema-changing workloads reach optimize's +/// binary graph-index artifact. Loss or misdirection must be delivered; +/// text-only corruption does not cover this artifact. #[test] #[serial] fn dst_schema_weather_persisted_lies_detected_or_harmless() { @@ -3526,36 +3528,21 @@ fn known_failure_family(msg: &str) -> Option<&'static str> { None } -/// THE FLEET: the volume instrument. Runs the full portfolio -/// across FRESH seed space (seeds no pin or hunt has ever used), each seed -/// one new life: (a) clean wide universe, (b) adapter-realm fault storm, -/// (c) ack-loss + client retry, (d) a crash-window universe (round-robin -/// over the catalog by seed — fresh-seed volume on the axis the hunt -/// covers systematically but only from seeds 7..12), (e) three crash -/// states from that seed's own enumeration space (probe learns W, -/// deterministic k picks). Every completed universe prints a -/// `dst fleet report` line (verdicts/channels/counters — the same -/// provenance columns the hunt records) so run tables read from the log. -/// A VIOLATION does not kill the pass: `run_universe_caught` records -/// (seed, arm, message) — a complete repro — and the fleet continues; -/// all failures are reported together at the end (red iff any). -/// Seed count: env `DST_FLEET_SEEDS` (default 60 ≈ a few minutes; -/// overnight scale = hundreds — set via a python subprocess wrapper, -/// never an env-prefix). Seed base: env `DST_FLEET_SEED_BASE` (default -/// 10_000; each pass owns a disjoint seed interval, reproducible by -/// base+count). Run explicitly: -/// cargo test -p omnigraph-dst dst_fleet -- --ignored --nocapture +/// Volume over clean, seeded engine-failure, adapter-ack and crash arms. +/// DST_FLEET_SEEDS defaults to 60; DST_FLEET_SEED_BASE defaults to 10_000. +/// Failures retain their scenario and replay after the full pass. #[test] #[serial] #[ignore = "instrument: fleet (N seeds x 7 universes, minutes to hours)"] fn dst_fleet() { + const DEFAULT_FLEET_SEEDS: u64 = 60; // The window arm schedules real crash windows — same setup the hunt // and the reach probe perform. let _s = omnigraph::seams::FailScenario::setup(); let n: u64 = std::env::var("DST_FLEET_SEEDS") .ok() .and_then(|s| s.trim().parse().ok()) - .unwrap_or(60); + .unwrap_or(DEFAULT_FLEET_SEEDS); // Fresh seed space: pins use <100, hunt 7..12, the first fleet pass 10_000..10_060. let base: u64 = std::env::var("DST_FLEET_SEED_BASE") .ok() @@ -3565,6 +3552,7 @@ fn dst_fleet() { // the auto-replay bundle can rerun the exact universe at pass end. let mut failures: Vec<(u64, String, String, String, Scenario)> = Vec::new(); let mut universes = 0usize; + let mut storm_effects = 0usize; let run = |seed: u64, arm: &str, root: String, @@ -3576,7 +3564,7 @@ fn dst_fleet() { // Per-universe provenance line (columns per the test doc). println!( "dst fleet report seed={seed} arm={arm} crashes={} crossed={} hit={} \ - writes={} acks_lost={} retries={} lance={} legal={} verdicts={:?} issues={:?}", + writes={} acks_lost={} retries={} lance={} legal={} engine_effects={} writes_after_effect={} verdicts={:?} issues={:?}", r.crashes, r.crossed, r.crash_state_hit, @@ -3585,6 +3573,8 @@ fn dst_fleet() { r.client_retries, r.lance_realm_injected, r.legal_rejections, + r.engine_effects_injected, + r.writes_after_engine_effect, r.reconcile_verdicts, r.known_issues ); @@ -3625,7 +3615,7 @@ fn dst_fleet() { &mut failures, ); universes += 1; - run( + let storm = run( seed, "storm", format!("shared-memory://dst-fleet-{seed}-storm"), @@ -3635,6 +3625,7 @@ fn dst_fleet() { faults: Some(omnigraph_dst::harness::FaultPlan { seed: seed.wrapping_mul(101), error_pct: 10, + engine_effect_error_pct: 35, read_error_pct: 5, latency_pct: 20, max_latency_ms: 5, @@ -3647,10 +3638,13 @@ fn dst_fleet() { }, &mut failures, ); + if let Some(report) = storm { + storm_effects += report.engine_effects_injected; + } universes += 1; run( seed, - "ack-retry", + "adapter-ack", format!("shared-memory://dst-fleet-{seed}-ack"), Scenario { seed, @@ -3757,6 +3751,14 @@ fn dst_fleet() { "fleet found {} violating universes (repros above, bundles below them)", failures.len() ); + let require_effect_coverage = n >= DEFAULT_FLEET_SEEDS; + println!( + "dst fleet coverage seeds={n} storm_effects={storm_effects} required={require_effect_coverage}" + ); + assert!( + !require_effect_coverage || storm_effects > 0, + "fleet coverage gap: the full pass delivered no engine-effect failures" + ); } /// the auto-replay triage bundle: rerun one failing diff --git a/crates/omnigraph-gqt/src/dst_runner.rs b/crates/omnigraph-gqt/src/dst_runner.rs index 9a401ccf3..15f7252d6 100644 --- a/crates/omnigraph-gqt/src/dst_runner.rs +++ b/crates/omnigraph-gqt/src/dst_runner.rs @@ -92,7 +92,7 @@ fn lifecycle_probe() -> ( std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)), ]; let guards = [ - &omnigraph::seams::catalog::INIT_AFTER_SCHEMA_CONTRACT_WRITTEN, + &omnigraph::seams::catalog::INIT_AFTER_COORDINATOR_INIT, &omnigraph::seams::catalog::OPEN_BEFORE_SCHEMA_CONTRACT_READ, ] .into_iter() diff --git a/crates/omnigraph-server/src/lib.rs b/crates/omnigraph-server/src/lib.rs index 5399b07ae..9e7e703b2 100644 --- a/crates/omnigraph-server/src/lib.rs +++ b/crates/omnigraph-server/src/lib.rs @@ -2042,12 +2042,13 @@ mod external_blob_startup_tests { Omnigraph::init(graph.to_string_lossy().as_ref(), schema) .await .unwrap(); - // A read-write open normally removes this matching no-op staging - // residue. The invalid policy must be rejected before graph open, so - // startup cannot perform even that safe recovery mutation first. - let staging = graph.join("_schema.pg.staging"); - std::fs::write(&staging, schema).unwrap(); - assert!(staging.exists()); + let recovery = graph.join("__recovery"); + std::fs::create_dir_all(&recovery).unwrap(); + std::fs::write(recovery.join("unresolved.json"), "malformed sidecar").unwrap(); + assert!(matches!( + Omnigraph::open(graph.to_str().unwrap()).await, + Err(OmniError::RecoveryRequired { .. }) + )); let policy: omnigraph::ExternalBlobPolicy = serde_json::from_value(serde_json::json!({ "mode": "allow", "bases": [{ @@ -2076,11 +2077,10 @@ mod external_blob_startup_tests { .contains("server-safe external Blob base may not use file://"), "unexpected refusal: {error:?}" ); - assert!( - staging.exists(), - "invalid server policy must be refused before read-write open recovery moves graph state" + assert_eq!( + std::fs::read_to_string(recovery.join("unresolved.json")).unwrap(), + "malformed sidecar" ); - assert_eq!(std::fs::read_to_string(staging).unwrap(), schema); } } diff --git a/crates/omnigraph-server/tests/boot_settings.rs b/crates/omnigraph-server/tests/boot_settings.rs index a8f3e5a6d..b1839d899 100644 --- a/crates/omnigraph-server/tests/boot_settings.rs +++ b/crates/omnigraph-server/tests/boot_settings.rs @@ -125,11 +125,13 @@ async fn data_trust_root_mismatch_refuses_before_recovery_open() { let tokens = data_tokens::DataTokens::new(); let temp = converged_cluster_dir("").await; let graph = temp.path().join("graphs/knowledge.omni"); - let schema = fs::read_to_string(temp.path().join("people.pg")).unwrap(); - // A read-write engine open normally cleans matching no-op schema staging. - // Wrong public trust must refuse before even that recovery effect. - let staging = graph.join("_schema.pg.staging"); - fs::write(&staging, &schema).unwrap(); + let recovery = graph.join("__recovery"); + fs::create_dir_all(&recovery).unwrap(); + fs::write(recovery.join("unresolved.json"), "malformed sidecar").unwrap(); + assert!(matches!( + Omnigraph::open(graph.to_str().unwrap()).await, + Err(omnigraph::error::OmniError::RecoveryRequired { .. }) + )); let trust_path = temp.path().join("trust.json"); fs::write(&trust_path, serde_json::to_vec(&tokens.document).unwrap()).unwrap(); let result = omnigraph_server::load_server_settings_with_data_token_trust( @@ -146,10 +148,6 @@ async fn data_trust_root_mismatch_refuses_before_recovery_open() { .to_string() .contains("serving-root binding") ); - assert!( - staging.exists(), - "invalid trust must not open the graph for recovery" - ); } #[tokio::test] @@ -159,8 +157,14 @@ async fn oidc_root_mismatch_refuses_even_beside_valid_native_trust_before_recove use serde_json::json; let temp = converged_cluster_dir("").await; - let staging = temp.path().join("graphs/knowledge.omni/_schema.pg.staging"); - fs::copy(temp.path().join("people.pg"), &staging).unwrap(); + let graph = temp.path().join("graphs/knowledge.omni"); + let recovery = graph.join("__recovery"); + fs::create_dir_all(&recovery).unwrap(); + fs::write(recovery.join("unresolved.json"), "malformed sidecar").unwrap(); + assert!(matches!( + Omnigraph::open(graph.to_str().unwrap()).await, + Err(omnigraph::error::OmniError::RecoveryRequired { .. }) + )); let root = format!( "file://{}", fs::canonicalize(temp.path()).unwrap().display() @@ -199,10 +203,6 @@ async fn oidc_root_mismatch_refuses_even_beside_valid_native_trust_before_recove .to_string() .contains("OIDC identity snapshot or serving-root binding") ); - assert!( - staging.exists(), - "OIDC failure opened the graph for recovery" - ); } #[tokio::test] @@ -1029,10 +1029,8 @@ rules: } } -/// Exercise the production listener and shutdown driver in a contained process. -/// A no-op schema pass holds the engine's schema gate so the HTTP write -/// parks at an async acquire; a blocking HTTP seam would keep the connection -/// alive by itself and could hide missing request ownership. +/// Exercise production shutdown while a schema refresh holds the exclusive gate. +/// The HTTP write parks asynchronously so a disconnected client can be observed. #[cfg(unix)] mod owned_shutdown { use super::*; @@ -1141,17 +1139,12 @@ mod owned_shutdown { let graph = graph_path(&root); let direct = Omnigraph::open(graph.to_str().unwrap()).await.unwrap(); use omnigraph::seams::catalog::{ - MUTATION_POST_STAGE_PRE_EFFECT_GATE, SCHEMA_APPLY_POST_SENTINEL, + MUTATION_POST_STAGE_PRE_EFFECT_GATE, SCHEMA_RELOAD_BEFORE_CONTRACT_READ, }; - let (hold_guard, hold) = SCHEMA_APPLY_POST_SENTINEL.hold(); + let (hold_guard, hold) = SCHEMA_RELOAD_BEFORE_CONTRACT_READ.hold(); let observed_root = root.clone(); let staged_hold = hold.clone(); let stage_guard = MUTATION_POST_STAGE_PRE_EFFECT_GATE.observe(move || { - // Stage against the accepted snapshot first, then interleave a - // no-op schema pass. It takes the exclusive gate without changing - // HEAD, so the HTTP write parks asynchronously on that gate after - // this callback returns. Blocking the HTTP worker itself would - // hide cancellation behind a connection that never gets polled. fs::write(observed_root.join("start-holder"), b"start").unwrap(); staged_hold.wait_until_reached(); fs::write(observed_root.join("http-staged"), b"reached").unwrap(); @@ -1172,11 +1165,7 @@ mod owned_shutdown { while !direct_root.join("start-holder").exists() { tokio::time::sleep(Duration::from_millis(10)).await; } - let result = direct - .apply_schema(&fs::read_to_string(fixture("test.pg")).unwrap()) - .await - .unwrap(); - assert!(!result.applied, "the gate holder must not move graph HEAD"); + direct.refresh().await.unwrap(); }); let signal_root = root.clone(); let mut terminate = diff --git a/crates/omnigraph-server/tests/data_routes.rs b/crates/omnigraph-server/tests/data_routes.rs index b76a1cfb7..f42e4cf4c 100644 --- a/crates/omnigraph-server/tests/data_routes.rs +++ b/crates/omnigraph-server/tests/data_routes.rs @@ -1511,7 +1511,7 @@ impl omnigraph::seams::Decide for MergeReturnHold { #[tokio::test(flavor = "multi_thread")] async fn disconnected_writes_keep_admission_until_the_original_operation_finishes() { use omnigraph::seams::catalog::{ - BRANCH_CREATE_POST_INVENTORY_PRE_NATIVE, SCHEMA_APPLY_POST_SENTINEL, + BRANCH_CREATE_POST_INVENTORY_PRE_NATIVE, SCHEMA_RELOAD_BEFORE_CONTRACT_READ, }; const ATOMIC: &str = r#"query atomic() { insert Person { name: "Owned", age: 41 } @@ -1585,12 +1585,7 @@ async fn disconnected_writes_keep_admission_until_the_original_operation_finishe // schema gate, without publishing a main graph commit. holder_db.branch_create("gate-holder").await.unwrap(); } else { - let unchanged_schema = fs::read_to_string(fixture("test.pg")).unwrap(); - let result = holder_db.apply_schema(&unchanged_schema).await.unwrap(); - assert!( - !result.applied, - "the holder must not alter the victim's authority" - ); + holder_db.refresh().await.unwrap(); } }); }); @@ -1598,7 +1593,7 @@ async fn disconnected_writes_keep_admission_until_the_original_operation_finishe let seam = if door == "/branches/merge" { &BRANCH_CREATE_POST_INVENTORY_PRE_NATIVE } else { - &SCHEMA_APPLY_POST_SENTINEL + &SCHEMA_RELOAD_BEFORE_CONTRACT_READ }; let guard = seam.install(Arc::new(MergeReturnHold { thread: holder_thread.thread().id(), @@ -4457,8 +4452,17 @@ async fn pre_effect_delete_failure_preserves_receipt_and_write_capacity_under_sl .unwrap(); assert_eq!(held_read.status(), StatusCode::OK); assert_eq!(operations.snapshot().active_reads, 1); - let schema = graph.join("_schema.pg"); - let saved = graph.join("schema-held.pg"); + let mut branch_refs = fs::read_dir(graph.join("__manifest/_refs/branches")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .filter(|path| { + path.extension() + .is_some_and(|extension| extension == "json") + }) + .collect::>(); + assert_eq!(branch_refs.len(), 1, "the fixture has one named branch"); + let branch_ref = branch_refs.pop().unwrap(); + let saved = fs::read(&branch_ref).unwrap(); let (status, output) = if compound { let request_app = app.clone(); let (start, started) = std::sync::mpsc::channel(); @@ -4477,11 +4481,10 @@ async fn pre_effect_delete_failure_preserves_receipt_and_write_capacity_under_sl )) }); let thread = request.thread().id(); - let source = schema.clone(); - let destination = saved.clone(); + let source = branch_ref.clone(); let guard = BRANCH_MERGE_PRE_RETURN.observe(move || { if std::thread::current().id() == thread { - fs::rename(&source, &destination).unwrap(); + fs::write(&source, b"{").unwrap(); } }); start.send(()).unwrap(); @@ -4489,7 +4492,7 @@ async fn pre_effect_delete_failure_preserves_receipt_and_write_capacity_under_sl drop(guard); result } else { - fs::rename(&schema, &saved).unwrap(); + fs::write(&branch_ref, b"{").unwrap(); json_response( &app, Request::builder() @@ -4501,7 +4504,7 @@ async fn pre_effect_delete_failure_preserves_receipt_and_write_capacity_under_sl ) .await }; - fs::rename(&saved, &schema).unwrap(); + fs::write(&branch_ref, &saved).unwrap(); if compound { assert_eq!(status, StatusCode::OK, "{output}"); assert_eq!(output["outcome"], "fast_forward"); @@ -4518,7 +4521,7 @@ async fn pre_effect_delete_failure_preserves_receipt_and_write_capacity_under_sl } assert!( !operations.snapshot().closed, - "a proven schema read failure has no deletion effects" + "a proven manifest read failure has no deletion effects" ); assert_eq!(workload.snapshot().ingress_count, 0); let (status, sentinel) = json_response( diff --git a/crates/omnigraph-server/tests/schema_routes.rs b/crates/omnigraph-server/tests/schema_routes.rs index 3e85425ae..e50aacb2a 100644 --- a/crates/omnigraph-server/tests/schema_routes.rs +++ b/crates/omnigraph-server/tests/schema_routes.rs @@ -9,9 +9,7 @@ use axum::body::Body; use axum::http::{Method, Request, StatusCode}; use omnigraph::db::{Omnigraph, ReadTarget}; use omnigraph::loader::LoadMode; -use omnigraph_server::api::{ - ChangeRequest, ErrorOutput, ReadRequest, SchemaApplyRequest, SchemaOutput, -}; +use omnigraph_server::api::{ErrorOutput, SchemaApplyRequest, SchemaOutput}; use omnigraph_server::{ AppState, GraphHandle, GraphId, GraphKey, PolicyEngine, build_app, workload, }; @@ -579,96 +577,6 @@ async fn schema_apply_route_rejects_when_non_main_branch_exists() { ); } -#[tokio::test(flavor = "multi_thread")] -async fn schema_drift_returns_conflict_for_snapshot_read_and_change() { - let (temp, app) = app_for_loaded_graph().await; - let graph = graph_path(temp.path()); - fs::write(graph.join("_schema.pg"), drifted_test_schema()).unwrap(); - - let (snapshot_status, snapshot_body) = json_response( - &app, - Request::builder() - .header(HTTP_API_CONTRACT_HEADER, HTTP_API_CONTRACT) - .uri(g("/snapshot?branch=main")) - .method(Method::GET) - .body(Body::empty()) - .unwrap(), - ) - .await; - let snapshot_error: ErrorOutput = serde_json::from_value(snapshot_body).unwrap(); - assert_eq!(snapshot_status, StatusCode::CONFLICT); - assert_eq!( - snapshot_error.code, - Some(omnigraph_server::api::ErrorCode::Conflict) - ); - assert!( - snapshot_error - .error - .contains("schema evolution is locked down in phase 1") - ); - - let read = ReadRequest { - query_source: fs::read_to_string(fixture("test.gq")).unwrap(), - query_name: Some("get_person".to_string()), - params: Some(json!({ "name": "Alice" })), - branch: Some("main".to_string()), - snapshot: None, - settings: None, - }; - let (read_status, read_body) = json_response( - &app, - Request::builder() - .header(HTTP_API_CONTRACT_HEADER, HTTP_API_CONTRACT) - .uri(g("/read")) - .method(Method::POST) - .header("content-type", "application/json") - .body(Body::from(serde_json::to_vec(&read).unwrap())) - .unwrap(), - ) - .await; - let read_error: ErrorOutput = serde_json::from_value(read_body).unwrap(); - assert_eq!(read_status, StatusCode::CONFLICT); - assert_eq!( - read_error.code, - Some(omnigraph_server::api::ErrorCode::Conflict) - ); - assert!( - read_error - .error - .contains("schema evolution is locked down in phase 1") - ); - - let change = ChangeRequest { - query: MUTATION_QUERIES.to_string(), - name: Some("insert_person".to_string()), - params: Some(json!({ "name": "Mina", "age": 28 })), - branch: Some("main".to_string()), - settings: None, - }; - let (change_status, change_body) = json_response( - &app, - Request::builder() - .header(HTTP_API_CONTRACT_HEADER, HTTP_API_CONTRACT) - .uri(g("/change")) - .method(Method::POST) - .header("content-type", "application/json") - .body(Body::from(serde_json::to_vec(&change).unwrap())) - .unwrap(), - ) - .await; - let change_error: ErrorOutput = serde_json::from_value(change_body).unwrap(); - assert_eq!(change_status, StatusCode::CONFLICT); - assert_eq!( - change_error.code, - Some(omnigraph_server::api::ErrorCode::Conflict) - ); - assert!( - change_error - .error - .contains("schema evolution is locked down in phase 1") - ); -} - #[tokio::test(flavor = "multi_thread")] async fn schema_route_returns_current_source() { let (_temp, app) = app_for_loaded_graph().await; diff --git a/crates/omnigraph-server/tests/support/mod.rs b/crates/omnigraph-server/tests/support/mod.rs index 205e9d9d1..17aec007c 100644 --- a/crates/omnigraph-server/tests/support/mod.rs +++ b/crates/omnigraph-server/tests/support/mod.rs @@ -301,12 +301,6 @@ pub fn get_request(uri: &str, token: &str) -> Request { .unwrap() } -pub fn drifted_test_schema() -> String { - fs::read_to_string(fixture("test.pg")) - .unwrap() - .replace("age: I32?", "age: I64?") -} - pub async fn manifest_dataset_version(graph: &Path) -> u64 { Omnigraph::open(graph.to_string_lossy().as_ref()) .await diff --git a/crates/omnigraph/src/db/catalog_tests.rs b/crates/omnigraph/src/db/catalog_tests.rs index f0e49a9f5..bb7db406c 100644 --- a/crates/omnigraph/src/db/catalog_tests.rs +++ b/crates/omnigraph/src/db/catalog_tests.rs @@ -283,7 +283,7 @@ async fn open_refuses_a_stamp_below_the_served_floor_before_any_effect() { }; let error = result.err().expect("a v9 stamp is below the served floor"); assert!( - error.to_string().contains("reads only v11 to v12"), + error.to_string().contains("reads only v13 to v13"), "{error}" ); assert!(error.to_string().contains("omnigraph upgrade"), "{error}"); @@ -308,7 +308,7 @@ async fn open_refuses_a_stamp_below_the_served_floor_before_any_effect() { #[tokio::test] async fn open_refuses_unknown_schema_features_before_recovery() { let _scenario = crate::seams::FailScenario::setup(); - for staged in [false, true] { + { let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); drop( @@ -316,21 +316,21 @@ async fn open_refuses_unknown_schema_features_before_recovery() { .await .unwrap(), ); - let live_path = dir.path().join(crate::db::schema_state::SCHEMA_IR_FILENAME); - let mut ir: serde_json::Value = - serde_json::from_str(&std::fs::read_to_string(&live_path).unwrap()).unwrap(); + let mut manifest = ManifestCoordinator::open(uri).await.unwrap(); + let live_row = manifest.read_schema_contract().await.unwrap(); + let mut ir: serde_json::Value = serde_json::from_str(&live_row.ir).unwrap(); ir["features"] .as_array_mut() .unwrap() .push(serde_json::Value::String("time-travel".into())); - let target = if staged { - dir.path() - .join(crate::db::schema_state::SCHEMA_IR_STAGING_FILENAME) - } else { - live_path - }; let tampered = serde_json::to_string(&ir).unwrap(); - std::fs::write(&target, &tampered).unwrap(); + manifest + .commit_changes(&[ManifestChange::SchemaContract(SchemaContractRow { + ir: tampered.clone(), + ..live_row + })]) + .await + .unwrap(); let before_version = open_manifest_dataset(uri, None) .await .unwrap() @@ -363,18 +363,21 @@ async fn open_refuses_unknown_schema_features_before_recovery() { .expect("an unknown feature name must refuse open"); assert!( error.to_string().contains("unknown to this build"), - "staged {staged}: {error}" + "{error}" ); assert_eq!( reached_effects.load(std::sync::atomic::Ordering::SeqCst), 0, "unknown feature names must refuse before the local write probe or recovery" ); - assert_eq!( - std::fs::read_to_string(&target).unwrap(), - tampered, - "staged {staged}: the refused artifact must be left in place" - ); + let planted = ManifestCoordinator::open(uri) + .await + .unwrap() + .read_schema_contract() + .await + .unwrap() + .ir; + assert_eq!(planted, tampered, "the refused row must be left unchanged"); assert_eq!( open_manifest_dataset(uri, None) .await @@ -1007,7 +1010,7 @@ mod migrations_tests { .unwrap(); set_stamp(&mut manifest, 10).await.unwrap(); let refused = guard_stamp(&manifest).unwrap_err().to_string(); - assert!(refused.contains("reads only v11 to v12"), "{refused}"); + assert!(refused.contains("reads only v13 to v13"), "{refused}"); { let _admission = admit_conversion_source(root, 10); assert_eq!(guard_stamp(&manifest).unwrap(), 10); diff --git a/crates/omnigraph/src/db/graph_coordinator.rs b/crates/omnigraph/src/db/graph_coordinator.rs index 05c8e495b..e85e4800a 100644 --- a/crates/omnigraph/src/db/graph_coordinator.rs +++ b/crates/omnigraph/src/db/graph_coordinator.rs @@ -9,11 +9,10 @@ use crate::error::{OmniError, Result}; use crate::storage::{StorageAdapter, normalize_root_uri}; use super::commit_graph::{CommitGraph, CommitGraphSnapshot, FirstParentEdge, GraphCommit}; -use super::is_internal_system_branch; use super::manifest::{ CapturedManifestProbe, DatasetUpdate, ExpectedTableVersions, GenesisManifestAttempt, LineageIntent, LineageRefresh, ManifestChange, ManifestCoordinator, ManifestIncarnation, - ManifestInitError, PublishPrecondition, + ManifestInitError, PublishPrecondition, SchemaContractRow, }; use super::snapshot::Snapshot; use crate::seams::{decide_seam, fail}; @@ -139,6 +138,7 @@ impl GraphCoordinator { pub(crate) async fn init_commit_with_session( root_uri: &str, catalog: &Catalog, + contract: &SchemaContractRow, control_session: &Arc, attempt: &GenesisManifestAttempt, ) -> std::result::Result { @@ -146,7 +146,7 @@ impl GraphCoordinator { // The genesis graph commit is folded into the manifest init write, so // `__manifest` is the single source of graph lineage from version one // (RFC-013 Phase 7). - ManifestCoordinator::init_commit(&root, catalog, control_session, attempt).await + ManifestCoordinator::init_commit(&root, catalog, contract, control_session, attempt).await } /// Reopen an acknowledgement-unknown manifest Create and construct a @@ -227,6 +227,40 @@ impl GraphCoordinator { Self::open_branch_with_session(root_uri, branch, storage, &control_session).await } + pub(crate) async fn open_with_contract( + root_uri: &str, + storage: Arc, + prepared: crate::db::manifest::PreparedManifestOpen, + ) -> Result<(Self, SchemaContractRow)> { + let root = normalize_root_uri(root_uri)?; + let (manifest, lineage_rows, contract) = + ManifestCoordinator::open_prepared_with_lineage_and_contract(&root, prepared).await?; + let commit_graph = CommitGraph::from_manifest_rows(&root, None, lineage_rows); + let mut coordinator = Self { + root_uri: root, + storage, + manifest, + commit_graph, + bound_branch: None, + }; + let contract = coordinator.refresh_contract_capture(contract).await?; + Ok((coordinator, contract)) + } + + async fn refresh_contract_capture( + &mut self, + contract: Result, + ) -> Result { + let captured = self.snapshot(); + self.refresh().await?; + self.manifest.validate_serving_format()?; + if self.snapshot().same_manifest_image(&captured) { + contract + } else { + self.read_schema_contract().await + } + } + pub(crate) async fn open_branch_with_session( root_uri: &str, branch: &str, @@ -318,6 +352,12 @@ impl GraphCoordinator { Snapshot::wrap(self.manifest.snapshot()) } + /// Read the contract of the same pinned manifest image as [`Self::snapshot`], + /// using captured content when available and a filtered scan otherwise. + pub(crate) async fn read_schema_contract(&self) -> Result { + self.manifest.read_schema_contract().await + } + pub fn current_branch(&self) -> Option<&str> { self.bound_branch.as_deref() } @@ -370,16 +410,7 @@ impl GraphCoordinator { } pub async fn branch_list(&self) -> Result> { - self.manifest.list_graph_branches().await.map(|branches| { - branches - .into_iter() - .filter(|branch| !is_internal_system_branch(branch)) - .collect() - }) - } - - pub(crate) async fn schema_apply_locked(&self) -> Result { - self.manifest.schema_apply_locked().await + self.manifest.list_graph_branches().await } pub(crate) async fn all_branches(&self) -> Result> { @@ -403,23 +434,9 @@ impl GraphCoordinator { self.manifest.create_branch(&branch).await } - pub(crate) async fn branch_delete(&mut self, name: &str) -> Result<()> { - let branch = normalize_branch_name(name)? - .ok_or_else(|| OmniError::manifest("cannot delete branch 'main'".to_string()))?; - if self.current_branch() == Some(branch.as_str()) { - return Err(OmniError::manifest_conflict(format!( - "cannot delete currently active branch '{}'", - branch - ))); - } - - self.manifest.delete_branch(&branch).await - } - /// Delete the branch represented by an operation-local post-gate capture. /// - /// Unlike [`Self::branch_delete`], this permits the disposable coordinator - /// itself to be bound to `name`. The exact captured BranchIdentifier fences + /// The disposable coordinator may be bound to `name`. Its captured BranchIdentifier fences /// delete/recreate ABA; the caller discards this coordinator after the /// native authority change. pub(crate) async fn branch_delete_captured( @@ -449,33 +466,48 @@ impl GraphCoordinator { pub async fn resolve_snapshot_id(&self, branch: &str) -> Result { let normalized = normalize_branch_name(branch)?; - let other = match normalized.as_deref() { - Some(branch) => { - GraphCoordinator::open_branch_with_session( - self.root_uri(), - branch, - Arc::clone(&self.storage), - &self.manifest.control_session(), - ) + let opened; + let coordinator = if normalized.as_deref() == self.current_branch() + && self + .probe_latest_incarnation() .await? - } - None => { - GraphCoordinator::open_with_session( - self.root_uri(), - Arc::clone(&self.storage), - &self.manifest.control_session(), - ) - .await? - } + .matches(&self.manifest_incarnation()) + { + self + } else { + opened = match normalized.as_deref() { + Some(branch) => { + GraphCoordinator::open_branch_with_session( + self.root_uri(), + branch, + Arc::clone(&self.storage), + &self.manifest.control_session(), + ) + .await? + } + None => { + GraphCoordinator::open_with_session( + self.root_uri(), + Arc::clone(&self.storage), + &self.manifest.control_session(), + ) + .await? + } + }; + &opened }; - Ok(other.head_commit_id().await?.unwrap_or_else(|| { - SnapshotId::synthetic( - other.current_branch(), - other.version(), - other.manifest_incarnation().e_tag.as_deref(), - ) - })) + Ok(coordinator + .effective_graph_head() + .await? + .map(SnapshotId::new) + .unwrap_or_else(|| { + SnapshotId::synthetic( + coordinator.current_branch(), + coordinator.version(), + coordinator.manifest_incarnation().e_tag.as_deref(), + ) + })) } pub async fn resolve_target(&self, target: &ReadTarget) -> Result { @@ -759,9 +791,7 @@ impl GraphCoordinator { /// Publish a pre-minted lineage intent under an explicit authority /// precondition. The intent's identity and timestamp remain stable across - /// publisher retries and can also be persisted by the caller before this - /// method is invoked (schema apply records the commit id in its staged - /// contract). + /// publisher retries. pub(crate) async fn commit_changes_with_intent_and_expected( &mut self, changes: &[ManifestChange], @@ -885,6 +915,125 @@ fn normalize_branch_name(branch: &str) -> Result> { mod tests { use super::*; + #[tokio::test] + async fn cold_open_refresh_uses_content_only_from_the_held_image() { + #[cfg(feature = "failpoints")] + let _scenario = crate::seams::FailScenario::setup(); + const CAPTURE_ERROR: &str = "captured contract content error"; + for advance in [false, true] { + for captured_error in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + let _owner = crate::db::Omnigraph::init(root, "node Person { name: String }") + .await + .unwrap(); + let session = crate::lance_access::control_session(); + let (manifest, lineage, captured) = + ManifestCoordinator::open_with_lineage_and_contract(root, None, &session) + .await + .unwrap(); + let mut expected = captured.unwrap(); + let old_head = expected.head.clone(); + let old_version = manifest.version(); + let mut reader = GraphCoordinator { + root_uri: normalize_root_uri(root).unwrap(), + storage: crate::storage::storage_for_uri(root).unwrap(), + manifest, + commit_graph: CommitGraph::from_manifest_rows(root, None, lineage), + bound_branch: None, + }; + let captured = if captured_error { + Err(OmniError::manifest_internal(CAPTURE_ERROR)) + } else { + Ok(expected.clone()) + }; + let expected_version = if advance { + expected.source = format!("\n{}\n", expected.source); + expected.ir = format!("\n{}\n", expected.ir); + assert_eq!(expected.head, old_head); + let mut writer = ManifestCoordinator::open_with_session(root, &session) + .await + .unwrap(); + let version = writer + .commit_changes(&[ManifestChange::SchemaContract(expected.clone())]) + .await + .unwrap(); + assert!(version > old_version); + version + } else { + old_version + }; + let probes = crate::instrumentation::QueryIoProbes::default(); + let scans = Arc::clone(&probes.manifest_scan_count); + let result = crate::instrumentation::with_query_io_probes( + probes, + reader.refresh_contract_capture(captured), + ) + .await; + assert_eq!(reader.version(), expected_version); + assert_eq!( + reader.manifest.snapshot().schema_contract(), + Some(&old_head) + ); + if captured_error && !advance { + match result.unwrap_err() { + OmniError::Manifest(error) => { + assert_eq!(error.kind, crate::error::ManifestErrorKind::Internal); + assert_eq!(error.message, CAPTURE_ERROR); + assert!(error.details.is_none()); + assert!(!error.publication_in_doubt); + } + error => panic!("unexpected capture error: {error:?}"), + } + } else { + assert_eq!(result.unwrap(), expected); + } + let scans = scans.load(std::sync::atomic::Ordering::Relaxed); + if advance { + assert!(scans > 0, "the replacement must read its pinned content"); + } else { + assert_eq!(scans, 0, "unchanged success/error must reuse the capture"); + } + } + } + } + + #[tokio::test] + async fn prepared_cold_open_refreshes_content_published_after_admission() { + #[cfg(feature = "failpoints")] + let _scenario = crate::seams::FailScenario::setup(); + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + let _owner = crate::db::Omnigraph::init(root, "node Person { name: String }") + .await + .unwrap(); + let session = crate::lance_access::control_session(); + let prepared = ManifestCoordinator::prepare_open_with_contract(root, &session) + .await + .unwrap(); + let mut writer = ManifestCoordinator::open_with_session(root, &session) + .await + .unwrap(); + let mut expected = writer.read_schema_contract().await.unwrap(); + let old_head = expected.head.clone(); + expected.source = format!("\n{}\n", expected.source); + expected.ir = format!("\n{}\n", expected.ir); + let version = writer + .commit_changes(&[ManifestChange::SchemaContract(expected.clone())]) + .await + .unwrap(); + let (reader, contract) = GraphCoordinator::open_with_contract( + root, + crate::storage::storage_for_uri(root).unwrap(), + prepared, + ) + .await + .unwrap(); + assert_eq!(reader.version(), version); + assert_eq!(contract.head, old_head); + assert_eq!(contract, expected); + } + fn commit( id: &str, parent_commit_id: Option<&str>, @@ -941,4 +1090,68 @@ mod tests { ResolvedCommitRange::Arbitrary { .. } )); } + + #[tokio::test] + async fn prepared_cold_open_refuses_a_legacy_stamp_published_after_admission() { + #[cfg(feature = "failpoints")] + let _scenario = crate::seams::FailScenario::setup(); + for admit_conversion in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + let _owner = crate::db::Omnigraph::init(root, "node Person { name: String }") + .await + .unwrap(); + let session = crate::lance_access::control_session(); + let prepared = ManifestCoordinator::prepare_open_with_contract(root, &session) + .await + .unwrap(); + let mut dataset = crate::db::manifest::layout::open_manifest_dataset_with_session( + root, None, &session, + ) + .await + .unwrap(); + let captured_version = dataset.version().version; + crate::db::manifest::migrations::set_stamp_for_test(&mut dataset, 12) + .await + .unwrap(); + assert!(dataset.version().version > captured_version); + assert_eq!( + crate::db::manifest::migrations::read_stamp(&dataset), + Some(12) + ); + let _admission = admit_conversion + .then(|| crate::db::manifest::migrations::admit_conversion_source(root, 12)); + let probes = crate::instrumentation::QueryIoProbes::default(); + let opens = Arc::clone(&probes.internal_open_count); + let result = crate::instrumentation::with_query_io_probes( + probes, + GraphCoordinator::open_with_contract( + root, + crate::storage::storage_for_uri(root).unwrap(), + prepared, + ), + ) + .await; + assert_eq!(opens.load(std::sync::atomic::Ordering::Relaxed), 1); + let error = result + .err() + .expect("final refreshed image must still be served format"); + let OmniError::Manifest(error) = error else { + panic!("expected typed format refusal: {error:?}"); + }; + assert_eq!(error.kind, crate::error::ManifestErrorKind::BadRequest); + assert!( + error.message.contains("internal schema v12"), + "{}", + error.message + ); + assert!( + error.message.contains("reads only v13"), + "{}", + error.message + ); + assert!(error.details.is_none()); + assert!(!error.publication_in_doubt); + } + } } diff --git a/crates/omnigraph/src/db/mod.rs b/crates/omnigraph/src/db/mod.rs index ecd3f1ca5..91e019a6c 100644 --- a/crates/omnigraph/src/db/mod.rs +++ b/crates/omnigraph/src/db/mod.rs @@ -11,11 +11,9 @@ mod system_roles_tests; pub(crate) mod upgrade; pub(crate) mod write_queue; -pub(crate) use crate::branch_names::{SCHEMA_APPLY_LOCK_BRANCH, is_schema_apply_lock_branch}; pub use commit_graph::GraphCommit; pub use graph_coordinator::{ReadTarget, ResolvedTarget, SnapshotId}; pub use manifest::{DatasetEntry, DatasetUpdate}; -pub(crate) use omnigraph::ensure_public_branch_ref; pub use omnigraph::{ CleanupPolicyOptions, CollectorCost, CollectorPathSnapshot, CollectorReport, CollectorRowSummary, DatasetCleanupStats, DatasetOptimizeStats, DatasetRepairStats, @@ -29,8 +27,11 @@ pub use omnigraph::{ pub(crate) use omnigraph::{WriteAuthorityToken, WriteTxn}; pub(crate) use omnigraph::{export_blob_values, logical_row_image}; pub(crate) use omnigraph_core::metadata::now_micros; -pub(crate) use schema_state::SchemaContractText; +pub(crate) use schema_state::SchemaContractIdentity; pub use snapshot::{Snapshot, SnapshotDataset, SnapshotScanner}; +#[cfg(feature = "dst")] +#[doc(hidden)] +pub use upgrade::dst_prepare_legacy_upgrade_fixture; pub use upgrade::{ UpgradeFinding, UpgradeMode, UpgradeOptions, UpgradeOutcome, UpgradeRecovery, UpgradeReport, UpgradeWork, upgrade_storage, upgrade_storage_as, @@ -108,10 +109,3 @@ impl MutationOpKind { } } } - -pub(crate) fn is_internal_system_branch(name: &str) -> bool { - // Legacy `__run__*` staging branches (Run state machine, removed MR-771) - // are swept off `__manifest` by the v2→v3 internal-schema migration, so the - // only internal branch the engine still creates is the schema-apply lock. - is_schema_apply_lock_branch(name) -} diff --git a/crates/omnigraph/src/db/omnigraph.rs b/crates/omnigraph/src/db/omnigraph.rs index 6f84458ee..789bb1001 100644 --- a/crates/omnigraph/src/db/omnigraph.rs +++ b/crates/omnigraph/src/db/omnigraph.rs @@ -20,7 +20,7 @@ use omnigraph_compiler::{ use crate::db::commit_graph::CommitGraphSnapshot; use crate::db::graph_coordinator::{GraphCoordinator, PublishedSnapshot, ResolvedCommitRange}; -use crate::error::{CompletionEvidence, OmniError, Result, dataset_subject}; +use crate::error::{OmniError, Result, dataset_subject}; use crate::runtime_cache::RuntimeCache; use crate::seams::{decide_seam, fail}; use crate::storage::{ @@ -59,21 +59,15 @@ pub(crate) use table_ops::OpenedForMutation; pub use table_ops::{FullTextIndexRebuildResult, PendingIndex, RebuiltFullTextIndex}; use super::commit_graph::GraphCommit; -use super::manifest::{GenesisManifestAttempt, ManifestChange, TableRegistration, TableTombstone}; +use super::manifest::{ + GenesisManifestAttempt, ManifestChange, SchemaContractRow, TableRegistration, TableTombstone, +}; use super::schema_state::{ - SCHEMA_SOURCE_FILENAME, SchemaContractText, SchemaStagingPolicy, SchemaStateRecovery, - StagedContract, inspect_staged_contract, load_validated_schema_contract, - load_validated_schema_contract_for_source, read_accepted_schema_ir, read_schema_contract_text, - read_schema_contract_text_for_source, read_schema_state_identity, recover_schema_state_files, - render_schema_contract, schema_ir_uri, schema_source_staging_uri, schema_source_uri, - schema_state_uri, validate_schema_contract, validate_schema_contract_text, - validate_schema_ir_against_snapshot, write_schema_contract, + SchemaContractIdentity, render_schema_contract, snapshot_contract_identity, + validate_schema_contract_row, validate_schema_ir_against_snapshot, }; use super::snapshot::Snapshot; -use super::{ - ReadTarget, ResolvedTarget, SCHEMA_APPLY_LOCK_BRANCH, SnapshotId, is_internal_system_branch, - is_schema_apply_lock_branch, -}; +use super::{ReadTarget, ResolvedTarget, SnapshotId}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MergeOutcome { @@ -118,9 +112,9 @@ pub struct SchemaApplyPreview { /// /// Threaded as `Option<&WriteTxn>` through the mutate/load write chain /// (`open_for_mutation_on_branch`, `commit_all`, `commit_updates_on_branch_with_expected`) -/// so a single write fully validates the schema contract once at capture and once -/// under the pre-effect gates, plus one cheap capture-fence marker read — never -/// once per table. When +/// so a single write takes the contract identity of its captured manifest +/// version once and compares the live version's identity under the pre-effect +/// gates — never once per table. When /// present, the per-table resolves source the pinned `base` entry instead of calling /// `resolved_branch_target` / `snapshot_for_branch` / `fresh_snapshot_for_branch` /// (each of which re-runs `ensure_schema_state_valid`). When absent (`None` — every @@ -157,6 +151,24 @@ impl WriteAuthorityToken { } } +impl SchemaContractIdentity { + /// The authority token of one captured branch view: the branch identity + /// and head beside the contract identity of the same manifest version. + fn write_authority( + &self, + branch_identifier: lance::dataset::refs::BranchIdentifier, + graph_head: Option, + ) -> WriteAuthorityToken { + WriteAuthorityToken { + branch_identifier, + graph_head, + schema_ir_hash: self.schema_ir_hash.clone(), + schema_identity_domain: self.schema_identity_domain.clone(), + schema_identity_version: self.schema_identity_version, + } + } +} + #[derive(Debug, Clone)] pub(crate) struct WriteTxn { /// The resolved branch (`None` = main). @@ -229,18 +241,6 @@ pub struct Omnigraph { coordinator: Arc>, table_store: TableStore, runtime_cache: RuntimeCache, - /// RFC 0067: this handle's schema apply published its manifest commit but - /// could not install the schema contract. The next write entry on - /// this handle installs it from the staged copy; other handles and - /// processes converge at their next read-write open. - pending_schema_install: std::sync::atomic::AtomicBool, - /// This handle acquired the schema-apply sentinel and then failed to - /// release it (liveness contract): the next write entry retries the - /// release before the sentinel gate, so a transient release fault never - /// wedges the handle until reopen. Only ever set after OUR acquire, so - /// the retry can never delete another process's live sentinel — a - /// foreign acquire is impossible while ours still stands. - pending_sentinel_release: std::sync::atomic::AtomicBool, /// Warm change-feed cut for this handle's bound branch. A cut (head, /// witness, genesis, lineage projection, forward child index) is a PURE /// projection of `__manifest`, so it is exactly valid while the manifest @@ -268,6 +268,8 @@ pub struct Omnigraph { /// accepted IR hash is the refresh fence: unlike source bytes, it changes /// when a drop/re-add returns to the same names with new identities. schema_view: Arc>, + /// Validated legacy contract used only by an admitted v10 conversion handle. + upgrade_schema_contract: Option, /// Root-scoped writer queues shared by every `Omnigraph` handle for this /// canonical local root identity (or opaque remote URI) in the process. /// Reachable from engine internals @@ -326,18 +328,10 @@ pub struct Omnigraph { embedding_config: Option>, } -/// Whether [`Omnigraph::open`] settles a staged schema contract on open. -/// -/// Settling writes: it installs or discards the staged contract files and -/// reclaims a stale schema-apply sentinel. -/// Read-only consumers — NDJSON export, `commit list`, `read`, schema -/// inspection — should not trigger writes (they may run with read-only -/// object-store credentials, and silent open-time mutations are -/// surprising). Table data needs no open-time pass: reads always resolve -/// through the manifest pin, which is the consistent snapshot. +/// Whether open checks write capability and refuses legacy recovery sidecars. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OpenMode { - /// Settle a staged schema contract on open. Default for `Omnigraph::open`. + /// Check write capability and legacy recovery. Default for `Omnigraph::open`. ReadWrite, /// Perform no open-time writes. Use for read-only consumers via /// [`Omnigraph::open_read_only`]. @@ -345,18 +339,10 @@ pub enum OpenMode { } /// Options for [`Omnigraph::init_with_options`]. -/// -/// `force` controls the safety preflight that prevents an -/// accidental re-init from overwriting an existing graph's schema -/// metadata. Default behavior (`force: false`) fails fast with -/// [`OmniError::AlreadyInitialized`] if `__manifest` or any of `_schema.pg`, -/// `_schema.ir.json`, or `__schema_state.json` already exists at the target -/// URI. With `force: true`, orphan schema files may be replaced only when no -/// `__manifest` exists. Force never rebinds an existing graph to a newly -/// minted schema identity domain and does not purge Lance datasets. +/// Both modes refuse an existing `__manifest` and preserve orphan objects. #[derive(Debug, Clone, Copy, Default)] pub struct InitOptions { - /// Replace orphan schema artifacts at a root with no `__manifest`. + /// Request force-init admission; existing graph authority still refuses. pub force: bool, } @@ -367,8 +353,7 @@ decide_seam! { } decide_seam! { - /// After native branch control settled this handle's pending schema - /// install, before it acquires its schema -> branch -> table gates. + /// Before native branch control acquires its schema, branch and table gates. pub static BRANCH_CONTROL_PRE_GATES = ("branch_control.pre_gates", AnyWrite, [Fail]); } @@ -386,12 +371,7 @@ decide_seam! { } decide_seam! { - pub static INIT_AFTER_SCHEMA_PG_WRITTEN = ("init.after_schema_pg_written", Unreachable, [Fail]); -} - -decide_seam! { - /// Open owns the schema gate and is about to read source/IR/state as one - /// catalog view. + /// Open owns the schema gate, after reading the row and before validating it. pub static OPEN_BEFORE_SCHEMA_CONTRACT_READ = ("open.before_schema_contract_read", Unreachable, [Fail]); } @@ -479,14 +459,6 @@ impl Omnigraph { let write_queue = crate::db::write_queue::WriteQueueManager::for_root(&write_queue_identity); - // Preflight before parse or write. Strict init refuses any schema - // artifact; force may recover orphan schema files but still refuses an - // existing manifest so a newly minted identity domain can never be - // attached to old tables. - // - // Closes the "init is destructive against existing state" - // class: there is no longer a code path where strict-mode - // `init` can mutate a populated graph root. preflight_init_target(&root, storage.as_ref(), options).await?; let system_columns = if legacy_system_columns { @@ -523,95 +495,26 @@ impl Omnigraph { let schema_identity_domain = schema_ir.schema_identity_domain.as_str().to_string(); let mut catalog = build_catalog_from_ir(&schema_ir)?; fixup_physical_schemas(&mut catalog)?; - let (_, ir_json, state_json) = render_schema_contract(&schema_ir, None)?; - let contract = SchemaContractText { - source: schema_source.to_string(), - ir_json, - state_json, - }; - - // Every init write needs atomic create-if-absent (the root init claim, - // the strict-mode `_schema.pg` defence, and each Lance commit), so - // refuse an incapable local - // filesystem here, while the root holds nothing to strand. + let manifest_contract = render_schema_contract(&schema_ir, schema_source)?; verify_local_create_if_absent(&root, storage.as_ref()).await?; - - // Both strict and force init take the same root-scoped durable claim. - // Force cannot use `_schema.pg` as its claim because replacing an - // orphan is its explicit job; without this separate authority two - // force callers can overwrite one another's contracts and the loser - // can delete the winner's schema after losing the manifest Create. - // The claim remains present through any owned cleanup and is released - // last, so no cooperating process can enter that window. let init_claim = acquire_init_claim(&root, storage.as_ref()).await?; - - // The first preflight is only a fast failure. Repeat it while holding - // the durable claim so a competitor that passed its own first probe - // cannot change the target between our check and schema writes. if let Err(err) = preflight_init_target(&root, storage.as_ref(), options).await { best_effort_release_init_claim(&init_claim, storage.as_ref()).await; return Err(err); } - // Keep `_schema.pg`'s conditional create in strict mode as defence in - // depth against an unsupported older writer that does not understand - // `__init_claim.json`. Cleanup authority comes from `init_claim`, not - // from this file. - let schema_pg_claimed = if options.force { - false - } else { - let schema_path = join_uri(&root, SCHEMA_SOURCE_FILENAME); - match storage - .write_text_if_absent(&schema_path, schema_source) - .await - { - Ok(true) => {} - Ok(false) => { - best_effort_release_init_claim(&init_claim, storage.as_ref()).await; - return Err(OmniError::AlreadyInitialized { uri: root.clone() }); - } - Err(err) => { - best_effort_cleanup_owned_init_artifacts(&root, storage.as_ref(), &init_claim) - .await; - return Err(err); - } - } - if let Err(err) = fail(&INIT_AFTER_SCHEMA_PG_WRITTEN) { - best_effort_cleanup_owned_init_artifacts(&root, storage.as_ref(), &init_claim) - .await; - return Err(err); - } - true - }; - let genesis_attempt = match GenesisManifestAttempt::mint(catalog.system_columns) { Ok(attempt) => attempt, Err(err) => { - best_effort_cleanup_owned_init_artifacts(&root, storage.as_ref(), &init_claim) - .await; + best_effort_release_init_claim(&init_claim, storage.as_ref()).await; return Err(err); } }; - // Run the commit phase. Errors before any per-table or manifest - // Dataset::write may clean the schema artifacts owned by this claim. - // Once physical graph initialization begins, an error is - // acknowledgement-unknown: the exact attempt-local genesis must be - // probed, and absence of that genesis cannot prove partial table - // Creates are absent. - // - // Coverage gap: a failure DURING `GraphCoordinator::init` can leave - // Lance per-type datasets and a partial `__manifest/` behind; those - // are not cleaned up here — recursive deletion needs the - // `StorageAdapter::delete_prefix` primitive deferred with - // `DELETE /graphs/{id}` (MR-668 PR 2b). Operators may need to - // remove the graph directory manually before retrying `init`. let coordinator = match init_commit_phase( &root, - &contract, + &manifest_contract, &catalog, - &storage, - !schema_pg_claimed, &lance_access.control_session(), &genesis_attempt, ) @@ -630,12 +533,7 @@ impl Omnigraph { } } } - Err(InitCommitError::BeforePhysicalInit(source)) => { - best_effort_cleanup_owned_init_artifacts(&root, storage.as_ref(), &init_claim) - .await; - return Err(source); - } - Err(InitCommitError::PhysicalInitOutcomeUnknown(source)) => { + Err(source) => { let probe = GraphCoordinator::open_exact_genesis_with_storage( &root, &genesis_attempt, @@ -679,9 +577,11 @@ impl Omnigraph { accepted_catalog: crate::runtime_cache::AcceptedCatalogMemo::default(), compiled_queries: crate::runtime_cache::CompiledQueryCache::default(), }); - read_caches - .accepted_catalog - .memoize(contract, Arc::clone(&catalog)); + read_caches.accepted_catalog.memoize( + coordinator.snapshot(), + manifest_contract, + Arc::clone(&catalog), + ); Ok(Self { root_uri: root.clone(), storage, @@ -693,10 +593,9 @@ impl Omnigraph { // sessions reuse the process-wide object-store registry. table_store: TableStore::new(&root, session), runtime_cache: RuntimeCache::default(), - pending_schema_install: std::sync::atomic::AtomicBool::new(false), - pending_sentinel_release: std::sync::atomic::AtomicBool::new(false), feed_cut_cache: tokio::sync::RwLock::new(None), read_caches, + upgrade_schema_contract: None, schema_view: Arc::new(ArcSwap::from_pointee(HandleSchemaView { catalog, source: Arc::new(schema_source.to_string()), @@ -713,8 +612,8 @@ impl Omnigraph { /// Open an existing graph (read-write). /// - /// Reads `_schema.pg`, parses it, builds the catalog, and opens `__manifest`. - /// Settles a staged schema contract before returning — see [`OpenMode`]. + /// Opens `__manifest`, reads the `schema_contract` row of its version and + /// builds the catalog from it. See [`OpenMode`] for admission checks. pub async fn open(uri: &str) -> Result { Self::open_with_storage_and_mode(uri, storage_for_uri(uri)?, OpenMode::ReadWrite).await } @@ -725,11 +624,11 @@ impl Omnigraph { Self::open_with_storage_and_mode(uri, storage_for_uri(uri)?, OpenMode::ReadOnly).await } - /// Observe that no recovery sidecar or staged schema artifact is present. + /// Observe that no legacy recovery sidecar is present. /// Performs no graph open, recovery, cleanup, or object-body reads. Any /// pending JSON, including malformed or unsupported sidecars, refuses. /// Listing refuses beyond one matching file, 1,024 unrelated entries or - /// 128 KiB of URI bytes; three fixed schema-staging paths are also probed. + /// 128 KiB of URI bytes. /// /// This is a point-in-time observation under the process-local schema gate, /// not writer exclusion or a transferable recovery capability. Callers must @@ -776,6 +675,24 @@ impl Omnigraph { uri: &str, storage: Arc, mode: OpenMode, + ) -> Result { + Self::open_with_contract(uri, storage, mode, None).await + } + + pub(super) async fn open_for_storage_upgrade( + uri: &str, + mode: OpenMode, + contract: SchemaContractRow, + ) -> Result { + validate_schema_contract_row(&contract)?; + Self::open_with_contract(uri, storage_for_uri(uri)?, mode, Some(contract)).await + } + + async fn open_with_contract( + uri: &str, + storage: Arc, + mode: OpenMode, + upgrade_schema_contract: Option, ) -> Result { let storage = crate::storage::decorate(storage); let root = normalize_root_uri(uri)?; @@ -789,100 +706,72 @@ impl Omnigraph { // storage format this binary does not read — rebuild via export/import). // Both open modes refuse: there is no in-place migration, and the check is // a stamp read with no object-store writes, so it is safe under ReadOnly. - crate::db::manifest::read_supported_internal_schema_version(&root).await?; - // Hold the same schema gate through format preflight and contract - // capture. A v3 live or staged IR must refuse before the local write - // probe, coordinator open, or the staged-contract pass can change files. + let control_session = lance_access.control_session(); + let prepared = if upgrade_schema_contract.is_some() { + let stamp = crate::db::manifest::read_supported_internal_schema_version(&root).await?; + if stamp != 10 { + return Err(OmniError::manifest( + "upgrade-only engine handle requires admitted format 10", + )); + } + None + } else { + Some( + crate::db::manifest::ManifestCoordinator::prepare_open_with_contract( + &root, + &control_session, + ) + .await?, + ) + }; let schema_contract_guard = write_queue.acquire_schema_exclusive().await; - crate::db::schema_state::refuse_unsupported_schema_versions(&root, storage.as_ref()) - .await?; - // Read-write opens write before the first user mutation (the - // staged-contract pass, schema-stamp migration), and every write needs - // atomic create-if-absent; read-only opens perform no writes. + let (coordinator, captured_contract) = if let Some(prepared) = prepared { + let (coordinator, contract) = + GraphCoordinator::open_with_contract(&root, Arc::clone(&storage), prepared).await?; + (coordinator, Some(contract)) + } else { + let mut coordinator = + GraphCoordinator::open_with_session(&root, Arc::clone(&storage), &control_session) + .await?; + coordinator.refresh().await?; + (coordinator, None) + }; + let contract = match &upgrade_schema_contract { + Some(contract) if coordinator.snapshot().schema_contract().is_none() => { + crate::db::schema_state::refuse_unsupported_schema_versions(&contract.ir)?; + contract.clone() + } + supplied => { + let stored = match captured_contract { + Some(contract) => contract, + None => coordinator.read_schema_contract().await?, + }; + crate::db::schema_state::refuse_unsupported_schema_versions(&stored.ir)?; + validate_schema_contract_row(&stored)?; + if supplied + .as_ref() + .is_some_and(|contract| contract != &stored) + { + return Err(OmniError::manifest( + "upgrade contract differs from the source manifest row", + )); + } + stored + } + }; if matches!(mode, OpenMode::ReadWrite) { verify_local_create_if_absent(&root, storage.as_ref()).await?; } - // Open the coordinator first so the staged-contract pass can compare - // its snapshot against any leftover staging files. - let control_session = lance_access.control_session(); - let mut coordinator = - GraphCoordinator::open_with_session(&root, Arc::clone(&storage), &control_session) - .await?; - // Schema publication is a three-file promotion plus an in-memory catalog - // swap. Every handle — including ReadOnly — must hold the root-scoped - // schema gate from its final coordinator refresh through the complete - // source/IR/state read and catalog construction. Otherwise an open can - // observe a partial promotion, or publish an old catalog after a live - // apply completed. ReadOnly still performs no writes; this gate - // only serializes its read with an in-process publisher. - // Refresh under the continuously held gate before either the - // staged-contract pass or contract capture, preserving the coherent - // snapshot boundary. - coordinator.refresh().await?; - // The staged-contract pass is gated on `OpenMode::ReadWrite`. - // Read-only consumers (NDJSON export, `commit list`, schema show) - // shouldn't trigger object-store mutations: they may run with - // read-only credentials, and silent open-time writes are surprising. - // The next ReadWrite open does the work. ReadOnly still performs the - // non-mutating coherence proof below: a published SchemaApply - // manifest outcome cannot be served with the old schema contract - // merely because installation is pending. if matches!(mode, OpenMode::ReadWrite) { - // A sidecar under `__recovery/` can only come from a build that - // predates detached table commits (RFC 0067); this build cannot - // interpret one, so refuse before the staged-contract pass. crate::db::upgrade::legacy_sidecars::refuse_legacy_sidecars( &root, storage.as_ref(), "read-write open", ) .await?; - // A staged schema contract names the graph commit that publishes - // it: install it when that commit is in lineage, discard it - // otherwise. The caller holds the exclusive schema permit. - recover_schema_state_files( - &root, - Arc::clone(&storage), - &coordinator.snapshot(), - SchemaStagingPolicy::PromoteOrDiscard, - ) - .await?; - // A crashed schema apply or system-column upgrade leaves its - // durable sentinel behind. The pass above settled its staging, so - // the sentinel is stale under the same one-mutation-process - // boundary; reclaim it. - if coordinator - .all_branches() - .await? - .iter() - .any(|branch| is_schema_apply_lock_branch(branch)) - { - tracing::warn!("reclaiming the schema apply sentinel left by a crashed apply"); - coordinator.branch_delete(SCHEMA_APPLY_LOCK_BRANCH).await?; - } - } else { - // ReadOnly writes nothing, but it must not pair a manifest that - // already carries a published schema outcome with the old live - // contract; only a read-write open installs the staged one. - crate::db::schema_state::ensure_read_only_schema_coherent(&root, storage.as_ref()) - .await?; } fail(&OPEN_BEFORE_SCHEMA_CONTRACT_READ)?; - // Read _schema.pg (after the staged-contract pass — it may have just - // been installed). - // The stamp guard and coordinator open above both read `__manifest`, - // so reaching this point proves that manifest is readable; it does not - // prove every referenced data table exists. A missing schema source is - // the schema-files-gone damage state. - let schema_path = schema_source_uri(&root); - let Some(schema_source) = storage.read_text_if_exists(&schema_path).await? else { - return Err(OmniError::manifest_not_found(format!( - "graph at '{root}' is missing its schema files: '_schema.pg' was not found although '__manifest' is readable; restore the matching '_schema.pg', '_schema.ir.json', and '__schema_state.json' contract from a backup, or rebuild a fresh graph from an existing export or backup" - ))); - }; - let contract = - read_schema_contract_text_for_source(&root, storage.as_ref(), schema_source).await?; - let (accepted_ir, accepted_state) = validate_schema_contract_text(&contract)?; + let (accepted_ir, accepted_state) = validate_schema_contract_row(&contract)?; validate_schema_ir_against_snapshot(&accepted_ir, &coordinator.snapshot())?; let schema_identity_domain = accepted_ir.schema_identity_domain.as_str().to_string(); let mut catalog = build_catalog_from_ir(&accepted_ir)?; @@ -897,10 +786,13 @@ impl Omnigraph { accepted_catalog: crate::runtime_cache::AcceptedCatalogMemo::default(), compiled_queries: crate::runtime_cache::CompiledQueryCache::default(), }); - read_caches - .accepted_catalog - .memoize(contract, Arc::clone(&catalog)); + read_caches.accepted_catalog.memoize( + coordinator.snapshot(), + contract, + Arc::clone(&catalog), + ); let db = Self { + upgrade_schema_contract, root_uri: root.clone(), storage, lance_access, @@ -911,8 +803,6 @@ impl Omnigraph { // sessions reuse the process-wide object-store registry. table_store: TableStore::new(&root, session), runtime_cache: RuntimeCache::default(), - pending_schema_install: std::sync::atomic::AtomicBool::new(false), - pending_sentinel_release: std::sync::atomic::AtomicBool::new(false), feed_cut_cache: tokio::sync::RwLock::new(None), read_caches, schema_view: Arc::new(ArcSwap::from_pointee(HandleSchemaView { @@ -1072,50 +962,154 @@ impl Omnigraph { .map_err(|err| OmniError::Policy(err.to_string())) } - /// Validates on every call, so a long-lived handle sees external drift of - /// the schema source, IR or state (`lifecycle::long_lived_handle_rejects_schema_*`). - pub(crate) async fn ensure_schema_state_valid( - &self, - ) -> Result { - validate_schema_contract(self.uri(), Arc::clone(&self.storage)).await + /// Validate the contract of the manifest version this handle's coordinator + /// holds: a memo hit is a contract validated when it was built; a miss reads + /// the row once. + pub(crate) async fn ensure_schema_state_valid(&self) -> Result<()> { + let snapshot = self.coordinator.read().await.snapshot(); + self.accepted_catalog_for_snapshot(&snapshot) + .await + .map(|_| ()) } - /// Load one operation-local catalog from the accepted schema contract while - /// the caller holds the root schema gate. + /// Load one operation-local catalog for the manifest version this handle's + /// coordinator holds while the caller holds the root schema gate. /// - /// Long-lived handles intentionally keep a warm ArcSwap catalog, so merely - /// validating the files does not make `self.catalog()` current after another - /// handle applies a schema. Control/legacy-adapter bridges use this capture - /// for planning and conservative table-gate enumeration. The caller MUST - /// already hold a schema permit (either side); this helper does not acquire - /// one because the gate is non-reentrant on one task. + /// Long-lived handles intentionally keep a warm ArcSwap catalog, so this + /// capture, not `self.catalog()`, is what maintenance plans against. The + /// caller MUST already hold a schema permit (either side); this helper does + /// not acquire one because the gate is non-reentrant on one task. pub(crate) async fn load_accepted_catalog_with_schema_gate_held(&self) -> Result> { - let catalog = self.build_accepted_catalog_with_schema_gate_held().await?; - let snapshot = self.coordinator.read().await.snapshot(); + let current_branch = self + .coordinator + .read() + .await + .current_branch() + .unwrap_or("main") + .to_string(); + let snapshot = self + .resolve_target_inner(&ReadTarget::branch(current_branch)) + .await? + .snapshot; + let (catalog, _) = self.accepted_catalog_for_snapshot(&snapshot).await?; validate_bound_catalog_against_snapshot(&catalog, &snapshot)?; Ok(catalog) } - /// Build the accepted operation-local catalog without joining it to this - /// handle's warm manifest snapshot. Coherent read capture uses this form so - /// it can run the manifest freshness probe first, then validate the catalog - /// against the exact resolved snapshot. Other callers should use - /// [`Self::load_accepted_catalog_with_schema_gate_held`] unless they perform - /// that post-resolution identity join themselves. - async fn build_accepted_catalog_with_schema_gate_held(&self) -> Result> { - let text = read_schema_contract_text(self.uri(), self.storage.as_ref()).await?; - if let Some(catalog) = self.read_caches.accepted_catalog.get(&text) { - return Ok(catalog); + /// Accept the contract bytes in this snapshot before reusing its catalog. + pub(crate) async fn accepted_catalog_for_snapshot( + &self, + snapshot: &Snapshot, + ) -> Result<(Arc, SchemaContractIdentity)> { + let accepted = self.accepted_schema_for_snapshot(snapshot).await?; + Ok((Arc::clone(&accepted.catalog), accepted.identity.clone())) + } + + async fn accepted_schema_for_snapshot( + &self, + snapshot: &Snapshot, + ) -> Result> { + let identity = match (&self.upgrade_schema_contract, snapshot.schema_contract()) { + (Some(contract), None) => SchemaContractIdentity::from(&contract.head), + _ => snapshot_contract_identity(snapshot)?, + }; + let previous = self.read_caches.accepted_catalog.current(); + if let Some(entry) = &previous + && entry.identity == identity + && entry.snapshot.same_manifest_image(snapshot) + { + return Ok(Arc::clone(entry)); } - let (schema_ir, _) = validate_schema_contract_text(&text)?; - let mut catalog = build_catalog_from_ir(&schema_ir)?; - fixup_physical_schemas(&mut catalog)?; - crate::instrumentation::record_catalog_build(); - let catalog = Arc::new(catalog); - self.read_caches + let row = self.read_schema_contract_row_for(snapshot).await?; + if self + .upgrade_schema_contract + .as_ref() + .is_some_and(|contract| contract != &row) + { + return Err(OmniError::manifest( + "upgrade contract differs from the source manifest row", + )); + } + let catalog = if let Some(entry) = &previous + && entry.identity == identity + && entry.row == row + { + Arc::clone(&entry.catalog) + } else { + let (schema_ir, loaded_identity) = validate_schema_contract_row(&row)?; + if loaded_identity != identity { + return Err(OmniError::manifest( + "loaded schema contract differs from captured snapshot identity", + )); + } + if let Some(entry) = &previous + && entry.identity == identity + { + Arc::clone(&entry.catalog) + } else { + let mut catalog = build_catalog_from_ir(&schema_ir)?; + fixup_physical_schemas(&mut catalog)?; + crate::instrumentation::record_catalog_build(); + Arc::new(catalog) + } + }; + Ok(self + .read_caches .accepted_catalog - .memoize(text, Arc::clone(&catalog)); - Ok(catalog) + .memoize(snapshot.clone(), row, catalog)) + } + + /// Read the schema contract captured with this snapshot, using the bound + /// coordinator only when it holds the same manifest image. + /// A capture without retained content reads its pinned dataset. + async fn read_schema_contract_row_for(&self, snapshot: &Snapshot) -> Result { + if snapshot.schema_contract().is_none() + && let Some(contract) = &self.upgrade_schema_contract + { + return Ok(contract.clone()); + } + { + let coord = self.coordinator.read().await; + if coord.snapshot().same_manifest_image(snapshot) { + return coord.read_schema_contract().await; + } + } + snapshot.read_schema_contract(self.uri()).await + } + + /// Join a native branch control's catalog to its post-gate capture; a + /// capture under another contract (an apply landed since the warm head) + /// retakes the gates under that contract and must still probe current. + async fn join_control_catalog_to_capture( + &self, + control_catalog: Arc, + identity: &SchemaContractIdentity, + branches: &[Option], + table_guards: Vec, + captured: &GraphCoordinator, + operation: &str, + ) -> Result<(Arc, Vec)> { + let captured_snapshot = captured.snapshot(); + let (captured_catalog, captured_identity) = self + .accepted_catalog_for_snapshot(&captured_snapshot) + .await?; + validate_bound_catalog_against_snapshot(&captured_catalog, &captured_snapshot)?; + if captured_identity == *identity { + return Ok((control_catalog, table_guards)); + } + drop(table_guards); + let control_catalog = captured_catalog; + let table_queue_keys = self.table_queue_keys_for_branches(branches, &control_catalog); + let table_guards = self.write_queue().acquire_many(&table_queue_keys).await; + let held = captured.manifest_incarnation(); + if !captured.probe_latest_incarnation().await?.matches(&held) { + return Err(OmniError::manifest_read_set_changed( + format!("schema_contract:{operation}"), + None, + None, + )); + } + Ok((control_catalog, table_guards)) } /// The per-graph read caches (`ReadCaches`): table handles, the accepted @@ -1216,18 +1210,6 @@ impl Omnigraph { .await } - pub(crate) async fn ensure_schema_apply_idle(&self, operation: &str) -> Result<()> { - schema_apply::ensure_schema_apply_idle(self, operation).await - } - - pub(crate) async fn ensure_schema_apply_not_locked(&self, operation: &str) -> Result<()> { - schema_apply::ensure_schema_apply_not_locked(self, operation).await - } - - pub(crate) async fn schema_apply_sentinel_present(&self) -> Result { - schema_apply::schema_apply_sentinel_present(self).await - } - /// Engine-facing trait surface around `TableStore`. /// /// This is the **only** accessor for engine code reaching into the @@ -1345,16 +1327,18 @@ impl Omnigraph { .capture_for_branch_control()) } - /// Open a capture-once write transaction (RFC-013 step 3b): validate the schema - /// contract ONCE and pin the base snapshot. The per-table opens take - /// `Option<&WriteTxn>` and, on the bound branch for the non-strict (Insert/Merge) - /// path, source the pinned base entry — instead of re-resolving (re-validating the - /// schema) per table. Strict ops, the fork path, and the commit-time revalidation - /// keep their own reads: `revalidate_write_txn` probes the manifest and reopens the - /// branch only on a mismatch (correctness machinery — see the handoff doc). + /// Open a capture-once write transaction (RFC-013 step 3b): pin the base + /// snapshot and take the contract of that one manifest version (its + /// `schema_contract` identity; the catalog from the memo or one row read). + /// The per-table opens take `Option<&WriteTxn>` and, on the bound branch + /// for the non-strict (Insert/Merge) path, source the pinned base entry — + /// instead of re-resolving per table. Strict ops, the fork path, and the + /// commit-time revalidation keep their own reads: `revalidate_write_txn` + /// probes the manifest and reopens the branch only on a mismatch + /// (correctness machinery — see the handoff doc). /// - /// "Once" covers the table-touch hot path captured here (the cost gate permits - /// one marker read plus one validation at pre-effect revalidation); it does + /// "Once" covers the table-touch hot path captured here (the cost gate + /// permits no contract file read at all); it does /// NOT yet /// cover edge endpoint /// / cardinality RI validation (`ensure_node_id_exists`, the loader's RI/cardinality), @@ -1365,80 +1349,22 @@ impl Omnigraph { /// per-graph `Session` (the dataset-opener unification); the S3 cost gate /// for that term is still owed (handoff §1d). pub(crate) async fn open_write_txn(&self, branch: Option<&str>) -> Result { - const MAX_CAPTURE_RETRIES: usize = 8; let branch = normalize_branch_name(branch.unwrap_or("main"))?; - let mut captures = 0; - loop { - // A standing sentinel under a busy gate is an apply in this - // process: park on the shared side, then recapture under the - // promoted contract. Under a free gate it is another process's - // apply, or a dead one: the typed refusal, once a second listing - // confirms it (RFC 2026-09-18-shared-schema-gate). - if self.schema_apply_sentinel_present().await? { - match self.write_queue().try_acquire_schema_shared() { - Some(free) => { - drop(free); - self.ensure_schema_apply_not_locked("write preparation") - .await?; - } - None => drop(self.write_queue().acquire_schema_shared().await), - } - tokio::task::yield_now().await; - continue; - } - captures += 1; - if captures > MAX_CAPTURE_RETRIES { - break; - } - // A schema apply publishes graph_head before promoting its staged - // contract. Read one fully validated IR/catalog, capture coherent - // manifest authority, then re-read the durable schema marker (the - // last promoted schema file) after a sentinel check. This accepts - // only (old head, old schema) or (new head, new schema), never the - // intermediate (new head, old schema) state, without paying for a - // second full schema parse during capture. - let (schema_ir, schema_state) = - load_validated_schema_contract(self.uri(), Arc::clone(&self.storage)).await?; - let (branch_identifier, graph_head, effective_graph_head, snapshot, manifest_probe) = - self.write_authority_for_known_branch(branch.as_deref(), true) - .await?; - self.ensure_schema_apply_not_locked("write preparation") - .await?; - let trailing_schema_state = - read_schema_state_identity(self.uri(), self.storage.as_ref()).await?; - - if schema_state != trailing_schema_state { - tokio::task::yield_now().await; - continue; - } - validate_schema_ir_against_snapshot(&schema_ir, &snapshot)?; - - let mut catalog = build_catalog_from_ir(&schema_ir)?; - fixup_physical_schemas(&mut catalog)?; - let schema_identity_domain = schema_ir.schema_identity_domain.as_str().to_string(); - return Ok(WriteTxn { - branch, - base: snapshot, - authority: WriteAuthorityToken { - branch_identifier, - graph_head, - schema_ir_hash: schema_state.schema_ir_hash, - schema_identity_domain, - schema_identity_version: schema_state.schema_identity_version, - }, - effective_graph_head, - caller_expected_graph_head: None, - catalog: Arc::new(catalog), - manifest_probe, - }); - } - - Err(OmniError::manifest_read_set_changed( - format!("write_authority:{}", branch.as_deref().unwrap_or("main")), - None, - None, - )) + let (branch_identifier, graph_head, effective_graph_head, snapshot, manifest_probe) = self + .write_authority_for_known_branch(branch.as_deref(), true) + .await?; + let (catalog, identity) = self.accepted_catalog_for_snapshot(&snapshot).await?; + validate_bound_catalog_against_snapshot(&catalog, &snapshot)?; + Ok(WriteTxn { + branch, + base: snapshot, + authority: identity.write_authority(branch_identifier, graph_head), + effective_graph_head, + caller_expected_graph_head: None, + catalog, + manifest_probe, + }) } /// Capture the source and target inputs for one branch merge under one @@ -1454,109 +1380,86 @@ impl Omnigraph { source_branch: Option<&str>, target_branch: Option<&str>, ) -> Result<(WriteTxn, WriteTxn, CommitGraphSnapshot, CommitGraphSnapshot)> { - const MAX_CAPTURE_RETRIES: usize = 8; let source_branch = normalize_branch_name(source_branch.unwrap_or("main"))?; let target_branch = normalize_branch_name(target_branch.unwrap_or("main"))?; + let source_authority = self + .merge_authority_for_known_branch(source_branch.as_deref()) + .await?; + let target_authority = self + .merge_authority_for_known_branch(target_branch.as_deref()) + .await?; - for _ in 0..MAX_CAPTURE_RETRIES { - self.ensure_schema_apply_not_locked("branch merge preparation") - .await?; - let (schema_ir, schema_state) = - load_validated_schema_contract(self.uri(), Arc::clone(&self.storage)).await?; - let source_authority = self - .merge_authority_for_known_branch(source_branch.as_deref()) - .await?; - let target_authority = self - .merge_authority_for_known_branch(target_branch.as_deref()) - .await?; - self.ensure_schema_apply_not_locked("branch merge preparation") - .await?; - let trailing_schema_state = - read_schema_state_identity(self.uri(), self.storage.as_ref()).await?; - if schema_state != trailing_schema_state { - tokio::task::yield_now().await; - continue; - } - - validate_schema_ir_against_snapshot(&schema_ir, &source_authority.3)?; - validate_schema_ir_against_snapshot(&schema_ir, &target_authority.3)?; - let mut catalog = build_catalog_from_ir(&schema_ir)?; - fixup_physical_schemas(&mut catalog)?; - let catalog = Arc::new(catalog); - let schema_identity_domain = schema_ir.schema_identity_domain.as_str().to_string(); - let ( - source_branch_identifier, - source_graph_head, - source_effective_graph_head, - source_base, - source_commits, - source_manifest_probe, - ) = source_authority; - let ( - target_branch_identifier, - target_graph_head, - target_effective_graph_head, - target_base, - target_commits, - target_manifest_probe, - ) = target_authority; - let make_txn = - |branch: Option, - (branch_identifier, graph_head, effective_graph_head, base, manifest_probe): ( - lance::dataset::refs::BranchIdentifier, - Option, - Option, - Snapshot, - crate::db::manifest::CapturedManifestProbe, - )| WriteTxn { - branch, - base, - authority: WriteAuthorityToken { - branch_identifier, - graph_head, - schema_ir_hash: schema_state.schema_ir_hash.clone(), - schema_identity_domain: schema_identity_domain.clone(), - schema_identity_version: schema_state.schema_identity_version, - }, - effective_graph_head, - caller_expected_graph_head: None, - catalog: Arc::clone(&catalog), - manifest_probe, - }; - return Ok(( - make_txn( - source_branch.clone(), - ( - source_branch_identifier, - source_graph_head, - source_effective_graph_head, - source_base, - source_manifest_probe, - ), - ), - make_txn( - target_branch.clone(), - ( - target_branch_identifier, - target_graph_head, - target_effective_graph_head, - target_base, - target_manifest_probe, - ), - ), - source_commits, - target_commits, + let (catalog, identity) = self + .accepted_catalog_for_snapshot(&source_authority.3) + .await?; + let (_, target_identity) = self + .accepted_catalog_for_snapshot(&target_authority.3) + .await?; + if target_identity != identity { + return Err(OmniError::manifest_read_set_changed( + "schema_ir_hash".to_string(), + Some(identity.schema_ir_hash), + Some(target_identity.schema_ir_hash), )); } - - Err(OmniError::manifest_read_set_changed( - format!( - "branch_merge_authority:{}->{}", - source_branch.as_deref().unwrap_or("main"), - target_branch.as_deref().unwrap_or("main") + validate_bound_catalog_against_snapshot(&catalog, &source_authority.3)?; + validate_bound_catalog_against_snapshot(&catalog, &target_authority.3)?; + let ( + source_branch_identifier, + source_graph_head, + source_effective_graph_head, + source_base, + source_commits, + source_manifest_probe, + ) = source_authority; + let ( + target_branch_identifier, + target_graph_head, + target_effective_graph_head, + target_base, + target_commits, + target_manifest_probe, + ) = target_authority; + let make_txn = + |branch: Option, + (branch_identifier, graph_head, effective_graph_head, base, manifest_probe): ( + lance::dataset::refs::BranchIdentifier, + Option, + Option, + Snapshot, + crate::db::manifest::CapturedManifestProbe, + )| WriteTxn { + branch, + base, + authority: identity.write_authority(branch_identifier, graph_head), + effective_graph_head, + caller_expected_graph_head: None, + catalog: Arc::clone(&catalog), + manifest_probe, + }; + Ok(( + make_txn( + source_branch.clone(), + ( + source_branch_identifier, + source_graph_head, + source_effective_graph_head, + source_base, + source_manifest_probe, + ), ), - None, - None, + make_txn( + target_branch.clone(), + ( + target_branch_identifier, + target_graph_head, + target_effective_graph_head, + target_base, + target_manifest_probe, + ), + ), + source_commits, + target_commits, )) } @@ -1574,78 +1477,46 @@ impl Omnigraph { source_txn: &WriteTxn, target_txn: &WriteTxn, ) -> Result<(WriteAuthorityToken, Snapshot, WriteAuthorityToken, Snapshot)> { - const MAX_CAPTURE_RETRIES: usize = 8; let source_branch = source_txn.branch.as_deref(); let target_branch = target_txn.branch.as_deref(); - - for _ in 0..MAX_CAPTURE_RETRIES { - self.ensure_schema_apply_not_locked("branch merge revalidation") + let source_current = source_txn.manifest_probe.is_current().await?; + let target_current = target_txn.manifest_probe.is_current().await?; + let source = if source_current { + ( + source_txn.authority.branch_identifier.clone(), + source_txn.authority.graph_head.clone(), + source_txn.base.clone(), + ) + } else { + let (branch_identifier, graph_head, _, snapshot, _) = self + .write_authority_for_known_branch(source_branch, true) .await?; - let (schema_ir, schema_state) = - load_validated_schema_contract(self.uri(), Arc::clone(&self.storage)).await?; - let source_current = source_txn.manifest_probe.is_current().await?; - let target_current = target_txn.manifest_probe.is_current().await?; - let source = if source_current { - ( - source_txn.authority.branch_identifier.clone(), - source_txn.authority.graph_head.clone(), - source_txn.base.clone(), - ) - } else { - let (branch_identifier, graph_head, _, snapshot, _) = self - .write_authority_for_known_branch(source_branch, true) - .await?; - (branch_identifier, graph_head, snapshot) - }; - let target = if target_current { - ( - target_txn.authority.branch_identifier.clone(), - target_txn.authority.graph_head.clone(), - target_txn.base.clone(), - ) - } else { - let (branch_identifier, graph_head, _, snapshot, _) = self - .write_authority_for_known_branch(target_branch, true) - .await?; - (branch_identifier, graph_head, snapshot) - }; - self.ensure_schema_apply_not_locked("branch merge revalidation") + (branch_identifier, graph_head, snapshot) + }; + let target = if target_current { + ( + target_txn.authority.branch_identifier.clone(), + target_txn.authority.graph_head.clone(), + target_txn.base.clone(), + ) + } else { + let (branch_identifier, graph_head, _, snapshot, _) = self + .write_authority_for_known_branch(target_branch, true) .await?; - let trailing_schema_state = - read_schema_state_identity(self.uri(), self.storage.as_ref()).await?; - if schema_state != trailing_schema_state { - tokio::task::yield_now().await; - continue; - } - - validate_schema_ir_against_snapshot(&schema_ir, &source.2)?; - validate_schema_ir_against_snapshot(&schema_ir, &target.2)?; - let schema_identity_domain = schema_ir.schema_identity_domain.as_str().to_string(); - let make_token = - |branch_identifier: lance::dataset::refs::BranchIdentifier, - graph_head: Option| WriteAuthorityToken { - branch_identifier, - graph_head, - schema_ir_hash: schema_state.schema_ir_hash.clone(), - schema_identity_domain: schema_identity_domain.clone(), - schema_identity_version: schema_state.schema_identity_version, - }; - return Ok(( - make_token(source.0, source.1), - source.2, - make_token(target.0, target.1), - target.2, - )); - } + (branch_identifier, graph_head, snapshot) + }; - Err(OmniError::manifest_read_set_changed( - format!( - "branch_merge_revalidation:{}->{}", - source_branch.unwrap_or("main"), - target_branch.unwrap_or("main") - ), - None, - None, + let (source_catalog, source_identity) = + self.accepted_catalog_for_snapshot(&source.2).await?; + let (target_catalog, target_identity) = + self.accepted_catalog_for_snapshot(&target.2).await?; + validate_bound_catalog_against_snapshot(&source_catalog, &source.2)?; + validate_bound_catalog_against_snapshot(&target_catalog, &target.2)?; + Ok(( + source_identity.write_authority(source.0, source.1), + source.2, + target_identity.write_authority(target.0, target.1), + target.2, )) } @@ -1653,10 +1524,11 @@ impl Omnigraph { &self, branch: Option<&str>, ) -> Result { - let (schema_ir, _) = - load_validated_schema_contract(self.uri(), Arc::clone(&self.storage)).await?; let resolved = self.resolved_branch_target_unchecked(branch).await?; - validate_schema_ir_against_snapshot(&schema_ir, &resolved.snapshot)?; + let (catalog, _) = self + .accepted_catalog_for_snapshot(&resolved.snapshot) + .await?; + validate_bound_catalog_against_snapshot(&catalog, &resolved.snapshot)?; Ok(resolved) } @@ -1704,9 +1576,10 @@ impl Omnigraph { Snapshot, crate::db::manifest::CapturedManifestProbe, )> { - { + let bound = { let coord = self.coordinator.read().await; - if branch == coord.current_branch() { + let bound = branch == coord.current_branch(); + if bound { let current = if fresh { let held = coord.manifest_incarnation(); coord.probe_latest_incarnation().await?.matches(&held) @@ -1723,6 +1596,24 @@ impl Omnigraph { )); } } + bound + }; + if !bound { + return Box::pin(async { + let cache = self.validated_cached_coordinator(branch).await?; + let coord = &cache + .as_ref() + .expect("validated authority cache entry is present") + .1; + Ok(( + coord.branch_identifier().await?, + coord.exact_graph_head(), + coord.effective_graph_head().await?, + coord.snapshot(), + coord.captured_manifest_probe(), + )) + }) + .await; } let coord = self.open_coordinator_for_branch(branch).await?; @@ -1846,13 +1737,44 @@ impl Omnigraph { Ok(cache) } + async fn cached_read_target( + &self, + branch: Option<&str>, + target: &ReadTarget, + ) -> Result> { + let cache = self.merge_authority_cache.lock().await; + let Some((_, coordinator)) = cache.as_ref().filter(|(key, coordinator)| { + key == branch.unwrap_or("main") && coordinator.current_branch() == branch + }) else { + return Ok(None); + }; + let held = coordinator.manifest_incarnation(); + if !coordinator.probe_latest_incarnation().await?.matches(&held) { + return Ok(None); + } + let graph_commit_id = coordinator.effective_graph_head().await?; + let snapshot_id = graph_commit_id + .as_deref() + .map(SnapshotId::new) + .unwrap_or_else(|| { + SnapshotId::synthetic( + coordinator.current_branch(), + coordinator.version(), + held.e_tag.as_deref(), + ) + }); + Ok(Some(ResolvedTarget { + requested: target.clone(), + branch: coordinator.current_branch().map(str::to_string), + snapshot_id, + graph_commit_id, + snapshot: coordinator.snapshot(), + })) + } + /// Revalidate a prepared mutation/load attempt after its branch/table /// gates are held and before any table effect. pub(crate) async fn revalidate_write_txn(&self, txn: &WriteTxn) -> Result { - // `commit_all` calls this while holding schema → branch → table gates. - // Recheck the durable sentinel inside that critical section so a schema - // apply observed after preparation cannot be followed by a table effect. - self.ensure_schema_apply_not_locked("write commit").await?; let bound = txn.branch.as_deref() == self.coordinator.read().await.current_branch(); let (branch_identifier, graph_head, effective_graph_head, snapshot) = if !bound && txn.manifest_probe.is_current().await? { @@ -1873,10 +1795,7 @@ impl Omnigraph { snapshot, ) }; - let (schema_ir, schema_state) = - load_validated_schema_contract(self.uri(), Arc::clone(&self.storage)).await?; - self.ensure_schema_apply_not_locked("write commit").await?; - validate_schema_ir_against_snapshot(&schema_ir, &snapshot)?; + let (_, live) = self.accepted_catalog_for_snapshot(&snapshot).await?; if let Some(expected) = txn.caller_expected_graph_head.as_deref() && effective_graph_head.as_deref() != Some(expected) { @@ -1911,28 +1830,28 @@ impl Omnigraph { graph_head, )); } - if schema_state.schema_ir_hash != txn.authority.schema_ir_hash { + if live.schema_ir_hash != txn.authority.schema_ir_hash { return Err(OmniError::manifest_read_set_changed( "schema_ir_hash".to_string(), Some(txn.authority.schema_ir_hash.clone()), - Some(schema_state.schema_ir_hash), + Some(live.schema_ir_hash), )); } - let schema_identity_domain = schema_ir.schema_identity_domain.as_str(); - if schema_identity_domain != txn.authority.schema_identity_domain { + if live.schema_identity_domain != txn.authority.schema_identity_domain { return Err(OmniError::manifest_read_set_changed( "schema_identity_domain".to_string(), Some(txn.authority.schema_identity_domain.clone()), - Some(schema_identity_domain.to_string()), + Some(live.schema_identity_domain), )); } - if schema_state.schema_identity_version != txn.authority.schema_identity_version { + if live.schema_identity_version != txn.authority.schema_identity_version { return Err(OmniError::manifest_read_set_changed( "schema_identity_version".to_string(), Some(txn.authority.schema_identity_version.to_string()), - Some(schema_state.schema_identity_version.to_string()), + Some(live.schema_identity_version.to_string()), )); } + validate_bound_catalog_against_snapshot(&txn.catalog, &snapshot)?; Ok(snapshot) } @@ -1943,10 +1862,9 @@ impl Omnigraph { } pub(crate) async fn fresh_snapshot_for_branch(&self, branch: Option<&str>) -> Result { - let (schema_ir, _) = - load_validated_schema_contract(self.uri(), Arc::clone(&self.storage)).await?; let snapshot = self.fresh_snapshot_for_branch_unchecked(branch).await?; - validate_schema_ir_against_snapshot(&schema_ir, &snapshot)?; + let (catalog, _) = self.accepted_catalog_for_snapshot(&snapshot).await?; + validate_bound_catalog_against_snapshot(&catalog, &snapshot)?; Ok(snapshot) } @@ -2034,13 +1952,14 @@ impl Omnigraph { // captures the schema contract and target coordinator coherently across // a concurrent schema apply. Lock order remains schema -> coordinator. let _schema_permit = self.write_queue().acquire_schema_exclusive().await; - let (schema_ir, _) = - load_validated_schema_contract(self.uri(), Arc::clone(&self.storage)).await?; let branch = normalize_branch_name(branch)?; let next = self.open_coordinator_for_branch(branch.as_deref()).await?; - validate_schema_ir_against_snapshot(&schema_ir, &next.snapshot())?; + let next_snapshot = next.snapshot(); + let (catalog, _) = self.accepted_catalog_for_snapshot(&next_snapshot).await?; + validate_bound_catalog_against_snapshot(&catalog, &next_snapshot)?; *self.coordinator.write().await = next; self.invalidate_read_caches().await; + self.reload_schema_view_from_coordinator().await?; Ok(()) } @@ -2055,204 +1974,49 @@ impl Omnigraph { *self.feed_cut_cache.write().await = None; } - /// Re-read the handle-local coordinator state from storage and install a - /// published schema contract another writer left staged, without restart. - /// - /// 1. `coordinator.refresh()` re-reads the manifest. - /// 2. `recover_schema_state_files` in promote-only mode installs a staged - /// contract whose publishing commit is already in lineage (RFC 0067) - /// and leaves anything else alone, since the apply that staged it may - /// still be live. A table pin needs no pass here: reads resolve it - /// through its staged version. - /// 3. The schema view reloads if the source changed and the read caches - /// drop. - /// - /// Steady-state cost: the staged-contract probe; no `__recovery/` listing - /// and no additional Lance reads. Engine-internal callers that hold the - /// schema gate MUST use - /// [`refresh_coordinator_only`](Self::refresh_coordinator_only). + /// Refresh the coordinator and its schema view from the published contract. pub async fn refresh(&self) -> Result<()> { - { - // Queue before coordinator: the documented lock order. The schema - // gate keeps a live apply's staging writes and this pass apart. - // Scope the coordinator write guard to this block: - // `reload_schema_if_source_changed` takes the coordinator read - // lock, and Tokio's RwLock is not reentrant. Pinned by - // `composite_flow_schema_apply_then_branch_ops_no_deadlock_in_refresh`. - let _serial = self.write_queue.acquire_schema_exclusive().await; - let mut coord = self.coordinator.write().await; - coord.refresh().await?; - let outcome = recover_schema_state_files( - &self.root_uri, - Arc::clone(&self.storage), - &coord.snapshot(), - SchemaStagingPolicy::PromoteOnly, - ) - .await?; - // An installed staging completes a published apply whose writer - // died before releasing its sentinel; release it here so the - // caller's write is not refused until the next open. - if matches!(outcome, SchemaStateRecovery::Promoted) - && coord - .all_branches() - .await? - .iter() - .any(|branch| is_schema_apply_lock_branch(branch)) - { - coord.branch_delete(SCHEMA_APPLY_LOCK_BRANCH).await?; - } - } - self.reload_schema_if_source_changed().await?; + let _serial = self.write_queue.acquire_schema_exclusive().await; + self.coordinator.write().await.refresh().await?; + self.reload_schema_view_from_coordinator().await?; self.invalidate_read_caches().await; Ok(()) } - /// Record that this handle acquired the schema-apply sentinel and could - /// not release it. The next write entry retries the release (liveness - /// contract: a live handle writes again once faults stop, without - /// reopening). - pub(crate) fn note_failed_sentinel_release(&self) { - self.pending_sentinel_release - .store(true, std::sync::atomic::Ordering::SeqCst); - } - - /// Finish this handle's own published-but-uninstalled schema contract - /// before a write plans against the manifest (RFC 0067). A schema apply or - /// system-column upgrade whose manifest commit landed but whose contract - /// installation failed sets `pending_schema_install`; every write entry - /// calls this, and the flags keep the common path free of any storage - /// probe. Other handles and processes converge at their next read-write - /// open or `refresh`. - /// - /// The same entry also retries a sentinel release this handle failed - /// (`note_failed_sentinel_release`), before the sentinel gate every write - /// takes, so a transient release fault never wedges the handle. - /// Returns whether completion attempted durable work. A caller must not - /// certify a later failure as pre-effect after such an attempt. Successful - /// completion alone does not make a later validation refusal uncertain. - pub(crate) async fn settle_pending_schema_install(&self) -> Result { - let released_sentinel = self - .pending_sentinel_release - .swap(false, std::sync::atomic::Ordering::SeqCst); - if released_sentinel && let Err(error) = schema_apply::release_schema_apply_lock(self).await - { - // Restore the flag so the retry is not lost, and fail loud: the - // sentinel this handle owns still stands, so the write would be - // refused at the gate anyway — with a less actionable message. - self.pending_sentinel_release - .store(true, std::sync::atomic::Ordering::SeqCst); - return Err(error.with_completion_evidence(CompletionEvidence::Uncertain)); - } - if !self - .pending_schema_install - .swap(false, std::sync::atomic::Ordering::SeqCst) - { - return Ok(released_sentinel); - } - let result = { - let _serial = self.write_queue.acquire_schema_exclusive().await; - let snapshot = self.coordinator.read().await.snapshot(); - recover_schema_state_files( - &self.root_uri, - Arc::clone(&self.storage), - &snapshot, - SchemaStagingPolicy::PromoteOnly, - ) - .await - }; - match result { - Ok(recovery) => { - if matches!(recovery, SchemaStateRecovery::Promoted) { - self.reload_schema_if_source_changed() - .await - .map_err(|error| { - error.with_completion_evidence(CompletionEvidence::Uncertain) - })?; - self.invalidate_read_caches().await; - } - Ok(true) - } - Err(error) => { - self.pending_schema_install - .store(true, std::sync::atomic::Ordering::SeqCst); - Err(error.with_completion_evidence(CompletionEvidence::Uncertain)) - } - } - } - - async fn reload_schema_if_source_changed(&self) -> Result<()> { - // Callers release their schema gate before this call. Reacquire it - // across the complete source/IR/state read and ArcSwap publication so a - // concurrent apply cannot interleave its sequential file promotions - // with this reload. - let _schema_permit = self.write_queue.acquire_schema_exclusive().await; + /// Refresh the schema view while the exclusive schema permit is held. + async fn reload_schema_view_from_coordinator(&self) -> Result<()> { fail(&SCHEMA_RELOAD_BEFORE_CONTRACT_READ)?; - let schema_path = schema_source_uri(&self.root_uri); - let schema_source = self.storage.read_text(&schema_path).await?; - let (accepted_ir, accepted_state) = load_validated_schema_contract_for_source( - &self.root_uri, - Arc::clone(&self.storage), - &schema_source, - ) - .await?; let live_snapshot = self.coordinator.read().await.snapshot(); - validate_schema_ir_against_snapshot(&accepted_ir, &live_snapshot)?; - let accepted_domain = accepted_ir.schema_identity_domain.as_str().to_string(); + let accepted = self.accepted_schema_for_snapshot(&live_snapshot).await?; + validate_bound_catalog_against_snapshot(&accepted.catalog, &live_snapshot)?; + let identity = &accepted.identity; let current = self.schema_view.load_full(); - if accepted_state.schema_ir_hash == current.schema_ir_hash - && accepted_domain == current.schema_identity_domain - && schema_source == *current.source + if identity.schema_ir_hash == current.schema_ir_hash + && identity.schema_identity_domain == current.schema_identity_domain + && identity.schema_identity_version == super::schema_state::SCHEMA_IDENTITY_VERSION + && accepted.row.source == *current.source { return Ok(()); } - let catalog = if accepted_state.schema_ir_hash == current.schema_ir_hash - && accepted_domain == current.schema_identity_domain - { - (*current.catalog).clone() - } else { - let mut catalog = build_catalog_from_ir(&accepted_ir)?; - fixup_physical_schemas(&mut catalog)?; - catalog - }; drop(current); - self.store_schema_view(catalog, schema_source, &accepted_ir)?; + let accepted_ir = accepted.catalog.bound_schema_ir().ok_or_else(|| { + OmniError::manifest_internal("accepted catalog carries no bound SchemaIR") + })?; + self.store_schema_view( + (*accepted.catalog).clone(), + accepted.row.source.clone(), + accepted_ir, + )?; Ok(()) } - /// Refresh coordinator state and invalidate the runtime cache WITHOUT - /// the staged-contract pass. Engine-internal callers that already hold - /// the schema gate (e.g. `schema_apply::apply_schema_with_lock`'s - /// internal lease-check refresh) need this variant: - /// [`refresh`](Self::refresh) reacquires that gate and would act on the - /// caller's own staged contract. + /// Refresh coordinator state while the caller owns the applicable schema permit. pub(crate) async fn refresh_coordinator_only(&self) -> Result<()> { self.coordinator.write().await.refresh().await?; self.invalidate_read_caches().await; Ok(()) } - /// The reprepare refresh: a write whose authority moved recaptures from - /// the coordinator alone, and takes the contract-lifecycle pass of - /// [`refresh`](Self::refresh) only when a published staging is waiting - /// to be installed. `open_write_txn` reads the accepted contract from - /// the store on every capture, so the schema view and the `PromoteOnly` - /// pass add nothing to an ordinary reprepare; taking their exclusive - /// permit there made every same-branch reprepare a process-wide barrier - /// (RFC 2026-09-18-shared-schema-gate, 2026-09-29 entry). - pub(crate) async fn refresh_for_reprepare(&self) -> Result<()> { - self.refresh_coordinator_only().await?; - if matches!( - inspect_staged_contract(&self.root_uri, self.storage.as_ref(), false).await?, - StagedContract::Marked { - published: true, - .. - } - ) { - self.refresh().await?; - } - Ok(()) - } - pub async fn resolve_snapshot(&self, branch: &str) -> Result { self.ensure_schema_state_valid().await?; self.coordinator @@ -2268,11 +2032,14 @@ impl Omnigraph { ) -> Result { let target = target.into(); let validate_live_snapshot = matches!(&target, ReadTarget::Branch(_)); - let (schema_ir, _) = - load_validated_schema_contract(self.uri(), Arc::clone(&self.storage)).await?; let resolved = self.resolve_target_after_schema_validation(target).await?; if validate_live_snapshot { - validate_schema_ir_against_snapshot(&schema_ir, &resolved.snapshot)?; + let (catalog, _) = self + .accepted_catalog_for_snapshot(&resolved.snapshot) + .await?; + validate_bound_catalog_against_snapshot(&catalog, &resolved.snapshot)?; + } else { + self.ensure_schema_state_valid().await?; } Ok(resolved) } @@ -2305,9 +2072,12 @@ impl Omnigraph { /// Capture one live/historical target snapshot and the accepted immutable /// catalog under the same process-local schema-publication gate. /// - /// The catalog is rebuilt from the accepted on-disk contract rather than - /// read from this handle's ArcSwap: a handle opened before another handle's - /// SchemaApply intentionally has a stale warm catalog until refresh. + /// A live Branch read serves the contract of the manifest version it + /// resolved (the freshness probe refreshes the coordinator, so a handle + /// opened before another handle's SchemaApply sees that apply's row); a + /// point-in-time read keeps the LIVE contract the handle's coordinator + /// holds, rebinding the image's tables by identity. Neither reads this + /// handle's ArcSwap view, which is stale until refresh. pub(crate) async fn capture_read_view( &self, target: impl Into, @@ -2316,12 +2086,15 @@ impl Omnigraph { let validate_live_snapshot = matches!(&target, ReadTarget::Branch(_)); let bind_historical_aliases = matches!(&target, ReadTarget::Snapshot(_)); let _schema_permit = self.write_queue().acquire_schema_shared().await; - let catalog = self.build_accepted_catalog_with_schema_gate_held().await?; let mut resolved = self.resolve_target_after_schema_validation(target).await?; if validate_live_snapshot { + let (catalog, _) = self + .accepted_catalog_for_snapshot(&resolved.snapshot) + .await?; validate_bound_catalog_against_snapshot(&catalog, &resolved.snapshot)?; return Ok((resolved, catalog)); } + let catalog = self.load_accepted_catalog_with_schema_gate_held().await?; if bind_historical_aliases { let catalog = self .catalog_for_image_vintage(&resolved.snapshot, catalog) @@ -2341,10 +2114,12 @@ impl Omnigraph { .current_branch() .unwrap_or("main") .to_string(); - let catalog = self.build_accepted_catalog_with_schema_gate_held().await?; let resolved = self .resolve_target_after_schema_validation(ReadTarget::branch(current_branch)) .await?; + let (catalog, _) = self + .accepted_catalog_for_snapshot(&resolved.snapshot) + .await?; validate_bound_catalog_against_snapshot(&catalog, &resolved.snapshot)?; Ok((resolved, catalog)) } @@ -2402,7 +2177,15 @@ impl Omnigraph { if image_vintage == catalog.system_columns { return Ok(catalog); } - let accepted_ir = read_accepted_schema_ir(self.uri(), Arc::clone(&self.storage)).await?; + let accepted_ir = catalog + .bound_schema_ir() + .ok_or_else(|| { + OmniError::manifest_internal( + "runtime catalog is not bound to an accepted identity-bearing SchemaIR" + .to_string(), + ) + })? + .clone(); let vintage_ir = if image_vintage == omnigraph_compiler::SYSTEM_COLUMNS_LEGACY { omnigraph_compiler::into_legacy_image_vintage(accepted_ir) } else { @@ -2432,8 +2215,14 @@ impl Omnigraph { { let coord = self.coordinator.read().await; if normalized.as_deref() != coord.current_branch() { - // Different branch: cold resolve (opens that branch). - return coord.resolve_target(target).await; + drop(coord); + if let Some(resolved) = self + .cached_read_target(normalized.as_deref(), target) + .await? + { + return Ok(resolved); + } + return self.coordinator.read().await.resolve_target(target).await; } let held = coord.manifest_incarnation(); if coord.probe_latest_incarnation().await?.matches(&held) { @@ -3075,7 +2864,6 @@ impl Omnigraph { &omnigraph_policy::ResourceScope::TargetBranch(name.to_string()), actor, )?; - ensure_public_branch_ref(name, "branch_create")?; let target = normalize_branch_name(name)? .ok_or_else(|| OmniError::manifest("cannot create branch 'main'".to_string()))?; let _export_exclusion = self.reserve_export_destructive_control()?; @@ -3083,51 +2871,39 @@ impl Omnigraph { .await .map_err(OmniError::before_effect)?; let source = self.active_branch().await; - let completed_prior_work = self.settle_pending_schema_install().await?; - let preparation_error = |error: OmniError| { - if completed_prior_work { - error.without_pre_effect_evidence() - } else { - error.before_effect() - } - }; - fail(&BRANCH_CONTROL_PRE_GATES).map_err(preparation_error)?; + fail(&BRANCH_CONTROL_PRE_GATES).map_err(OmniError::before_effect)?; let _schema_permit = self.write_queue().acquire_schema_exclusive().await; let _branch_guards = self .write_queue() .acquire_branches(&[source.clone(), Some(target.clone())]) .await; - self.ensure_schema_apply_not_locked("branch_create") - .await - .map_err(preparation_error)?; - let control_catalog = self - .build_accepted_catalog_with_schema_gate_held() - .await - .map_err(preparation_error)?; - let table_queue_keys = self.table_queue_keys_for_branches( - &[source.clone(), Some(target.clone())], - &control_catalog, - ); - let _table_guards = self.write_queue().acquire_many(&table_queue_keys).await; - self.ensure_schema_apply_not_locked("branch_create") + let branches = [source.clone(), Some(target.clone())]; + let warm_snapshot = self.coordinator.read().await.snapshot(); + let (control_catalog, identity) = self + .accepted_catalog_for_snapshot(&warm_snapshot) .await - .map_err(preparation_error)?; + .map_err(OmniError::before_effect)?; + let table_queue_keys = self.table_queue_keys_for_branches(&branches, &control_catalog); + let table_guards = self.write_queue().acquire_many(&table_queue_keys).await; self.ensure_schema_state_valid() .await - .map_err(preparation_error)?; + .map_err(OmniError::before_effect)?; let mut source_coord = self .capture_branch_control_source(source.as_deref()) .await - .map_err(preparation_error)?; - validate_bound_catalog_against_snapshot(&control_catalog, &source_coord.snapshot()) - .map_err(preparation_error)?; - source_coord.branch_create(&target).await.map_err(|error| { - if completed_prior_work { - error.without_pre_effect_evidence() - } else { - error - } - })?; + .map_err(OmniError::before_effect)?; + let (_control_catalog, _table_guards) = self + .join_control_catalog_to_capture( + control_catalog, + &identity, + &branches, + table_guards, + &source_coord, + "branch_create", + ) + .await + .map_err(OmniError::before_effect)?; + source_coord.branch_create(&target).await?; self.invalidate_read_caches().await; Ok(()) } @@ -3166,25 +2942,16 @@ impl Omnigraph { }, actor, )?; - self.branch_create_from_impl(target, name, false).await + self.branch_create_from_impl(target, name).await } - async fn branch_create_from_impl( - &self, - from: impl Into, - name: &str, - allow_internal_refs: bool, - ) -> Result<()> { + async fn branch_create_from_impl(&self, from: impl Into, name: &str) -> Result<()> { let target = from.into(); let ReadTarget::Branch(branch_name) = target else { return Err(OmniError::manifest( "branch creation from pinned snapshots is not supported yet".to_string(), )); }; - if !allow_internal_refs { - ensure_public_branch_ref(&branch_name, "branch_create_from")?; - ensure_public_branch_ref(name, "branch_create_from")?; - } let branch = normalize_branch_name(&branch_name)?; let target_branch = normalize_branch_name(name)? .ok_or_else(|| OmniError::manifest("cannot create branch 'main'".to_string()))?; @@ -3192,57 +2959,42 @@ impl Omnigraph { self.ensure_schema_state_valid() .await .map_err(OmniError::before_effect)?; - let completed_prior_work = self.settle_pending_schema_install().await?; - let preparation_error = |error: OmniError| { - if completed_prior_work { - error.without_pre_effect_evidence() - } else { - error.before_effect() - } - }; - fail(&BRANCH_CONTROL_PRE_GATES).map_err(preparation_error)?; + fail(&BRANCH_CONTROL_PRE_GATES).map_err(OmniError::before_effect)?; let _schema_permit = self.write_queue().acquire_schema_exclusive().await; let _branch_guards = self .write_queue() .acquire_branches(&[branch.clone(), Some(target_branch.clone())]) .await; - self.ensure_schema_apply_not_locked("branch_create_from") - .await - .map_err(preparation_error)?; - let control_catalog = self - .build_accepted_catalog_with_schema_gate_held() + let branches = [branch.clone(), Some(target_branch.clone())]; + let warm_snapshot = self.coordinator.read().await.snapshot(); + let (control_catalog, identity) = self + .accepted_catalog_for_snapshot(&warm_snapshot) .await - .map_err(preparation_error)?; - let table_queue_keys = self.table_queue_keys_for_branches( - &[branch.clone(), Some(target_branch.clone())], - &control_catalog, - ); - let _table_guards = self.write_queue().acquire_many(&table_queue_keys).await; - self.ensure_schema_apply_not_locked("branch_create_from") - .await - .map_err(preparation_error)?; + .map_err(OmniError::before_effect)?; + let table_queue_keys = self.table_queue_keys_for_branches(&branches, &control_catalog); + let table_guards = self.write_queue().acquire_many(&table_queue_keys).await; self.ensure_schema_state_valid() .await - .map_err(preparation_error)?; + .map_err(OmniError::before_effect)?; let mut source_coord = self .capture_branch_control_source(branch.as_deref()) .await - .map_err(preparation_error)?; - validate_bound_catalog_against_snapshot(&control_catalog, &source_coord.snapshot()) - .map_err(preparation_error)?; + .map_err(OmniError::before_effect)?; + let (_control_catalog, _table_guards) = self + .join_control_catalog_to_capture( + control_catalog, + &identity, + &branches, + table_guards, + &source_coord, + "branch_create_from", + ) + .await + .map_err(OmniError::before_effect)?; // A locally owned source coordinator cannot be swapped by a concurrent // `branch_create_from`; the ref write is durable whichever handle // issued it. - source_coord - .branch_create(&target_branch) - .await - .map_err(|error| { - if completed_prior_work { - error.without_pre_effect_evidence() - } else { - error - } - })?; + source_coord.branch_create(&target_branch).await?; self.invalidate_read_caches().await; Ok(()) } @@ -3272,67 +3024,70 @@ impl Omnigraph { &omnigraph_policy::ResourceScope::TargetBranch(name.to_string()), actor, )?; - ensure_public_branch_ref(name, "branch_delete")?; let branch = normalize_branch_name(name)? .ok_or_else(|| OmniError::manifest("cannot delete branch 'main'".to_string()))?; let _export_exclusion = self.reserve_export_destructive_control()?; self.ensure_schema_state_valid() .await .map_err(OmniError::before_effect)?; - let completed_prior_work = self.settle_pending_schema_install().await?; - let preparation_error = |error: OmniError| { - if completed_prior_work { - error.without_pre_effect_evidence() - } else { - error.before_effect() - } - }; - fail(&BRANCH_CONTROL_PRE_GATES).map_err(preparation_error)?; + fail(&BRANCH_CONTROL_PRE_GATES).map_err(OmniError::before_effect)?; let _schema_permit = self.write_queue().acquire_schema_shared().await; let _branch_guard = self.write_queue().acquire_branch(Some(&branch)).await; - // Purge only after taking the branch gate. Merge capture takes the - // same branch-gate -> cache-lock order, so no later insert for this - // incarnation can race between invalidation and deletion. let mut cache = self.merge_authority_cache.lock().await; - if cache + let cached_target = if cache .as_ref() .is_some_and(|(cached_branch, _)| cached_branch == &branch) { - *cache = None; - } + cache.take().map(|(_, coordinator)| coordinator) + } else { + None + }; drop(cache); - self.ensure_schema_apply_not_locked("branch_delete") + let branches = [Some(branch.clone())]; + let warm_snapshot = self.coordinator.read().await.snapshot(); + let (control_catalog, identity) = self + .accepted_catalog_for_snapshot(&warm_snapshot) .await - .map_err(preparation_error)?; - let control_catalog = self - .build_accepted_catalog_with_schema_gate_held() - .await - .map_err(preparation_error)?; - let table_queue_keys = - self.table_queue_keys_for_branches(&[Some(branch.clone())], &control_catalog); - let _table_guards = self.write_queue().acquire_many(&table_queue_keys).await; - fail(&BRANCH_DELETE_POST_TABLE_GATES).map_err(preparation_error)?; - self.ensure_schema_apply_not_locked("branch_delete") - .await - .map_err(preparation_error)?; + .map_err(OmniError::before_effect)?; + let table_queue_keys = self.table_queue_keys_for_branches(&branches, &control_catalog); + let table_guards = self.write_queue().acquire_many(&table_queue_keys).await; + fail(&BRANCH_DELETE_POST_TABLE_GATES).map_err(OmniError::before_effect)?; self.ensure_schema_state_valid() .await - .map_err(preparation_error)?; - let mut target_control = self - .open_coordinator_for_branch(Some(branch.as_str())) + .map_err(OmniError::before_effect)?; + let cached_target = match cached_target { + Some(coordinator) + if coordinator.current_branch() == Some(branch.as_str()) + && coordinator + .probe_latest_incarnation() + .await + .map_err(OmniError::before_effect)? + .matches(&coordinator.manifest_incarnation()) => + { + Some(coordinator) + } + _ => None, + }; + let mut target_control = match cached_target { + Some(coordinator) => coordinator, + None => self + .open_coordinator_for_branch(Some(branch.as_str())) + .await + .map_err(OmniError::before_effect)?, + }; + let (_control_catalog, _table_guards) = self + .join_control_catalog_to_capture( + control_catalog, + &identity, + &branches, + table_guards, + &target_control, + "branch_delete", + ) .await - .map_err(preparation_error)?; - validate_bound_catalog_against_snapshot(&control_catalog, &target_control.snapshot()) - .map_err(preparation_error)?; + .map_err(OmniError::before_effect)?; self.delete_captured_branch_storage(&branch, &mut target_control) .await - .map_err(|error| { - if completed_prior_work { - error.without_pre_effect_evidence() - } else { - error - } - }) } pub async fn get_commit(&self, commit_id: &str) -> Result { @@ -3485,16 +3240,6 @@ async fn warm_resolved_target( }) } -pub(crate) fn ensure_public_branch_ref(branch: &str, operation: &str) -> Result<()> { - if is_internal_system_branch(branch) { - return Err(OmniError::manifest(format!( - "{} does not allow internal system ref '{}'", - operation, branch - ))); - } - Ok(()) -} - fn concat_or_empty_batches(schema: Arc, batches: Vec) -> Result { if batches.is_empty() { return Ok(RecordBatch::new_empty(schema)); @@ -3727,11 +3472,6 @@ pub(crate) fn read_schema_shape_for_vintage( } /// Root-scoped durable ownership for graph initialization. -/// -/// This is deliberately separate from `_schema.pg`: force init is allowed to -/// replace an orphan schema source, so that file cannot arbitrate two force -/// callers. The claim is transient control state, not part of manifest schema -/// v6 and not graph authority after `__manifest` exists. const INIT_CLAIM_FILENAME: &str = "__init_claim.json"; const INIT_CLAIM_PAYLOAD_VERSION: u32 = 1; @@ -3794,17 +3534,6 @@ async fn preflight_init_target( uri: root.to_string(), }); } - for candidate in [ - schema_source_uri(root), - schema_ir_uri(root), - schema_state_uri(root), - ] { - if storage.exists(&candidate).await? { - return Err(OmniError::AlreadyInitialized { - uri: root.to_string(), - }); - } - } Ok(()) } } @@ -3822,12 +3551,7 @@ decide_seam! { pub static LOCAL_CREATE_IF_ABSENT_PROBE = ("storage.local_create_if_absent_probe", AnyWrite, [Fail]); } -/// Refuse a local read-write bind (`init`, or `open` for read-write) whose -/// filesystem cannot do atomic create-if-absent (no `hard_link(2)`: Android -/// app storage, FAT/exFAT — issue #453). The root `__init_claim.json`, the -/// strict-mode `_schema.pg` defence, and every Lance commit need it on every -/// write, and it is a property of the mount behind the root (a store can be -/// copied), so probe per root, per bind. +/// Probe the local filesystem capability used by the init claim and Lance commits. async fn verify_local_create_if_absent(root: &str, storage: &dyn StorageAdapter) -> Result<()> { if storage_kind_for_uri(root)? != StorageKind::Local { return Ok(()); @@ -3851,9 +3575,7 @@ async fn verify_local_create_if_absent(root: &str, storage: &dyn StorageAdapter) ))) } -/// `--force` may replace orphan schema files, but it must never mint a new -/// identity domain over an existing source-of-truth manifest. Reusing the root -/// would make old rows appear to belong to unrelated freshly allocated IDs. +/// Refuse rebinding existing graph authority to a fresh identity domain. async fn refuse_force_init_over_existing_manifest( root: &str, storage: &dyn StorageAdapter, @@ -3868,68 +3590,25 @@ async fn refuse_force_init_over_existing_manifest( } } -/// Commit phase of `Omnigraph::init_with_storage`: every durable write of -/// graph creation, ending with the stamped `__manifest` Create commit — the -/// graph's commit point, and this function's final operation. The returned -/// [`InitCommitError`] distinguishes a failure before any physical graph -/// initialization from an acknowledgement-unknown physical result; only the -/// former grants schema cleanup authority. Everything after a confirmed commit lives in -/// `init_post_commit_checks`. -/// -/// Failpoints fire at the phase boundaries: -/// * `init.after_schema_pg_written` — `_schema.pg` is on disk. In strict mode -/// this fires in the caller immediately after the atomic ownership claim; in -/// force mode it fires here after the explicit overwrite. -/// * `init.after_schema_contract_written` — `_schema.pg` + `_schema.ir.json` -/// + `__schema_state.json` are on disk. -enum InitCommitError { - BeforePhysicalInit(OmniError), - PhysicalInitOutcomeUnknown(OmniError), -} - -decide_seam! { - pub static INIT_AFTER_SCHEMA_CONTRACT_WRITTEN = ("init.after_schema_contract_written", Unreachable, [Fail]); -} - async fn init_commit_phase( root: &str, - contract: &SchemaContractText, + manifest_contract: &SchemaContractRow, catalog: &Catalog, - storage: &Arc, - write_schema_pg: bool, control_session: &Arc, attempt: &GenesisManifestAttempt, -) -> std::result::Result { - if write_schema_pg { - let schema_path = join_uri(root, SCHEMA_SOURCE_FILENAME); - storage - .write_text(&schema_path, &contract.source) - .await - .map_err(InitCommitError::BeforePhysicalInit)?; - fail(&INIT_AFTER_SCHEMA_PG_WRITTEN).map_err(InitCommitError::BeforePhysicalInit)?; - } - - write_schema_contract(root, storage.as_ref(), contract) - .await - .map_err(InitCommitError::BeforePhysicalInit)?; - fail(&INIT_AFTER_SCHEMA_CONTRACT_WRITTEN).map_err(InitCommitError::BeforePhysicalInit)?; - - // From this invocation onward, per-table Dataset::write(Create) calls may - // already have landed even if the graph manifest itself is not visible. - // Collapse every graph-init error to the unknown-outcome arm: releasing - // the claim would allow a retry to collide with a late/ack-lost table - // Create and could strand a hybrid root. - GraphCoordinator::init_commit_with_session(root, catalog, control_session, attempt) - .await - .map_err(|error| InitCommitError::PhysicalInitOutcomeUnknown(error.into_source())) +) -> Result { + GraphCoordinator::init_commit_with_session( + root, + catalog, + manifest_contract, + control_session, + attempt, + ) + .await + .map_err(|error| error.into_source()) } -/// Everything past the commit point (see `init_commit_phase`): reads the -/// committed manifest back, assembles the coordinator, and validates. The -/// graph is complete before this runs, so callers must not run -/// `best_effort_cleanup_init_artifacts` on error. The -/// `init.post_manifest_create` and `init.after_coordinator_init` failpoints -/// both fire inside this phase. +/// Read the published genesis and validate its schema and table identity. async fn init_post_commit_checks( root: &str, manifest_dataset: Dataset, @@ -3957,70 +3636,6 @@ async fn finish_init_coordinator( Ok(coordinator) } -/// Best-effort cleanup of init-phase artifacts while `claim` excludes every -/// cooperating initializer. Schema artifacts are removed first and the claim -/// is released last; reversing that order would let a new force caller enter -/// and have its contract deleted by this attempt. -async fn best_effort_cleanup_owned_init_artifacts( - root: &str, - storage: &dyn StorageAdapter, - claim: &InitClaim, -) { - if best_effort_cleanup_init_artifacts(root, storage).await { - best_effort_release_init_claim(claim, storage).await; - } else { - tracing::warn!( - target: "omnigraph::init::claim", - uri = %claim.uri, - "init cleanup had an indeterminate delete outcome; retaining the durable claim so a later initializer cannot race a delayed schema deletion", - ); - } -} - -decide_seam! { - /// Inject an indeterminate schema-artifact delete during pre-physical init - /// cleanup. The original init error must win and the durable claim must be - /// retained so a delayed delete cannot race another initializer. - pub static INIT_SCHEMA_CLEANUP_DELETE = ("init.schema_cleanup_delete", Unreachable, [Fail]); -} - -/// Best-effort deletion of the three schema artifacts. This primitive must be -/// called only by `best_effort_cleanup_owned_init_artifacts`, while the caller -/// retains the root init claim, and never past a committed manifest outcome -/// (issue #495). -/// -/// Removes the three schema files: `_schema.pg`, `_schema.ir.json`, -/// `__schema_state.json`. Lance datasets and `__manifest/` are not -/// touched here — recursive directory deletion requires a -/// `StorageAdapter::delete_prefix` primitive that's deferred along -/// with `DELETE /graphs/{id}` (MR-668 PR 2b). -/// -/// Failures to delete are logged via `tracing::warn` and do not mask -/// the original init error. -async fn best_effort_cleanup_init_artifacts(root: &str, storage: &dyn StorageAdapter) -> bool { - let mut complete = true; - for uri in [ - schema_source_uri(root), - schema_ir_uri(root), - schema_state_uri(root), - ] { - let deletion = match fail(&INIT_SCHEMA_CLEANUP_DELETE) { - Ok(()) => storage.delete(&uri).await, - Err(err) => Err(err), - }; - if let Err(err) = deletion { - complete = false; - tracing::warn!( - target: "omnigraph::init::cleanup", - uri = %uri, - error = %err, - "init failed; best-effort cleanup could not delete artifact", - ); - } - } - complete -} - fn schema_table_key(type_kind: SchemaTypeKind, name: &str) -> String { match type_kind { SchemaTypeKind::Node => format!("node:{}", name), @@ -4295,9 +3910,9 @@ edge WorksAt: Person -> Company async fn delete(&self, uri: &str) -> Result<()> { if [ - schema_source_uri(&self.root), - schema_ir_uri(&self.root), - schema_state_uri(&self.root), + join_uri(&self.root, "_schema.pg"), + join_uri(&self.root, "_schema.ir.json"), + join_uri(&self.root, "__schema_state.json"), ] .iter() .any(|candidate| candidate == uri) @@ -4405,18 +4020,8 @@ edge WorksAt: Person -> Company assert!(matches!(loser, OmniError::InitializationClaimed { .. })); assert_eq!(schema_deletes.load(Ordering::SeqCst), 0); - assert!( - dir.path().join("_schema.pg").exists(), - "winning init must leave _schema.pg in place" - ); - assert!( - dir.path().join("_schema.ir.json").exists(), - "winning init must leave _schema.ir.json in place" - ); - assert!( - dir.path().join("__schema_state.json").exists(), - "winning init must leave __schema_state.json in place" - ); + let reopened = Omnigraph::open(&uri).await.unwrap(); + assert_eq!(reopened.schema_source().as_str(), TEST_SCHEMA); assert!( !dir.path().join(INIT_CLAIM_FILENAME).exists(), "the completed winner must release its transient init claim" @@ -4455,11 +4060,6 @@ edge WorksAt: Person -> Company 0, "the claim loser must never delete the winner's schema artifacts" ); - assert_eq!( - std::fs::read_to_string(dir_path_from_uri(uri).join(SCHEMA_SOURCE_FILENAME)).unwrap(), - winning_schema, - "the durable schema contract must belong to the claim winner" - ); assert!( !dir_path_from_uri(uri).join(INIT_CLAIM_FILENAME).exists(), "the completed winner must release its transient init claim" @@ -4512,9 +4112,9 @@ edge WorksAt: Person -> Company let root = normalize_root_uri(uri).unwrap(); let storage = ObjectStorageAdapter::local(); let artifacts = [ - (schema_source_uri(&root), "orphan source"), - (schema_ir_uri(&root), "orphan ir"), - (schema_state_uri(&root), "orphan state"), + (join_uri(&root, "_schema.pg"), "orphan source"), + (join_uri(&root, "_schema.ir.json"), "orphan ir"), + (join_uri(&root, "__schema_state.json"), "orphan state"), ]; for (artifact_uri, contents) in &artifacts { storage.write_text(artifact_uri, contents).await.unwrap(); @@ -4587,34 +4187,26 @@ edge WorksAt: Person -> Company ) .await .unwrap(); - assert!(adapter.writes().contains(&join_uri(uri, "_schema.pg"))); - assert!(adapter.writes().contains(&join_uri(uri, "_schema.ir.json"))); - assert!( - adapter - .writes() - .contains(&join_uri(uri, "__schema_state.json")) - ); - + for file in ["_schema.pg", "_schema.ir.json", "__schema_state.json"] { + assert!(!adapter.writes().contains(&join_uri(uri, file))); + } + let reads_before_open = adapter.reads().len(); + let exists_before_open = adapter.exists_checks().len(); Omnigraph::open_with_storage(uri, adapter.clone()) .await .unwrap(); - assert!(adapter.reads().contains(&join_uri(uri, "_schema.pg"))); - assert!(adapter.reads().contains(&join_uri(uri, "_schema.ir.json"))); - assert!( - adapter - .reads() - .contains(&join_uri(uri, "__schema_state.json")) - ); - assert!( - adapter - .exists_checks() - .contains(&join_uri(uri, "_schema.ir.json")) - ); - assert!( - adapter - .exists_checks() - .contains(&join_uri(uri, "__schema_state.json")) - ); + let open_reads = adapter.reads().split_off(reads_before_open); + let open_exists = adapter.exists_checks().split_off(exists_before_open); + for file in ["_schema.pg", "_schema.ir.json", "__schema_state.json"] { + assert!( + !open_reads.contains(&join_uri(uri, file)), + "open must not read {file}" + ); + assert!( + !open_exists.contains(&join_uri(uri, file)), + "open must not probe {file}" + ); + } // (Phase B retired `_graph_commits.lance`: open no longer probes for it.) } @@ -4710,7 +4302,7 @@ edge WorksAt: Person -> Company .properties .contains_key("nickname") ); - assert!(dir.path().join("_schema.pg").exists()); + assert!(!dir.path().join("_schema.pg").exists()); } #[tokio::test] @@ -4735,6 +4327,149 @@ edge WorksAt: Person -> Company assert!(rows[0].get("age").is_none()); } + #[tokio::test] + async fn test_write_revalidation_admits_changed_contract_content() { + #[cfg(feature = "failpoints")] + let _scenario = crate::seams::FailScenario::setup(); + for corruption in ["source", "ir", "formatting"] { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); + let txn = db.open_write_txn(None).await.unwrap(); + let mut foreign = omnigraph_catalog::ManifestCoordinator::open(uri) + .await + .unwrap(); + let mut row = foreign.read_schema_contract().await.unwrap(); + match corruption { + "source" => row.source = "node Different { age: String }".to_string(), + "ir" => row.ir = "not valid JSON".to_string(), + _ => row.source = format!("\n{}\n", row.source), + } + foreign + .commit_changes(&[omnigraph_catalog::ManifestChange::SchemaContract(row)]) + .await + .unwrap(); + let before = foreign.version(); + let result = db.revalidate_write_txn(&txn).await; + if corruption == "formatting" { + let snapshot = result.unwrap(); + assert_eq!(snapshot.graph_manifest_version(), before); + let accepted = db.accepted_catalog_for_snapshot(&snapshot).await.unwrap().0; + assert!(Arc::ptr_eq(&accepted, &txn.catalog)); + } else { + let error = result.unwrap_err().to_string(); + let expected = if corruption == "source" { + "source no longer matches" + } else { + "schema contract in the schema_contract row is invalid" + }; + assert!(error.contains(expected), "{error}"); + } + foreign.refresh_with_lineage().await.unwrap(); + assert_eq!(foreign.version(), before); + } + } + + #[tokio::test] + async fn test_schema_contract_sync_uses_recreated_native_branch_at_same_version() { + #[cfg(feature = "failpoints")] + let _scenario = crate::seams::FailScenario::setup(); + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let owner = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); + owner.branch_create("dev").await.unwrap(); + let mut reader = Omnigraph::open(uri).await.unwrap(); + reader.sync_branch("dev").await.unwrap(); + seed_person_row(&mut reader, "Alice", Some(30)).await; + let old = reader.snapshot().await; + owner.branch_delete("dev").await.unwrap(); + let desired = TEST_SCHEMA + .replace("node Person {\n", "node Human @rename_from(\"Person\") {\n") + .replace("edge Knows: Person -> Person", "edge Knows: Human -> Human") + .replace( + "edge WorksAt: Person -> Company", + "edge WorksAt: Human -> Company", + ); + owner.apply_schema(&desired).await.unwrap(); + owner.branch_create("dev").await.unwrap(); + let fresh = Omnigraph::open(uri).await.unwrap(); + fresh.sync_branch("dev").await.unwrap(); + let replacement = fresh.snapshot().await; + assert_eq!( + old.graph_manifest_version(), + replacement.graph_manifest_version() + ); + assert_ne!(old.native_branch(), replacement.native_branch()); + assert_ne!(old.schema_contract(), replacement.schema_contract()); + reader.sync_branch("dev").await.unwrap(); + assert!(reader.snapshot().await.dataset("node:Human").is_some()); + assert!(reader.snapshot().await.dataset("node:Person").is_none()); + assert!(reader.catalog().node_type_id("Human").is_some()); + } + + #[tokio::test] + async fn test_schema_contract_retired_snapshot_cannot_poison_catalog_memo() { + #[cfg(feature = "failpoints")] + let _scenario = crate::seams::FailScenario::setup(); + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let owner = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); + owner.branch_create("dev").await.unwrap(); + let mut old_reader = Omnigraph::open(uri).await.unwrap(); + old_reader.sync_branch("dev").await.unwrap(); + seed_person_row(&mut old_reader, "Alice", Some(30)).await; + let old = old_reader.snapshot().await; + let old_identity = snapshot_contract_identity(&old).unwrap(); + owner.branch_delete("dev").await.unwrap(); + let desired = TEST_SCHEMA.replace("age: I32?", "age: I32?\n nickname: String?"); + owner.apply_schema(&desired).await.unwrap(); + owner.branch_create("dev").await.unwrap(); + let reader = Omnigraph::open(uri).await.unwrap(); + let current = reader.snapshot_for_branch(Some("dev")).await.unwrap(); + let current_identity = snapshot_contract_identity(¤t).unwrap(); + assert_eq!( + old.graph_manifest_version(), + current.graph_manifest_version() + ); + assert_ne!(old.native_branch(), current.native_branch()); + assert_ne!(old_identity, current_identity); + assert!( + reader + .read_caches + .accepted_catalog + .get(&old_identity) + .is_none() + ); + match reader.accepted_catalog_for_snapshot(&old).await { + Ok((catalog, identity)) => { + assert_eq!(identity, old_identity); + assert!(catalog.node_property_id("Person", "nickname").is_none()); + let memo = reader + .read_caches + .accepted_catalog + .get(&old_identity) + .unwrap(); + assert!(memo.node_property_id("Person", "nickname").is_none()); + } + Err(_) => { + assert!( + reader + .read_caches + .accepted_catalog + .get(&old_identity) + .is_none() + ); + assert!( + reader + .read_caches + .accepted_catalog + .get(¤t_identity) + .is_some() + ); + } + } + } + #[tokio::test] async fn test_apply_schema_renames_type_and_preserves_historical_snapshot() { #[cfg(feature = "failpoints")] @@ -4874,61 +4609,4 @@ edge WorksAt: Person -> Company assert!(db.storage().has_btree_index(&ds, "__id").await.unwrap()); assert!(db.storage().has_fts_index(&ds, "name").await.unwrap()); } - - #[tokio::test] - async fn test_open_for_mutation_rejects_while_schema_apply_locked() { - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap(); - let db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - db.coordinator - .write() - .await - .branch_create(SCHEMA_APPLY_LOCK_BRANCH) - .await - .unwrap(); - - let err = db - .open_for_mutation("node:Person", crate::db::MutationOpKind::Insert) - .await - .unwrap_err(); - assert!( - err.to_string() - .contains("write is unavailable while schema apply is in progress") - ); - } - - #[tokio::test] - async fn test_commit_updates_rejects_while_schema_apply_locked() { - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap(); - let mut db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - db.coordinator - .write() - .await - .branch_create(SCHEMA_APPLY_LOCK_BRANCH) - .await - .unwrap(); - - let err = db.commit_updates(&[]).await.unwrap_err(); - assert!( - err.to_string() - .contains("write commit is unavailable while schema apply is in progress") - ); - } - - #[tokio::test] - async fn test_branch_list_hides_schema_apply_lock_branch() { - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap(); - let db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - db.coordinator - .write() - .await - .branch_create(SCHEMA_APPLY_LOCK_BRANCH) - .await - .unwrap(); - - let branches = db.branch_list().await.unwrap(); - assert_eq!(branches, vec!["main".to_string()]); - } } diff --git a/crates/omnigraph/src/db/omnigraph/collector.rs b/crates/omnigraph/src/db/omnigraph/collector.rs index 3c8008377..efd271f80 100644 --- a/crates/omnigraph/src/db/omnigraph/collector.rs +++ b/crates/omnigraph/src/db/omnigraph/collector.rs @@ -840,12 +840,6 @@ pub(crate) async fn plan_collection( let registry = main.clone(); let mut main = Some(main); for branch in branches { - if branch - .as_deref() - .is_some_and(crate::db::is_internal_system_branch) - { - continue; - } let opened = match branch { None => main.take().ok_or_else(|| { OmniError::manifest_internal("collector captured main more than once") diff --git a/crates/omnigraph/src/db/omnigraph/export.rs b/crates/omnigraph/src/db/omnigraph/export.rs index f1ac83edd..75639219d 100644 --- a/crates/omnigraph/src/db/omnigraph/export.rs +++ b/crates/omnigraph/src/db/omnigraph/export.rs @@ -104,8 +104,6 @@ impl Omnigraph { actual: 2, } })?; - - self.settle_pending_schema_install().await?; let (resolved, catalog) = self.capture_read_view(ReadTarget::branch(branch)).await?; let snapshot = resolved.snapshot; let selected_tables = export_type_keys(&snapshot, type_names)?; @@ -129,7 +127,7 @@ impl Omnigraph { branch: &str, scope: &crate::changes::ChangeFeedScope, ) -> Result<(crate::changes::ChangeBaseline, ExportCut)> { - let parts = capture_baseline_parts(self, branch, scope, true).await?; + let parts = capture_baseline_parts(self, branch, scope).await?; Ok(( parts.handshake, ExportCut { @@ -265,7 +263,6 @@ async fn capture_baseline_parts( db: &Omnigraph, branch: &str, scope: &crate::changes::ChangeFeedScope, - heal: bool, ) -> Result { let slot = db.write_queue().try_acquire_export_cut().ok_or_else(|| { OmniError::ResourceLimitExceeded { @@ -274,9 +271,6 @@ async fn capture_baseline_parts( actual: 2, } })?; - if heal { - db.settle_pending_schema_install().await?; - } let normalized_branch = Some(branch).filter(|branch| *branch != "main"); let cut = db .coordinator @@ -339,7 +333,7 @@ pub(super) async fn capture_change_baseline( scope: &crate::changes::ChangeFeedScope, writer: &mut W, ) -> Result { - let parts = capture_baseline_parts(db, branch, scope, false).await?; + let parts = capture_baseline_parts(db, branch, scope).await?; let _slot = parts.slot; let mut emit = |chunk: Vec| std::future::ready(writer.write_all(&chunk).map_err(OmniError::from)); diff --git a/crates/omnigraph/src/db/omnigraph/optimize.rs b/crates/omnigraph/src/db/omnigraph/optimize.rs index 5a5e2c2bf..3d40515d5 100644 --- a/crates/omnigraph/src/db/omnigraph/optimize.rs +++ b/crates/omnigraph/src/db/omnigraph/optimize.rs @@ -224,7 +224,6 @@ decide_seam! { pub async fn optimize_all_datasets(db: &Omnigraph) -> Result> { let _export_exclusion = db.reserve_export_destructive_control()?; db.ensure_schema_state_valid().await?; - db.ensure_schema_apply_idle("optimize").await?; // Capture complete graph authority before entering any writer gate, then // revalidate it after schema -> main -> table acquisition. A concurrent @@ -238,7 +237,6 @@ pub async fn optimize_all_datasets(db: &Omnigraph) -> Result Resu } db.ensure_schema_state_valid().await?; - db.ensure_schema_apply_idle("repair").await?; // Repair may adopt physical HEADs into graph authority. Bind the entire // attempt to one accepted view and revalidate after schema -> main -> table @@ -160,7 +159,6 @@ pub async fn repair_all_datasets(db: &Omnigraph, options: RepairOptions) -> Resu // the replacement's same public alias and new identity. let _schema_permit = db.write_queue().acquire_schema_shared().await; db.refresh_coordinator_only().await?; - db.ensure_schema_apply_not_locked("repair").await?; let catalog = db.load_accepted_catalog_with_schema_gate_held().await?; let _main_branch_guard = db.write_queue().acquire_branch(None).await; diff --git a/crates/omnigraph/src/db/omnigraph/schema_apply.rs b/crates/omnigraph/src/db/omnigraph/schema_apply.rs index 8f467583a..ba7f86a50 100644 --- a/crates/omnigraph/src/db/omnigraph/schema_apply.rs +++ b/crates/omnigraph/src/db/omnigraph/schema_apply.rs @@ -55,7 +55,7 @@ fn resolve_desired_schema_ir( .map_err(|error| OmniError::manifest(error.to_string()))?; if source_hash != resolved_hash { return Err(OmniError::manifest( - "desired schema source does not match its resolved schema; refusing before schema apply staging", + "desired schema source does not match its resolved schema; refusing before schema apply effects", )); } for diagnostic in &resolution.diagnostics { @@ -109,8 +109,7 @@ pub(super) async fn plan_schema( desired_schema_source: &str, options: SchemaApplyOptions, ) -> Result { - db.ensure_schema_state_valid().await?; - let accepted_ir = read_accepted_schema_ir(db.uri(), Arc::clone(&db.storage)).await?; + let accepted_ir = accepted_ir_for_planning(db).await?; let desired_ir = resolve_desired_schema_ir(&accepted_ir, desired_schema_source)?; let mut plan = plan_schema_migration(&accepted_ir, &desired_ir) .map_err(|err| OmniError::manifest(err.to_string()))?; @@ -118,6 +117,17 @@ pub(super) async fn plan_schema( Ok(plan) } +/// The accepted IR a plan is made against: the contract of the live view the +/// handle resolves now (probe, refresh when the manifest moved), not the +/// handle's warm ArcSwap catalog. +async fn accepted_ir_for_planning(db: &Omnigraph) -> Result { + let (_, catalog) = db.capture_current_read_view().await?; + catalog + .bound_schema_ir() + .cloned() + .ok_or_else(|| OmniError::manifest_internal("accepted catalog carries no bound SchemaIR")) +} + struct PlannedSchemaApply { plan: SchemaMigrationPlan, desired_ir: SchemaIR, @@ -129,8 +139,7 @@ async fn plan_schema_for_apply( desired_schema_source: &str, options: SchemaApplyOptions, ) -> Result { - db.ensure_schema_state_valid().await?; - let accepted_ir = read_accepted_schema_ir(db.uri(), Arc::clone(&db.storage)).await?; + let accepted_ir = accepted_ir_for_planning(db).await?; plan_schema_for_apply_from_accepted(db, desired_schema_source, options, &accepted_ir).await } @@ -138,14 +147,6 @@ decide_seam! { pub static SCHEMA_APPLY_AFTER_MANIFEST_COMMIT = ("schema_apply.after_manifest_commit", Unreachable, [Fail]); } -decide_seam! { - pub static SCHEMA_APPLY_AFTER_STAGING_WRITE = ("schema_apply.after_staging_write", Unreachable, [Fail]); -} - -decide_seam! { - pub static SCHEMA_APPLY_BEFORE_STAGING_WRITE = ("schema_apply.before_staging_write", Unreachable, [Fail]); -} - decide_seam! { /// After each SchemaApply table effect commits (a detached rewrite or a /// new-table create), before the next table effect or the publication. @@ -153,19 +154,10 @@ decide_seam! { } decide_seam! { - /// Under the schema-apply sentinel and every gate, before the first - /// table effect; shared by schema apply and the RFC 0040 system-column - /// upgrade, neither of which arms a sidecar (RFC 0067). + /// Under the schema, branch and table gates, before the first table effect. pub static SCHEMA_APPLY_POST_LOCK_PRE_EFFECT = ("schema_apply.post_lock_pre_effect", Unreachable, [Fail]); } -decide_seam! { - /// Right after the durable sentinel lands, under the exclusive schema - /// permit and before any planning: the first crossing proves the apply - /// got past every shared holder. - pub static SCHEMA_APPLY_POST_SENTINEL = ("schema_apply.post_sentinel", Unreachable, [Fail]); -} - async fn plan_schema_for_apply_from_accepted( db: &Omnigraph, desired_schema_source: &str, @@ -173,14 +165,9 @@ async fn plan_schema_for_apply_from_accepted( accepted_ir: &SchemaIR, ) -> Result { let branches = db.coordinator.read().await.all_branches().await?; - // Skip `main` and internal system branches (the schema-apply lock branch, - // the cluster-wide schema-apply serializer). Legacy `__run__*` staging - // branches were swept off `__manifest` by the v2→v3 migration that runs in - // `Omnigraph::open(ReadWrite)` before this check (MR-770), so they no - // longer appear here. let blocking_branches = branches .into_iter() - .filter(|branch| branch != "main" && !is_internal_system_branch(branch)) + .filter(|branch| branch != "main") .collect::>(); if !blocking_branches.is_empty() { return Err(OmniError::manifest_conflict(format!( @@ -257,48 +244,8 @@ where let _export_exclusion = db.reserve_export_destructive_control()?; - // Install this handle's published-but-uninstalled schema contract, if any, - // before planning against the accepted contract. - let completed_prior_work = db.settle_pending_schema_install().await?; - - // Process-local schema gate, EXCLUSIVE side: schema apply is a - // contract-lifecycle pass, so it excludes every shared holder (writers, - // maintenance, branch control, read captures) and they exclude it. The - // permit is taken before the branch/table gates and before the durable - // sentinel, and retained through sentinel release, so no shared holder - // can revalidate against a contract this apply is about to replace. - // The native sentinel remains the cross-handle / crash-visible - // authority; this permit removes the avoidable same-handle race (RFC - // 2026-09-18-shared-schema-gate). let _schema_gate = db.write_queue().acquire_schema_exclusive().await; - acquire_schema_apply_lock(db).await.map_err(|error| { - if completed_prior_work { - error.without_pre_effect_evidence() - } else { - error - } - })?; - let result = async { - fail(&SCHEMA_APPLY_POST_SENTINEL)?; - apply_schema_with_lock(db, desired_schema_source, options, actor, validate_catalog).await - } - .await; - let release_result = release_schema_apply_lock(db).await; - if release_result.is_err() { - // Liveness: the next write entry on this handle retries the release - // before the sentinel gate, so the failed delete never wedges it. - db.note_failed_sentinel_release(); - } - match (result, release_result) { - (Ok(result), Ok(())) => Ok(result), - (Ok(_), Err(err)) => Err(err), - (Err(err), Ok(())) => Err(err), - // Preserve the primary diagnostic, but a failed sentinel completion - // cannot be hidden behind its ordinary validation/refusal category. - (Err(err), Err(_)) => { - Err(err.with_completion_evidence(crate::error::CompletionEvidence::Uncertain)) - } - } + apply_schema_with_lock(db, desired_schema_source, options, actor, validate_catalog).await } pub(super) async fn apply_schema_with_lock( @@ -311,19 +258,22 @@ pub(super) async fn apply_schema_with_lock( where F: FnOnce(&Catalog) -> Result<()>, { - // Capture the accepted contract, compiled catalog, manifest snapshot, and - // main authority as one operation-local view while the schema gate and - // durable SchemaApply sentinel are held. A long-lived handle's ArcSwap - // catalog may lag another handle, so it is never an authority here. - db.refresh_coordinator_only().await?; - let (accepted_ir, accepted_schema_state) = - load_validated_schema_contract(db.uri(), Arc::clone(&db.storage)).await?; - let mut accepted_catalog = build_catalog_from_ir(&accepted_ir)?; - fixup_physical_schemas(&mut accepted_catalog)?; - let accepted_catalog = Arc::new(accepted_catalog); + db.refresh_coordinator_only() + .await + .map_err(OmniError::before_effect)?; + let (accepted_catalog, accepted_identity) = { + let snapshot = db.coordinator.read().await.snapshot(); + db.accepted_catalog_for_snapshot(&snapshot) + .await + .map_err(OmniError::before_effect)? + }; + let accepted_ir = accepted_catalog.bound_schema_ir().cloned().ok_or_else(|| { + OmniError::manifest_internal("accepted catalog carries no bound SchemaIR") + })?; let planned = plan_schema_for_apply_from_accepted(db, desired_schema_source, options, &accepted_ir) - .await?; + .await + .map_err(OmniError::before_effect)?; validate_catalog(&planned.desired_catalog)?; let PlannedSchemaApply { plan, @@ -549,12 +499,6 @@ where let mut table_tombstones = BTreeMap::)>::new(); - // Preflight every existing-table participant against the captured - // snapshot. A rewrite advances the identity-owned dataset detached from - // its manifest pin (RFC 0067); a type rename is metadata-only and keeps - // identity, path, version and Lance history. Metadata/tombstone-only - // applies have no table effects: their only durable pre-publication state - // is the staged schema contract. for table_key in &rewritten_tables { if added_tables.contains(table_key) { continue; @@ -636,12 +580,6 @@ where .datasets() .map(|entry| (entry.type_key.clone(), entry.native_dataset_branch.clone())) .collect(); - // The outer `apply_schema` holds the exclusive schema permit from before - // sentinel creation through sentinel release. Per-table guards here - // therefore cover only the concrete table effects; re-acquiring either - // side of the schema gate on this task deadlocks — the gate is - // non-reentrant, and even a shared re-entry parks behind any queued - // writer under the plain-mode write-preferring lock. let _main_branch_guard = db.write_queue().acquire_branch(None).await; let _schema_apply_queue_guards = db .write_queue() @@ -686,17 +624,20 @@ where current_graph_head, )); } - let current_schema_state = read_schema_state_identity(db.uri(), db.storage.as_ref()).await?; - if current_schema_state != accepted_schema_state { + let current_snapshot = db.coordinator.read().await.snapshot(); + let (current_catalog, current_identity) = + db.accepted_catalog_for_snapshot(¤t_snapshot).await?; + validate_bound_catalog_against_snapshot(¤t_catalog, ¤t_snapshot)?; + if current_identity != accepted_identity { return Err(OmniError::manifest_read_set_changed( "schema_identity", Some(format!( "{}:{}", - accepted_schema_state.schema_identity_version, accepted_schema_state.schema_ir_hash + accepted_identity.schema_identity_version, accepted_identity.schema_ir_hash )), Some(format!( "{}:{}", - current_schema_state.schema_identity_version, current_schema_state.schema_ir_hash + current_identity.schema_identity_version, current_identity.schema_ir_hash )), )); } @@ -708,28 +649,6 @@ where existing_heads.insert(entry.type_key.clone(), head); } - // Only added types are first-touch paths; rename targets reuse the - // existing identity-owned path. An added type's path is a deterministic - // function of the accepted identity allocator, so an attempt that died - // after creating the dataset left it exactly where the retry creates it. - // Nothing references an unregistered incarnation path (identities are - // never reused and registration is this apply's own publication), so - // under the schema sentinel such a leftover is garbage: reclaim it before - // the strict version-one create. - for table_key in &added_tables { - let identity = table_identity_for_schema_key(&desired_ir, table_key)?; - let table_path = crate::db::manifest::table_path_for_identity(table_key, identity)?; - let dataset_uri = db.storage().dataset_uri(&table_path); - if db.storage_adapter().exists(&dataset_uri).await? { - tracing::warn!( - table_key, - dataset_uri, - "reclaiming an unregistered dataset left at the added type's path by an abandoned schema apply" - ); - db.storage_adapter().delete_prefix(&dataset_uri).await?; - } - } - // Lance's logical Blob rewrite input cannot represent an existing // external offset/length range. Discover that unsupported persisted state // across the complete rewrite set before any added / lexically earlier @@ -756,13 +675,7 @@ where .await?; } - // The staged contract is bound to this apply's graph commit (RFC 0067): - // a read-write open installs it once that commit is in lineage and - // discards it otherwise. - let publication = crate::db::schema_state::SchemaPublication { - graph_commit_id: lineage_intent.graph_commit_id.clone(), - parent_commit_id: base_graph_head.clone(), - }; + let graph_commit_id = lineage_intent.graph_commit_id.clone(); let mut published_commit: Option = None; let effects = async { @@ -774,34 +687,77 @@ where let table_path = crate::db::manifest::table_path_for_identity(table_key, identity)?; let dataset_uri = db.storage().dataset_uri(&table_path); let schema = schema_for_table_key(&desired_catalog, table_key)?; - let batch = RecordBatch::new_empty(schema); - let staged = db.storage().stage_create(&dataset_uri, batch).await?; - let outcome = db - .storage() - .commit_staged_create_exact(&dataset_uri, staged) - .await?; - if !outcome.is_exact() { - return Err(OmniError::manifest_internal(format!( - "SchemaApply first-touch '{}' committed outside its version-one create", - table_key - ))); - } - let ds = outcome.into_snapshot(); - // Indexes for the new table are materialized off the critical path by - // ensure_indices/optimize (iss-848); a 0-row table is never trainable - // anyway. The @index intent is recorded in the persisted catalog/IR. + let existing = match db.storage().open_dataset_head(&dataset_uri, None).await { + Ok(existing) => Some(existing), + Err(error) + if error.storage_failure().is_some_and(|failure| { + failure.kind == crate::error::StorageFailureKind::NotFound + }) => + { + None + } + Err(error) => return Err(error), + }; + let (ds, detached_transaction) = if let Some(existing) = existing { + if db + .storage() + .validate_initial_empty_table(&existing, &schema) + .await? + { + (existing, None) + } else { + let staged = db + .storage() + .stage_overwrite(&existing, RecordBatch::new_empty(schema)) + .await?; + let witness = crate::table_store::StagingWitness::new( + &base_branch_identifier, + base_graph_head.as_deref(), + )?; + let (detached, transaction) = db + .storage() + .commit_staged_detached(existing, staged, &witness) + .await?; + (detached, Some(transaction)) + } + } else { + let staged = db + .storage() + .stage_create(&dataset_uri, RecordBatch::new_empty(schema)) + .await?; + let outcome = db + .storage() + .commit_staged_create_exact(&dataset_uri, staged) + .await?; + if !outcome.is_exact() { + return Err(OmniError::manifest_internal(format!( + "SchemaApply first-touch '{}' committed outside its version-one create", + table_key + ))); + } + (outcome.into_snapshot(), None) + }; let state = db.storage().table_state(&dataset_uri, &ds).await?; + let (published_dataset_version, version_metadata) = + if let Some(transaction) = detached_transaction { + ( + 2, + state + .version_metadata + .with_staged(state.version, transaction.uuid) + .with_last_linear_version(Some(1)), + ) + } else { + (1, state.version_metadata.with_last_linear_version(Some(1))) + }; expected_table_versions.insert(identity, 0); table_registrations.insert(table_key.clone(), (identity, table_path)); - let version_metadata = state - .version_metadata - .with_last_linear_version(Some(state.version)); table_updates.insert( identity, crate::db::DatasetUpdate { identity, type_key: table_key.clone(), - published_dataset_version: state.version, + published_dataset_version, native_dataset_branch: None, entity_count: state.row_count, version_metadata, @@ -976,33 +932,10 @@ where })); } - // Stage the schema contract bound to this apply's graph commit. The - // state file is written last, so a complete staging is exactly one - // whose state file exists; the install pass reads the marker from it. - fail(&SCHEMA_APPLY_BEFORE_STAGING_WRITE)?; - let (_, ir_json, state_json) = crate::db::schema_state::render_schema_contract( + manifest_changes.push(ManifestChange::SchemaContract(render_schema_contract( &desired_ir, - Some(publication.clone()), - )?; - db.storage - .write_text( - &schema_source_staging_uri(&db.root_uri), - desired_schema_source, - ) - .await?; - db.storage - .write_text( - &crate::db::schema_state::schema_ir_staging_uri(&db.root_uri), - &ir_json, - ) - .await?; - db.storage - .write_text( - &crate::db::schema_state::schema_state_staging_uri(&db.root_uri), - &state_json, - ) - .await?; - fail(&SCHEMA_APPLY_AFTER_STAGING_WRITE)?; + desired_schema_source, + )?)); let precondition = crate::db::manifest::PublishPrecondition::ExactGraphHead( crate::db::manifest::GraphHeadExpectation::new( @@ -1025,38 +958,18 @@ where &precondition, ) .await?; - published_commit = Some(publication.graph_commit_id.clone()); - - fail(&SCHEMA_APPLY_AFTER_MANIFEST_COMMIT)?; - // Install the contract from memory rather than by renaming the - // staging (another process's open may have discarded it), then retire - // the staging. Every write is idempotent; a crash here leaves the - // staged copy for the next open to install the same way. - db.storage - .write_text(&schema_source_uri(&db.root_uri), desired_schema_source) - .await?; - write_schema_contract( - &db.root_uri, - db.storage.as_ref(), - &SchemaContractText { - source: desired_schema_source.to_string(), - ir_json, - state_json, - }, - ) - .await?; - crate::db::schema_state::cleanup_staging_files(&db.root_uri, db.storage.as_ref()).await?; + published_commit = Some(graph_commit_id); db.store_schema_view( desired_catalog, desired_schema_source.to_string(), &desired_ir, )?; - db.coordinator.write().await.refresh().await?; db.runtime_cache.invalidate_all().await; if changed_edge_tables { db.invalidate_graph_index().await; } + fail(&SCHEMA_APPLY_AFTER_MANIFEST_COMMIT)?; Ok::(graph_manifest_version) } .await; @@ -1064,16 +977,8 @@ where let manifest_version = match effects { Ok(manifest_version) => manifest_version, Err(error) => { - // Before publication nothing referenced is durable: detached - // versions, a created dataset and the staged contract are garbage - // that the next open and cleanup retire. After publication the - // manifest is authoritative and only the contract installation - // is pending, which the next read-write open or this handle's next - // write entry completes from the staged copy. return Err(match published_commit { Some(graph_commit_id) => { - db.pending_schema_install - .store(true, std::sync::atomic::Ordering::SeqCst); OmniError::recovery_required(graph_commit_id, error.to_string()) } None => error, @@ -1148,111 +1053,6 @@ async fn cleanup_dataset_old_versions(db: &Omnigraph, full_uri: &str) -> Result< Ok(()) } -pub(super) async fn ensure_schema_apply_idle(db: &Omnigraph, operation: &str) -> Result<()> { - db.refresh_coordinator_only().await?; - ensure_schema_apply_not_locked(db, operation).await -} - -pub(super) async fn acquire_schema_apply_lock(db: &Omnigraph) -> Result<()> { - // Only these initial reads precede this acquisition's durable sentinel. - // Composing callers strip the proof if they already attempted effects. - db.ensure_schema_state_valid() - .await - .map_err(OmniError::before_effect)?; - db.refresh_coordinator_only() - .await - .map_err(OmniError::before_effect)?; - let branches = db - .coordinator - .read() - .await - .all_branches() - .await - .map_err(OmniError::before_effect)?; - if branches - .iter() - .any(|branch| is_schema_apply_lock_branch(branch)) - { - return Err(OmniError::manifest_conflict( - "schema apply is already in progress".to_string(), - )); - } - - db.coordinator - .write() - .await - .branch_create(SCHEMA_APPLY_LOCK_BRANCH) - .await?; - db.refresh_coordinator_only().await?; - - let blocking_branches = db - .coordinator - .read() - .await - .all_branches() - .await? - .into_iter() - .filter(|branch| branch != "main" && !is_internal_system_branch(branch)) - .collect::>(); - if !blocking_branches.is_empty() { - let refusal = OmniError::manifest_conflict(format!( - "schema apply requires a graph with only main; found non-main branches: {}", - blocking_branches.join(", ") - )); - // Best-effort release of the sentinel we just took; a failure arms the - // handle-local retry (liveness contract), so the next write entry on - // this handle releases it before the sentinel gate instead of staying - // wedged until a read-write open. - if let Err(release_error) = release_schema_apply_lock(db).await { - db.note_failed_sentinel_release(); - tracing::warn!( - error = %release_error, - "failed to release the schema-apply sentinel after a mono-branch refusal; \ - the next write entry on this handle retries the release" - ); - return Err( - refusal.with_completion_evidence(crate::error::CompletionEvidence::Uncertain) - ); - } - return Err(refusal); - } - - Ok(()) -} - -pub(super) async fn release_schema_apply_lock(db: &Omnigraph) -> Result<()> { - // Idempotent: an open or a `refresh` that installed this apply's - // published staging may already have reclaimed the sentinel (RFC 0067). - let mut coordinator = db.coordinator.write().await; - if coordinator - .all_branches() - .await? - .iter() - .any(|branch| is_schema_apply_lock_branch(branch)) - { - coordinator.branch_delete(SCHEMA_APPLY_LOCK_BRANCH).await?; - } - drop(coordinator); - db.refresh_coordinator_only().await -} - -/// Whether a schema apply's durable sentinel stands on this graph: the -/// cross-handle and cross-process signal that a contract-lifecycle pass is -/// in flight. -pub(super) async fn schema_apply_sentinel_present(db: &Omnigraph) -> Result { - db.coordinator.read().await.schema_apply_locked().await -} - -pub(super) async fn ensure_schema_apply_not_locked(db: &Omnigraph, operation: &str) -> Result<()> { - if schema_apply_sentinel_present(db).await? { - return Err(OmniError::manifest_conflict(format!( - "{} is unavailable while schema apply is in progress", - operation - ))); - } - Ok(()) -} - pub(super) async fn batch_for_schema_apply_rewrite( db: &Omnigraph, source_ds: &SnapshotHandle, diff --git a/crates/omnigraph/src/db/omnigraph/system_column_upgrade.rs b/crates/omnigraph/src/db/omnigraph/system_column_upgrade.rs index 0d0323886..108aab314 100644 --- a/crates/omnigraph/src/db/omnigraph/system_column_upgrade.rs +++ b/crates/omnigraph/src/db/omnigraph/system_column_upgrade.rs @@ -1,19 +1,7 @@ -//! RFC 0040 Rollout step 3: the explicit system-column upgrade of one -//! legacy-vintage graph. Renames `id`/`src`/`dst` to `__id`/`__src`/`__dst` -//! in every node and edge table and installs the current-vintage schema -//! contract. Since RFC 0067 it has schema apply's shape: each rename-only -//! `Project` commits detached from the table's pin, the staged contract -//! names the graph commit that publishes it, and one manifest CAS publishes -//! every pin. It writes no -//! recovery record: a failure before the CAS leaves only reclaimable detached -//! versions and a staging the next read-write open discards; one after it -//! leaves published detached pins and a contract the next read-write -//! open (or this handle's next write) installs. Since v10 both vintages share -//! one `__manifest` stamp, so the upgrade moves no stamp; the vintage is the -//! contract's `system-columns` feature. +//! Upgrade system-column names with detached table renames and one manifest +//! publication containing every table pin and the schema contract row. use super::*; -use crate::db::schema_state::SchemaState; use crate::db::upgrade::UpgradeMode; use crate::seams::{catalog, fail}; use omnigraph_compiler::{SYSTEM_COLUMNS_LEGACY, SYSTEM_COLUMNS_V3}; @@ -170,9 +158,6 @@ pub(super) async fn upgrade_system_columns( graph_manifest_version: None, }; - if !options.check { - db.settle_pending_schema_install().await?; - } let (_shared_gate, _exclusive_gate): ( Option, Option, @@ -190,14 +175,20 @@ pub(super) async fn upgrade_system_columns( .ok_or_else(|| OmniError::manifest_internal("opened graph has no internal-schema stamp"))?; report.stamp_before = stamp; report.stamp_after = stamp; - let (accepted_ir, accepted_schema_state) = - load_validated_schema_contract(db.uri(), Arc::clone(&db.storage)).await?; + let (accepted, accepted_identity) = { + let snapshot = db.coordinator.read().await.snapshot(); + ( + db.read_schema_contract_row_for(&snapshot).await?, + snapshot_contract_identity(&snapshot)?, + ) + }; + let (accepted_ir, _) = validate_schema_contract_row(&accepted)?; if accepted_ir.system_columns() == SYSTEM_COLUMNS_V3 { report.outcome = SystemColumnUpgradeOutcome::AlreadyCurrent; return Ok(report); } - preflight(db, &accepted_ir, stamp, &mut report).await?; + preflight(db, &accepted_ir, &accepted.source, stamp, &mut report).await?; if !report.findings.is_empty() { return Ok(report); } @@ -207,22 +198,14 @@ pub(super) async fn upgrade_system_columns( } let _export_exclusion = db.reserve_export_destructive_control()?; - // This path may already have completed a prior contract installation. - // Do not let acquisition's local read proof certify the entire upgrade. - super::schema_apply::acquire_schema_apply_lock(db) - .await - .map_err(OmniError::without_pre_effect_evidence)?; - let result = execute_with_lock(db, actor, &accepted_ir, &accepted_schema_state).await; - let release_result = super::schema_apply::release_schema_apply_lock(db).await; - if release_result.is_err() { - // Liveness: the next write entry on this handle retries the release - // before the sentinel gate, so the failed delete never wedges it. - db.note_failed_sentinel_release(); - } - let graph_manifest_version = match (result, release_result) { - (Ok(version), Ok(())) => version, - (Ok(_), Err(err)) | (Err(err), _) => return Err(err), - }; + let graph_manifest_version = execute_with_lock( + db, + actor, + &accepted_ir, + &accepted.source, + &accepted_identity, + ) + .await?; report.outcome = SystemColumnUpgradeOutcome::Completed; report.stamp_after = crate::db::manifest::internal_schema_stamp_at(db.uri(), None) .await? @@ -234,6 +217,7 @@ pub(super) async fn upgrade_system_columns( async fn preflight( db: &Omnigraph, accepted_ir: &SchemaIR, + accepted_source: &str, stamp: u32, report: &mut SystemColumnUpgradeReport, ) -> Result<()> { @@ -242,7 +226,7 @@ async fn preflight( .all_branches() .await? .into_iter() - .filter(|branch| branch != "main" && !is_internal_system_branch(branch)) + .filter(|branch| branch != "main") .collect::>(); if !blocking_branches.is_empty() { report.refuse(format!( @@ -272,35 +256,28 @@ async fn preflight( crate::db::manifest::INTERNAL_MANIFEST_SCHEMA_VERSION )); } - if report.findings.is_empty() { - let source = db - .storage - .read_text(&schema_source_uri(&db.root_uri)) - .await?; - if let Err(error) = render_system_column_upgrade_target(accepted_ir, &source) { - report.refuse(error.to_string()); - } + if report.findings.is_empty() + && let Err(error) = render_system_column_upgrade_target(accepted_ir, accepted_source) + { + report.refuse(error.to_string()); } Ok(()) } -/// Stage, publish and install the upgrade under the schema-apply sentinel. +/// Publish the table renames and schema contract under the exclusive schema permit. /// Returns the published graph manifest version. async fn execute_with_lock( db: &Omnigraph, actor: Option<&str>, accepted_ir: &SchemaIR, - accepted_schema_state: &SchemaState, + accepted_source: &str, + accepted_identity: &SchemaContractIdentity, ) -> Result { db.refresh_coordinator_only().await?; - let accepted_source = db - .storage - .read_text(&schema_source_uri(&db.root_uri)) - .await?; let SystemColumnUpgradeTarget { desired_ir, desired_source, - } = render_system_column_upgrade_target(accepted_ir, &accepted_source)?; + } = render_system_column_upgrade_target(accepted_ir, accepted_source)?; let mut desired_catalog = build_catalog_from_ir(&desired_ir)?; fixup_physical_schemas(&mut desired_catalog)?; @@ -355,17 +332,20 @@ async fn execute_with_lock( current_graph_head, )); } - let current_schema_state = read_schema_state_identity(db.uri(), db.storage.as_ref()).await?; - if ¤t_schema_state != accepted_schema_state { + let current_snapshot = db.coordinator.read().await.snapshot(); + let (current_catalog, current_identity) = + db.accepted_catalog_for_snapshot(¤t_snapshot).await?; + validate_bound_catalog_against_snapshot(¤t_catalog, ¤t_snapshot)?; + if ¤t_identity != accepted_identity { return Err(OmniError::manifest_read_set_changed( "schema_identity", Some(format!( "{}:{}", - accepted_schema_state.schema_identity_version, accepted_schema_state.schema_ir_hash + accepted_identity.schema_identity_version, accepted_identity.schema_ir_hash )), Some(format!( "{}:{}", - current_schema_state.schema_identity_version, current_schema_state.schema_ir_hash + current_identity.schema_identity_version, current_identity.schema_ir_hash )), )); } @@ -384,13 +364,7 @@ async fn execute_with_lock( existing_heads.insert(entry.type_key.clone(), head); } - // The staged contract is bound to this upgrade's graph commit (RFC 0067): - // a read-write open installs it once that commit is in lineage and - // discards it otherwise. - let publication = crate::db::schema_state::SchemaPublication { - graph_commit_id: lineage_intent.graph_commit_id.clone(), - parent_commit_id: base_graph_head.clone(), - }; + let graph_commit_id = lineage_intent.graph_commit_id.clone(); let mut published_commit: Option = None; let effects = async { @@ -442,30 +416,10 @@ async fn execute_with_lock( fail(&catalog::SCHEMA_APPLY_POST_TABLE_COMMIT)?; } - // The state file is written last, so a complete staging is exactly - // one whose state file exists; the install pass reads the marker from - // it. - fail(&catalog::SCHEMA_APPLY_BEFORE_STAGING_WRITE)?; - let (_, ir_json, state_json) = crate::db::schema_state::render_schema_contract( + manifest_changes.push(ManifestChange::SchemaContract(render_schema_contract( &desired_ir, - Some(publication.clone()), - )?; - db.storage - .write_text(&schema_source_staging_uri(&db.root_uri), &desired_source) - .await?; - db.storage - .write_text( - &crate::db::schema_state::schema_ir_staging_uri(&db.root_uri), - &ir_json, - ) - .await?; - db.storage - .write_text( - &crate::db::schema_state::schema_state_staging_uri(&db.root_uri), - &state_json, - ) - .await?; - fail(&catalog::SCHEMA_APPLY_AFTER_STAGING_WRITE)?; + &desired_source, + )?)); let precondition = crate::db::manifest::PublishPrecondition::ExactGraphHead( crate::db::manifest::GraphHeadExpectation::new( @@ -488,47 +442,20 @@ async fn execute_with_lock( &precondition, ) .await?; - published_commit = Some(publication.graph_commit_id.clone()); - - fail(&catalog::SCHEMA_APPLY_AFTER_MANIFEST_COMMIT)?; - // Install the contract from memory rather than by renaming the - // staging (another process's open may have discarded it), then retire - // the staging. Every write is idempotent; a crash here leaves the - // staged copy for the next open to install the same way. - db.storage - .write_text(&schema_source_uri(&db.root_uri), &desired_source) - .await?; - crate::db::schema_state::write_schema_contract( - &db.root_uri, - db.storage.as_ref(), - &crate::db::schema_state::SchemaContractText { - source: desired_source.clone(), - ir_json, - state_json, - }, - ) - .await?; - crate::db::schema_state::cleanup_staging_files(&db.root_uri, db.storage.as_ref()).await?; + published_commit = Some(graph_commit_id); db.store_schema_view(desired_catalog, desired_source, &desired_ir)?; - db.coordinator.write().await.refresh().await?; db.runtime_cache.invalidate_all().await; db.invalidate_graph_index().await; + fail(&catalog::SCHEMA_APPLY_AFTER_MANIFEST_COMMIT)?; Ok::(graph_manifest_version) } .await; match effects { Ok(version) => Ok(version), - // Before publication nothing referenced is durable: the detached - // renames and the staged contract are garbage the next open and - // cleanup retire. After it the manifest is authoritative and only the - // contract installation is pending, which the next read-write open or - // this handle's next write entry completes from the staged copy. Err(error) => Err(match published_commit { Some(graph_commit_id) => { - db.pending_schema_install - .store(true, std::sync::atomic::Ordering::SeqCst); OmniError::recovery_required(graph_commit_id, error.to_string()) } None => error, diff --git a/crates/omnigraph/src/db/omnigraph/table_ops.rs b/crates/omnigraph/src/db/omnigraph/table_ops.rs index 799337c11..aee6b96a4 100644 --- a/crates/omnigraph/src/db/omnigraph/table_ops.rs +++ b/crates/omnigraph/src/db/omnigraph/table_ops.rs @@ -85,7 +85,6 @@ pub(super) async fn rebuild_full_text_indices_on_as( ) -> Result { let branch = normalize_branch_name(branch)?; let public_branch = branch.as_deref().unwrap_or("main"); - ensure_public_branch_ref(public_branch, "rebuild_full_text_indices")?; db.enforce( omnigraph_policy::PolicyAction::Change, &omnigraph_policy::ResourceScope::Branch(public_branch.to_string()), @@ -233,8 +232,6 @@ async fn maintain_indices_for_branch( // Install this handle's pending schema contract, if any, before capturing // the index plan's base. The under-gate revalidation below closes the race // between this capture and the first table effect. - db.settle_pending_schema_install().await?; - db.ensure_schema_apply_idle("ensure_indices").await?; let txn = db.open_write_txn(branch).await?; let snapshot = txn.base.clone(); let mut pending_by_table = HashMap::>::new(); @@ -896,7 +893,6 @@ pub(super) async fn open_for_mutation_on_branch( op_kind: crate::db::MutationOpKind, txn: Option<&crate::db::WriteTxn>, ) -> Result { - db.ensure_schema_apply_not_locked("write").await?; // Source the resolved (snapshot, branch). With a `WriteTxn` the contract was // validated once at capture, so use the pinned base + resolved branch instead // of `resolved_branch_target` (which re-runs `ensure_schema_state_valid`). The @@ -1265,7 +1261,6 @@ pub(super) async fn commit_updates( db: &mut Omnigraph, updates: &[crate::db::DatasetUpdate], ) -> Result { - db.ensure_schema_apply_not_locked("write commit").await?; let current_branch = db .coordinator .read() @@ -1288,7 +1283,6 @@ pub(super) async fn commit_updates_on_branch_with_expected( txn: &crate::db::WriteTxn, lineage_intent: crate::db::manifest::LineageIntent, ) -> Result { - db.ensure_schema_apply_not_locked("write commit").await?; if branch != txn.branch.as_deref() { return Err(OmniError::manifest_internal( "publication branch differs from its captured write transaction", diff --git a/crates/omnigraph/src/db/schema_state.rs b/crates/omnigraph/src/db/schema_state.rs index 3bea72d02..dc3c11ca3 100644 --- a/crates/omnigraph/src/db/schema_state.rs +++ b/crates/omnigraph/src/db/schema_state.rs @@ -1,42 +1,24 @@ use std::collections::{BTreeMap, BTreeSet}; -use std::sync::Arc; use omnigraph_compiler::schema::parser::parse_persisted_schema_contract; use omnigraph_compiler::{ SchemaIR, SchemaIdentityDomain, SchemaShape, compile_schema_shape, schema_ir_hash, schema_ir_pretty_json, schema_shape_hash, schema_shape_hash_from_ir, validate_schema_ir, }; -use serde::{Deserialize, Serialize}; -use tracing::warn; +use serde::Deserialize; use crate::db::Snapshot; -use crate::db::manifest::{TableIdentity, table_path_for_identity}; +use crate::db::manifest::{ + SchemaContractHead, SchemaContractRow, TableIdentity, table_path_for_identity, +}; use crate::error::{OmniError, Result}; -use crate::storage::{StorageAdapter, join_uri}; - -pub(crate) const SCHEMA_SOURCE_FILENAME: &str = "_schema.pg"; -pub(crate) const SCHEMA_IR_FILENAME: &str = "_schema.ir.json"; -pub(crate) const SCHEMA_STATE_FILENAME: &str = "__schema_state.json"; -// Staging filenames used by atomic schema apply. Schema apply writes to these -// first, then commits the manifest, then installs the contract and retires the -// staging. A read-write open reconciles any leftover staging files against the -// manifest. -pub(crate) const SCHEMA_SOURCE_STAGING_FILENAME: &str = "_schema.pg.staging"; -pub(crate) const SCHEMA_IR_STAGING_FILENAME: &str = "_schema.ir.json.staging"; -pub(crate) const SCHEMA_STATE_STAGING_FILENAME: &str = "__schema_state.json.staging"; +pub(crate) const SCHEMA_IDENTITY_VERSION: u32 = 2; -const SCHEMA_STATE_FORMAT_VERSION: u32 = 2; -const SCHEMA_IDENTITY_VERSION: u32 = 2; - -/// Refuse a newer live or staged schema before open can perform recovery. -/// Only the version envelope is inspected here. Partial/malformed artifacts -/// remain subject to the existing recovery and complete contract validation; -/// this is not an alternate schema reader or an in-place format migration. -pub(crate) async fn refuse_unsupported_schema_versions( - root_uri: &str, - storage: &dyn StorageAdapter, -) -> Result<()> { +/// Refuse a newer schema contract before open builds anything from it. Only +/// the version envelope of the `schema_contract` row's IR text is inspected; +/// a partial or malformed row remains subject to the complete row validation. +pub(crate) fn refuse_unsupported_schema_versions(live_ir: &str) -> Result<()> { #[derive(Deserialize)] struct VersionEnvelope { ir_version: u32, @@ -48,276 +30,131 @@ pub(crate) async fn refuse_unsupported_schema_versions( features: BTreeSet, } - for filename in [SCHEMA_IR_FILENAME, SCHEMA_IR_STAGING_FILENAME] { - let Some(text) = storage - .read_text_if_exists(&join_uri(root_uri, filename)) - .await? - else { - continue; - }; - let Ok(envelope) = serde_json::from_str::(&text) else { - continue; - }; - if !omnigraph_compiler::is_supported_ir_version(envelope.ir_version) { - return Err(schema_lock_conflict(format!( - "unsupported ir_version {} in {filename} (supported {}, {} and {}); open will not recover or migrate this schema", - envelope.ir_version, - omnigraph_compiler::SCHEMA_IR_VERSION, - omnigraph_compiler::SCHEMA_IR_VERSION_EDGE_KEYS, - omnigraph_compiler::SCHEMA_IR_VERSION_FEATURES, - ))); - } - let Ok(envelope) = serde_json::from_str::(&text) else { - continue; - }; - if let Some(unknown) = envelope - .features - .iter() - .find(|name| !omnigraph_compiler::is_known_feature(name)) - { - return Err(schema_lock_conflict(format!( - "schema feature '{unknown}' in {filename} is unknown to this build; upgrade omnigraph before opening this graph; open will not recover or migrate this schema" - ))); - } + let Ok(envelope) = serde_json::from_str::(live_ir) else { + return Ok(()); + }; + if !omnigraph_compiler::is_supported_ir_version(envelope.ir_version) { + return Err(schema_lock_conflict(format!( + "unsupported ir_version {} in the schema_contract row (supported {}, {} and {}); open will not recover or migrate this schema", + envelope.ir_version, + omnigraph_compiler::SCHEMA_IR_VERSION, + omnigraph_compiler::SCHEMA_IR_VERSION_EDGE_KEYS, + omnigraph_compiler::SCHEMA_IR_VERSION_FEATURES, + ))); + } + let Ok(envelope) = serde_json::from_str::(live_ir) else { + return Ok(()); + }; + if let Some(unknown) = envelope + .features + .iter() + .find(|name| !omnigraph_compiler::is_known_feature(name)) + { + return Err(schema_lock_conflict(format!( + "schema feature '{unknown}' in the schema_contract row is unknown to this build; upgrade omnigraph before opening this graph; open will not recover or migrate this schema" + ))); } Ok(()) } -const MISSING_SCHEMA_CONTRACT_MESSAGE: &str = "graph is missing the mandatory identity-bearing schema contract (_schema.ir.json and __schema_state.json); automatic bootstrap is not supported"; -const INCOMPLETE_SCHEMA_CONTRACT_MESSAGE: &str = "graph schema contract is incomplete: _schema.ir.json and __schema_state.json must both be present"; -const INVALID_SCHEMA_IR_SUBJECT: &str = "accepted compiled schema contract"; -const INVALID_SCHEMA_STATE_SUBJECT: &str = "graph schema state"; - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct SchemaState { - pub(crate) format_version: u32, - pub(crate) schema_shape_hash: String, +/// The small fields of the `schema_contract` row of one `__manifest` version, +/// as the engine `Snapshot` exposes them: the identity every capture compares +/// and the key of `AcceptedCatalogMemo`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct SchemaContractIdentity { pub(crate) schema_ir_hash: String, pub(crate) schema_identity_version: u32, pub(crate) schema_identity_domain: String, - /// The graph commit that publishes this contract (RFC 0067). Schema apply - /// and the system-column upgrade stage the contract with the commit they - /// are about to publish; a read-write open promotes the staging when that - /// commit is in main's lineage and discards it otherwise. Init writes no - /// marker. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) publication: Option, } -/// The manifest publication a staged schema contract belongs to. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct SchemaPublication { - pub(crate) graph_commit_id: String, - pub(crate) parent_commit_id: Option, +impl From<&SchemaContractHead> for SchemaContractIdentity { + fn from(head: &SchemaContractHead) -> Self { + Self { + schema_ir_hash: head.schema_ir_hash.clone(), + schema_identity_version: head.schema_identity_version, + schema_identity_domain: head.schema_identity_domain.clone(), + } + } } -impl SchemaState { - fn from_ir(schema_ir: &SchemaIR) -> Result { +impl SchemaContractIdentity { + /// The identity a contract writer records for `schema_ir`: the IR + /// validated, hashed, and stamped with this build's identity version. + pub(crate) fn from_ir(schema_ir: &SchemaIR) -> Result { validate_schema_ir(schema_ir).map_err(|error| schema_lock_conflict(error.to_string()))?; Ok(Self { - format_version: SCHEMA_STATE_FORMAT_VERSION, - schema_shape_hash: schema_shape_hash_from_ir(schema_ir) - .map_err(|error| schema_lock_conflict(error.to_string()))?, schema_ir_hash: schema_ir_hash(schema_ir) .map_err(|error| schema_lock_conflict(error.to_string()))?, schema_identity_version: SCHEMA_IDENTITY_VERSION, schema_identity_domain: schema_ir.schema_identity_domain.as_str().to_string(), - publication: None, }) } -} - -pub(crate) async fn validate_schema_contract( - root_uri: &str, - storage: Arc, -) -> Result { - load_validated_schema_contract(root_uri, storage) - .await - .map(|(_, state)| state) -} - -/// Load the accepted IR and its schema identity from one validated contract -/// read. Mutation/load preparation carries the catalog built from this exact IR -/// beside the identity in its `WriteTxn`; consulting the handle-global catalog -/// would let a long-lived handle combine a newly observed schema token with an -/// older in-memory plan. -pub(crate) async fn load_validated_schema_contract( - root_uri: &str, - storage: Arc, -) -> Result<(SchemaIR, SchemaState)> { - let text = read_schema_contract_text(root_uri, storage.as_ref()).await?; - validate_schema_contract_text(&text) -} - -/// Validate the complete durable schema contract against source bytes already -/// captured under the root schema gate. Open and refresh use this form so the -/// source that populates the handle is exactly the source whose semantic shape -/// was checked against the accepted identity-bearing IR. -pub(crate) async fn load_validated_schema_contract_for_source( - root_uri: &str, - storage: Arc, - source: &str, -) -> Result<(SchemaIR, SchemaState)> { - let text = read_schema_contract_text_for_source(root_uri, storage.as_ref(), source.to_string()) - .await?; - validate_schema_contract_text(&text) -} - -/// The three durable schema-contract files as read, byte-exact. Memo key of -/// `ReadCaches::accepted_catalog` (see `AcceptedCatalogMemo`). -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct SchemaContractText { - pub(crate) source: String, - pub(crate) ir_json: String, - pub(crate) state_json: String, -} - -/// Read the schema contract's bytes: three `read_text` and two `exists` calls -/// (the per-query drift detection the lifecycle tests pin). A missing or -/// incomplete contract reports an unparseable source first; a storage error -/// from the existence probes is reported before a parse error, and so is a -/// failing IR or state read on the present-contract path. -pub(crate) async fn read_schema_contract_text( - root_uri: &str, - storage: &dyn StorageAdapter, -) -> Result { - let source = storage.read_text(&schema_source_uri(root_uri)).await?; - read_schema_contract_text_for_source(root_uri, storage, source).await -} -/// [`read_schema_contract_text`] over a source already read under the root -/// schema gate. -pub(crate) async fn read_schema_contract_text_for_source( - root_uri: &str, - storage: &dyn StorageAdapter, - source: String, -) -> Result { - match read_schema_contract(root_uri, storage).await? { - SchemaContractRead::Present { - ir_json, - state_json, - } => Ok(SchemaContractText { - source, - ir_json, - state_json, - }), - SchemaContractRead::MissingAll => { - compile_schema_source(&source)?; - Err(schema_lock_conflict(MISSING_SCHEMA_CONTRACT_MESSAGE)) - } - SchemaContractRead::PartialMissing => { - compile_schema_source(&source)?; - Err(schema_lock_conflict(INCOMPLETE_SCHEMA_CONTRACT_MESSAGE)) + fn head(&self) -> SchemaContractHead { + SchemaContractHead { + schema_ir_hash: self.schema_ir_hash.clone(), + schema_identity_version: self.schema_identity_version, + schema_identity_domain: self.schema_identity_domain.clone(), } } } -/// Validate contract bytes: source compiled, IR and state parsed, IR checked -/// against the state, source shape checked against the state. -pub(crate) fn validate_schema_contract_text( - text: &SchemaContractText, -) -> Result<(SchemaIR, SchemaState)> { - let current_source_shape = compile_schema_source(&text.source)?; - let (ir, state) = parse_schema_contract(&text.ir_json, &text.state_json)?; - validate_persisted_schema_contract(&ir, &state)?; - validate_current_source_matches(&state, ¤t_source_shape, &ir)?; - Ok((ir, state)) -} - -fn parse_schema_contract(ir_json: &str, state_json: &str) -> Result<(SchemaIR, SchemaState)> { - let ir = serde_json::from_str::(ir_json) - .map_err(|err| invalid_contract_file(INVALID_SCHEMA_IR_SUBJECT, SCHEMA_IR_FILENAME, err))?; - let state = serde_json::from_str::(state_json).map_err(|err| { - invalid_contract_file(INVALID_SCHEMA_STATE_SUBJECT, SCHEMA_STATE_FILENAME, err) - })?; - Ok((ir, state)) -} - -fn invalid_contract_file(subject: &str, file: &str, err: impl std::fmt::Display) -> OmniError { - schema_lock_conflict(format!("{subject} in {file} is invalid: {err}")) -} - -/// Read only the durable schema-identity marker. Schema apply promotes this -/// file after `_schema.pg` and `_schema.ir.json`, then releases its sentinel. -/// A capture path that already performed one full contract validation can use a -/// trailing marker read to detect the publish-before-promotion window without -/// paying for a second full source+IR parse. -pub(crate) async fn read_schema_state_identity( - root_uri: &str, - storage: &dyn StorageAdapter, -) -> Result { - let text = storage.read_text(&schema_state_uri(root_uri)).await?; - let state = serde_json::from_str::(&text).map_err(|err| { - invalid_contract_file(INVALID_SCHEMA_STATE_SUBJECT, SCHEMA_STATE_FILENAME, err) - })?; - validate_schema_state_envelope(&state)?; - Ok(state) +/// The identity of `snapshot`'s `schema_contract` row; a served version +/// always carries one (the stamp guard refuses older layouts at open). +pub(crate) fn snapshot_contract_identity(snapshot: &Snapshot) -> Result { + snapshot.schema_contract().ok_or_else(|| { + schema_lock_conflict(format!( + "__manifest version {} carries no schema_contract row", + snapshot.graph_manifest_version() + )) + }) } -/// The IR and state JSON of `schema_ir` as the contract writers put them on -/// the object store, with the state they encode. +/// The `schema_contract` row a publish writes for `schema_ir` and the `.pg` +/// text it was resolved from: the IR rendered as its exact stored text and +/// the identity of that IR. pub(crate) fn render_schema_contract( schema_ir: &SchemaIR, - publication: Option, -) -> Result<(SchemaState, String, String)> { - let ir_json = schema_ir_pretty_json(schema_ir) + source: &str, +) -> Result { + let ir = schema_ir_pretty_json(schema_ir) .map_err(|err| OmniError::manifest_internal(err.to_string()))?; - let mut state = SchemaState::from_ir(schema_ir)?; - state.publication = publication; - let state_json = serde_json::to_string_pretty(&state).map_err(|err| { - OmniError::manifest_internal(format!("serialize schema state error: {}", err)) - })?; - Ok((state, ir_json, state_json)) -} - -/// Write the IR and state of an already-rendered contract to their final -/// filenames; init keeps the same text to seed its accepted-catalog memo. -pub(crate) async fn write_schema_contract( - root_uri: &str, - storage: &dyn StorageAdapter, - contract: &SchemaContractText, -) -> Result<()> { - write_schema_contract_to( - storage, - &schema_ir_uri(root_uri), - &schema_state_uri(root_uri), - &contract.ir_json, - &contract.state_json, - ) - .await -} - -async fn write_schema_contract_to( - storage: &dyn StorageAdapter, - ir_uri: &str, - state_uri: &str, - ir_json: &str, - state_json: &str, -) -> Result<()> { - storage.write_text(ir_uri, ir_json).await?; - storage.write_text(state_uri, state_json).await + let identity = SchemaContractIdentity::from_ir(schema_ir)?; + Ok(SchemaContractRow { + source: source.to_string(), + ir, + head: identity.head(), + }) } -pub(crate) async fn read_accepted_schema_ir( - root_uri: &str, - storage: Arc, -) -> Result { - match read_schema_contract(root_uri, storage.as_ref()).await? { - SchemaContractRead::Present { - ir_json, - state_json, - } => { - let (ir, state) = parse_schema_contract(&ir_json, &state_json)?; - validate_persisted_schema_contract(&ir, &state)?; - Ok(ir) - } - SchemaContractRead::MissingAll => Err(schema_lock_conflict( - "graph is missing the mandatory identity-bearing schema contract; automatic bootstrap is not supported", - )), - SchemaContractRead::PartialMissing => Err(schema_lock_conflict( - "graph schema contract is incomplete: _schema.ir.json and __schema_state.json must both be present", - )), +/// Validate one `schema_contract` row: the IR parses as a valid +/// identity-bearing SchemaIR whose hash is the row's `schema_ir_hash`, the +/// identity envelope is this build's, and the source compiles to the IR's +/// semantic shape. +pub(crate) fn validate_schema_contract_row( + row: &SchemaContractRow, +) -> Result<(SchemaIR, SchemaContractIdentity)> { + let current_source_shape = compile_schema_source(&row.source)?; + let ir = serde_json::from_str::(&row.ir).map_err(|err| { + schema_lock_conflict(format!( + "accepted compiled schema contract in the schema_contract row is invalid: {err}" + )) + })?; + let identity = SchemaContractIdentity::from_ir(&ir)?; + validate_identity_envelope(&row.head)?; + if identity.schema_ir_hash != row.head.schema_ir_hash { + return Err(schema_lock_conflict(format!( + "schema_contract row schema_ir_hash {} is not the hash of its IR text ({}); accepted compiled schema does not match the recorded schema state", + row.head.schema_ir_hash, identity.schema_ir_hash + ))); } + if identity.schema_identity_domain != row.head.schema_identity_domain { + return Err(schema_lock_conflict( + "accepted compiled schema identity domain does not match the recorded schema state", + )); + } + validate_current_source_matches(&ir, ¤t_source_shape)?; + Ok((ir, identity)) } /// Prove that one accepted identity-bearing SchemaIR and one live manifest @@ -330,6 +167,18 @@ pub(crate) async fn read_accepted_schema_ir( pub(crate) fn validate_schema_ir_against_snapshot( schema_ir: &SchemaIR, snapshot: &Snapshot, +) -> Result<()> { + validate_schema_ir_against_entries( + schema_ir, + snapshot.datasets(), + snapshot.graph_manifest_version(), + ) +} + +pub(crate) fn validate_schema_ir_against_entries<'a>( + schema_ir: &SchemaIR, + entries: impl Iterator, + manifest_version: u64, ) -> Result<()> { validate_schema_ir(schema_ir).map_err(|error| { schema_manifest_conflict(format!("accepted SchemaIR is invalid: {error}")) @@ -378,28 +227,24 @@ pub(crate) fn validate_schema_ir_against_snapshot( } let mut manifest_by_identity = BTreeMap::::new(); - for entry in snapshot.datasets() { + for entry in entries { let Some((expected_identity, expected_path)) = expected_by_alias.get(&entry.type_key) else { return Err(schema_manifest_conflict(format!( "manifest v{} contains live table '{}' that is absent from accepted SchemaIR", - snapshot.graph_manifest_version(), - entry.type_key + manifest_version, entry.type_key ))); }; if entry.identity != *expected_identity { return Err(schema_manifest_conflict(format!( "manifest v{} table '{}' has identity {}, but accepted SchemaIR requires {}", - snapshot.graph_manifest_version(), - entry.type_key, - entry.identity, - expected_identity + manifest_version, entry.type_key, entry.identity, expected_identity ))); } if entry.dataset_path != *expected_path { return Err(schema_manifest_conflict(format!( "manifest v{} table '{}' has non-canonical path '{}', expected '{}' for identity {}", - snapshot.graph_manifest_version(), + manifest_version, entry.type_key, entry.dataset_path, expected_path, @@ -410,182 +255,77 @@ pub(crate) fn validate_schema_ir_against_snapshot( { return Err(schema_manifest_conflict(format!( "manifest v{} aliases '{previous}' and '{}' to the same table identity {}", - snapshot.graph_manifest_version(), - entry.type_key, - entry.identity + manifest_version, entry.type_key, entry.identity ))); } } for (table_key, (identity, _)) in expected_by_alias { - if snapshot.dataset(&table_key).is_none() { + if !manifest_by_identity.contains_key(&identity) { return Err(schema_manifest_conflict(format!( "accepted SchemaIR table '{table_key}' with identity {identity} is missing from manifest v{}", - snapshot.graph_manifest_version() + manifest_version ))); } } Ok(()) } -pub(crate) fn schema_source_uri(root_uri: &str) -> String { - join_uri(root_uri, SCHEMA_SOURCE_FILENAME) -} - -pub(crate) fn schema_ir_uri(root_uri: &str) -> String { - join_uri(root_uri, SCHEMA_IR_FILENAME) -} - -pub(crate) fn schema_state_uri(root_uri: &str) -> String { - join_uri(root_uri, SCHEMA_STATE_FILENAME) -} - -pub(crate) fn schema_source_staging_uri(root_uri: &str) -> String { - join_uri(root_uri, SCHEMA_SOURCE_STAGING_FILENAME) -} - -pub(crate) fn schema_ir_staging_uri(root_uri: &str) -> String { - join_uri(root_uri, SCHEMA_IR_STAGING_FILENAME) -} - -pub(crate) fn schema_state_staging_uri(root_uri: &str) -> String { - join_uri(root_uri, SCHEMA_STATE_STAGING_FILENAME) -} - -enum SchemaContractRead { - Present { ir_json: String, state_json: String }, - MissingAll, - PartialMissing, -} - -async fn read_schema_contract( - root_uri: &str, - storage: &dyn StorageAdapter, -) -> Result { - let ir_uri = schema_ir_uri(root_uri); - let state_uri = schema_state_uri(root_uri); - let ir_exists = storage.exists(&ir_uri).await?; - let state_exists = storage.exists(&state_uri).await?; - - match (ir_exists, state_exists) { - (false, false) => Ok(SchemaContractRead::MissingAll), - (true, true) => { - let ir_json = storage.read_text(&ir_uri).await?; - let state_json = storage.read_text(&state_uri).await?; - Ok(SchemaContractRead::Present { - ir_json, - state_json, - }) - } - _ => Ok(SchemaContractRead::PartialMissing), +/// The identity envelope every served contract carries: this build's +/// identity version and a well-formed identity domain. +pub(crate) fn validate_identity_envelope(head: &SchemaContractHead) -> Result<()> { + if head.schema_identity_version != SCHEMA_IDENTITY_VERSION { + return Err(schema_lock_conflict(format!( + "graph schema identity version {} is unsupported", + head.schema_identity_version + ))); } -} - -async fn read_schema_ir_at(storage: &dyn StorageAdapter, uri: &str) -> Result { - let text = storage.read_text(uri).await?; - serde_json::from_str::(&text).map_err(|error| { - schema_lock_conflict(format!( - "accepted compiled schema contract at '{uri}' is invalid: {error}" - )) - }) -} - -async fn read_schema_state_at(storage: &dyn StorageAdapter, uri: &str) -> Result { - let text = storage.read_text(uri).await?; - serde_json::from_str::(&text).map_err(|error| { - schema_lock_conflict(format!("graph schema state at '{uri}' is invalid: {error}")) - }) -} - -fn validate_persisted_schema_contract(ir: &SchemaIR, state: &SchemaState) -> Result<()> { - validate_schema_state_envelope(state)?; - validate_schema_ir(ir).map_err(|error| { - schema_lock_conflict(format!( - "accepted compiled schema is not a valid identity-bearing IR: {error}" - )) + SchemaIdentityDomain::parse(&head.schema_identity_domain).map_err(|error| { + schema_lock_conflict(format!("graph schema identity domain is invalid: {error}")) })?; - - let actual_hash = schema_ir_hash(ir).map_err(|err| schema_lock_conflict(err.to_string()))?; - if actual_hash != state.schema_ir_hash { - return Err(schema_lock_conflict( - "accepted compiled schema does not match the recorded schema state", - )); - } - - let projected_shape_hash = - schema_shape_hash_from_ir(ir).map_err(|err| schema_lock_conflict(err.to_string()))?; - if projected_shape_hash != state.schema_shape_hash { - return Err(schema_lock_conflict( - "accepted compiled schema's semantic projection does not match the recorded schema shape", - )); - } - - if ir.schema_identity_domain.as_str() != state.schema_identity_domain { - return Err(schema_lock_conflict( - "accepted compiled schema identity domain does not match the recorded schema state", - )); - } - Ok(()) } -fn validate_current_source_matches( - state: &SchemaState, - current_source_shape: &SchemaShape, +/// The `.pg` source must compile to the semantic shape of the accepted IR, +/// canonicalized at the IR's system-column vintage. +pub(crate) fn validate_current_source_matches( ir: &SchemaIR, + current_source_shape: &SchemaShape, ) -> Result<()> { let current_source_shape = current_source_shape.canonicalized_for_system_columns( omnigraph_compiler::system_columns_for_features(&ir.features), ); let current_hash = schema_shape_hash(¤t_source_shape) .map_err(|err| schema_lock_conflict(err.to_string()))?; - if current_hash != state.schema_shape_hash { + let accepted_hash = + schema_shape_hash_from_ir(ir).map_err(|err| schema_lock_conflict(err.to_string()))?; + if current_hash != accepted_hash { return Err(schema_lock_conflict( - "current _schema.pg no longer matches the accepted compiled schema", + "schema-contract source no longer matches the accepted compiled schema", )); } Ok(()) } -fn validate_schema_state_envelope(state: &SchemaState) -> Result<()> { - if state.format_version != SCHEMA_STATE_FORMAT_VERSION { - return Err(schema_lock_conflict(format!( - "graph schema state format {} is unsupported", - state.format_version - ))); - } - if state.schema_identity_version != SCHEMA_IDENTITY_VERSION { - return Err(schema_lock_conflict(format!( - "graph schema identity version {} is unsupported", - state.schema_identity_version - ))); - } - SchemaIdentityDomain::parse(&state.schema_identity_domain).map_err(|error| { - schema_lock_conflict(format!("graph schema identity domain is invalid: {error}")) - })?; - Ok(()) -} - -fn compile_schema_source(source: &str) -> Result { - // This source is already bound to the persisted identity-bearing IR and - // state hashes validated below. Use the compatibility parser so a v6 root - // admitted before v0.10 with body-level @unique(Blob) remains openable for - // inspection/export. Init and desired schema apply use the strict parser. +/// Compile a persisted `.pg` source with the compatibility parser, so a root +/// admitted before v0.10 with body-level @unique(Blob) remains openable; init +/// and the desired side of a schema apply use the strict parser. +pub(crate) fn compile_schema_source(source: &str) -> Result { let schema = parse_persisted_schema_contract(source).map_err(|err| { schema_lock_conflict(format!( - "current _schema.pg is not a valid accepted schema definition: {}", + "schema-contract source is not a valid accepted schema definition: {}", err )) })?; compile_schema_shape(&schema).map_err(|err| { schema_lock_conflict(format!( - "current _schema.pg could not be compiled into the accepted schema shape: {}", + "schema-contract source could not be compiled into the accepted schema shape: {}", err )) }) } -fn schema_lock_conflict(detail: impl Into) -> OmniError { +pub(crate) fn schema_lock_conflict(detail: impl Into) -> OmniError { OmniError::manifest_conflict(format!( "schema evolution is locked down in phase 1: {}; manual coordination is required", detail.into() @@ -598,317 +338,3 @@ fn schema_manifest_conflict(detail: impl Into) -> OmniError { detail.into() )) } - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum SchemaStateRecovery { - Noop, - Discarded, - Promoted, -} - -/// What the schema staging files on the object store are, judged against -/// main's lineage (RFC 0067): the staged state carries the graph commit that -/// publishes the contract, and that commit's presence in lineage is the only -/// authority for promoting or discarding the staging. -pub(crate) enum StagedContract { - /// No staging file exists. - None, - /// Staging files exist without the state file that closes the staging - /// write; the writer crashed before its contract staging was complete. - Incomplete, - /// A complete staging without a publication marker. No current writer - /// stages this way, so it is unowned and fails closed. - Unmarked, - /// A complete staging bound to one manifest publication. - Marked { - state: SchemaState, - /// The marker's commit is in main's lineage: the publication happened. - published: bool, - }, -} - -/// `thorough` also probes the source and IR staging files when the state -/// file is absent, to tell an incomplete staging from none; a caller that -/// only acts on a complete staging passes `false` and pays one probe. -pub(crate) async fn inspect_staged_contract( - root_uri: &str, - storage: &dyn StorageAdapter, - thorough: bool, -) -> Result { - let state_staging = schema_state_staging_uri(root_uri); - if !storage.exists(&state_staging).await? { - if !thorough { - return Ok(StagedContract::None); - } - let any_other = storage.exists(&schema_source_staging_uri(root_uri)).await? - || storage.exists(&schema_ir_staging_uri(root_uri)).await?; - return Ok(if any_other { - StagedContract::Incomplete - } else { - StagedContract::None - }); - } - let state = read_schema_state_at(storage, &state_staging).await?; - let Some(publication) = state.publication.clone() else { - return Ok(StagedContract::Unmarked); - }; - let (commits, _) = - crate::db::manifest::ManifestCoordinator::read_graph_lineage_at(root_uri, None).await?; - let published = match commits - .iter() - .find(|commit| commit.graph_commit_id == publication.graph_commit_id) - { - Some(commit) => { - if commit.parent_commit_id != publication.parent_commit_id - || commit.graph_branch.is_some() - { - return Err(schema_lock_conflict(format!( - "schema staging names graph commit '{}' with parent {:?}, but lineage records that commit with parent {:?} on branch {:?}", - publication.graph_commit_id, - publication.parent_commit_id, - commit.parent_commit_id, - commit.graph_branch - ))); - } - true - } - None => false, - }; - Ok(StagedContract::Marked { state, published }) -} - -/// Prove that a read-only open can pair its manifest snapshot with the live -/// schema contract without writing. A staged contract names the graph commit -/// that publishes it (RFC 0067): once that commit is in lineage the manifest -/// already carries the new table set, and only a read-write open may install -/// the contract files, so serving the old catalog over it would be -/// incoherent. An unpublished staging is inert garbage the reader ignores. -pub(crate) async fn ensure_read_only_schema_coherent( - root_uri: &str, - storage: &dyn StorageAdapter, -) -> Result<()> { - if let StagedContract::Marked { - state, - published: true, - } = inspect_staged_contract(root_uri, storage, false).await? - { - let graph_commit_id = state - .publication - .map(|publication| publication.graph_commit_id) - .unwrap_or_default(); - return Err(OmniError::recovery_required( - graph_commit_id.clone(), - format!( - "read-only open found SchemaApply manifest outcome for graph commit '{}' but the schema contract promotion is pending; run a read-write open to finish it", - graph_commit_id - ), - )); - } - Ok(()) -} - -/// What a read-write pass may do with schema staging it does not own. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum SchemaStagingPolicy { - /// The open-time pass: promote a published staging, discard an unpublished - /// one. Discarding relies on the one-mutation-process boundary: a live - /// apply in another process loses its staging, and its own promotion then - /// installs the contract from memory. - PromoteOrDiscard, - /// The write-entry pass and `refresh`: promote a published staging and - /// leave anything else alone, since the apply that wrote it may still be - /// live. - PromoteOnly, -} - -/// Promote or discard schema staging left by a crashed schema apply. -/// -/// RFC 0067: the staged `__schema_state.json.staging` names the graph commit -/// that publishes the contract. That commit in main's lineage means the -/// manifest already carries the new registrations, so the contract files must -/// follow (promotion is idempotent per file); its absence means the manifest -/// never moved, so the staging is garbage. -pub(crate) async fn recover_schema_state_files( - root_uri: &str, - storage: Arc, - snapshot: &Snapshot, - policy: SchemaStagingPolicy, -) -> Result { - let thorough = policy == SchemaStagingPolicy::PromoteOrDiscard; - match inspect_staged_contract(root_uri, storage.as_ref(), thorough).await? { - StagedContract::None => Ok(SchemaStateRecovery::Noop), - StagedContract::Incomplete => { - if policy == SchemaStagingPolicy::PromoteOnly { - return Ok(SchemaStateRecovery::Noop); - } - warn!( - "schema apply crashed while staging its contract; removing the incomplete staging (manifest v{})", - snapshot.graph_manifest_version() - ); - cleanup_staging_files(root_uri, storage.as_ref()).await?; - Ok(SchemaStateRecovery::Discarded) - } - // Every writer marks its staging with the publishing commit, so an - // unmarked one comes from a build that predates RFC 0067 or from - // manual edits; neither can be judged here. - StagedContract::Unmarked => Err(schema_lock_conflict(format!( - "found complete schema staging files without a publication marker; inspect _schema.pg.staging against _schema.pg and remove the staging files to keep the live schema (manifest v{})", - snapshot.graph_manifest_version() - ))), - StagedContract::Marked { state, published } => { - if published { - warn!( - "schema apply crashed after publishing graph commit {}; promoting its staged contract (manifest v{})", - state - .publication - .as_ref() - .map(|publication| publication.graph_commit_id.as_str()) - .unwrap_or("?"), - snapshot.graph_manifest_version() - ); - promote_exact_schema_staging(root_uri, storage.as_ref(), &state.schema_ir_hash) - .await?; - return Ok(SchemaStateRecovery::Promoted); - } - if policy == SchemaStagingPolicy::PromoteOnly { - return Ok(SchemaStateRecovery::Noop); - } - warn!( - "schema apply crashed before publishing; removing its staged contract and keeping the live schema (manifest v{})", - snapshot.graph_manifest_version() - ); - cleanup_staging_files(root_uri, storage.as_ref()).await?; - Ok(SchemaStateRecovery::Discarded) - } - } -} - -pub(crate) async fn cleanup_staging_files( - root_uri: &str, - storage: &dyn StorageAdapter, -) -> Result<()> { - storage.delete(&schema_source_staging_uri(root_uri)).await?; - storage.delete(&schema_ir_staging_uri(root_uri)).await?; - storage.delete(&schema_state_staging_uri(root_uri)).await?; - Ok(()) -} - -pub(crate) async fn complete_staging_rename( - root_uri: &str, - storage: &dyn StorageAdapter, -) -> Result<()> { - // Each rename is independent and idempotent: if the source no longer - // exists (already renamed) we skip it. This handles partial-rename - // recovery (e.g. one file renamed before crash). - rename_if_present( - storage, - &schema_source_staging_uri(root_uri), - &schema_source_uri(root_uri), - ) - .await?; - rename_if_present( - storage, - &schema_ir_staging_uri(root_uri), - &schema_ir_uri(root_uri), - ) - .await?; - rename_if_present( - storage, - &schema_state_staging_uri(root_uri), - &schema_state_uri(root_uri), - ) - .await?; - Ok(()) -} - -/// Verify that every durable staging artifact belongs to one exact -/// SchemaApply target. A partial promotion is accepted only when the live -/// identity already equals that same target; mismatched files are never -/// renamed or deleted on another apply's behalf. -pub(crate) async fn validate_exact_schema_staging_target( - root_uri: &str, - storage: &dyn StorageAdapter, - target_schema_ir_hash: &str, -) -> Result<()> { - let pg_staging = schema_source_staging_uri(root_uri); - let ir_staging = schema_ir_staging_uri(root_uri); - let state_staging = schema_state_staging_uri(root_uri); - let pg_exists = storage.exists(&pg_staging).await?; - let ir_exists = storage.exists(&ir_staging).await?; - let state_exists = storage.exists(&state_staging).await?; - // Promotion renames source -> IR -> state independently. A crash between - // any two renames therefore leaves a mixture of live and staging files. - // Verify each artifact at whichever location currently owns it instead of - // using the state marker as a proxy for the whole three-file contract. - let source_uri = if pg_exists { - pg_staging - } else { - schema_source_uri(root_uri) - }; - let source = storage.read_text(&source_uri).await?; - let source_shape = compile_schema_source(&source)?; - - let ir_uri = if ir_exists { - ir_staging - } else { - schema_ir_uri(root_uri) - }; - let ir = read_schema_ir_at(storage, &ir_uri).await?; - let ir_hash = schema_ir_hash(&ir).map_err(|error| schema_lock_conflict(error.to_string()))?; - if ir_hash != target_schema_ir_hash { - return Err(schema_lock_conflict(format!( - "compiled schema at '{}' does not belong to the exact SchemaApply intent", - ir_uri - ))); - } - - let state_uri = if state_exists { - state_staging - } else { - schema_state_uri(root_uri) - }; - let state = read_schema_state_at(storage, &state_uri).await?; - validate_persisted_schema_contract(&ir, &state)?; - validate_current_source_matches(&state, &source_shape, &ir)?; - if state.schema_ir_hash != target_schema_ir_hash { - return Err(schema_lock_conflict(format!( - "schema identity at '{}' does not belong to the exact SchemaApply intent", - state_uri - ))); - } - Ok(()) -} - -pub(crate) async fn promote_exact_schema_staging( - root_uri: &str, - storage: &dyn StorageAdapter, - target_schema_ir_hash: &str, -) -> Result<()> { - validate_exact_schema_staging_target(root_uri, storage, target_schema_ir_hash).await?; - complete_staging_rename(root_uri, storage).await?; - let live_source = storage.read_text(&schema_source_uri(root_uri)).await?; - let live_shape = compile_schema_source(&live_source)?; - let live_ir = read_schema_ir_at(storage, &schema_ir_uri(root_uri)).await?; - let live = read_schema_state_at(storage, &schema_state_uri(root_uri)).await?; - validate_persisted_schema_contract(&live_ir, &live)?; - validate_current_source_matches(&live, &live_shape, &live_ir)?; - let live_ir_hash = - schema_ir_hash(&live_ir).map_err(|error| schema_lock_conflict(error.to_string()))?; - if live.schema_ir_hash != target_schema_ir_hash || live_ir_hash != target_schema_ir_hash { - return Err(schema_lock_conflict( - "exact SchemaApply promotion completed without the intended live identity", - )); - } - Ok(()) -} - -async fn rename_if_present( - storage: &dyn StorageAdapter, - from_uri: &str, - to_uri: &str, -) -> Result<()> { - if storage.exists(from_uri).await? { - storage.rename_text(from_uri, to_uri).await?; - } - Ok(()) -} diff --git a/crates/omnigraph/src/db/snapshot.rs b/crates/omnigraph/src/db/snapshot.rs index e29ed79c0..54df63377 100644 --- a/crates/omnigraph/src/db/snapshot.rs +++ b/crates/omnigraph/src/db/snapshot.rs @@ -10,6 +10,7 @@ use lance_table::format::IndexMetadata; use omnigraph_compiler::catalog::Catalog; use crate::db::DatasetEntry; +use crate::db::schema_state::SchemaContractIdentity; use crate::error::Result; /// Immutable point-in-time view of the database. @@ -193,6 +194,10 @@ impl SnapshotDataset { } impl Snapshot { + pub(crate) fn same_manifest_image(&self, other: &Self) -> bool { + self.inner.same_manifest_image(&other.inner) + } + pub(crate) fn wrap(inner: omnigraph_catalog::Snapshot) -> Self { Self { inner } } @@ -207,6 +212,18 @@ impl Snapshot { &mut self.inner } + /// Load the contract at this snapshot's captured native branch and version. + pub(crate) async fn read_schema_contract( + &self, + root_uri: &str, + ) -> Result { + omnigraph_catalog::ManifestCoordinator::read_schema_contract_for_snapshot( + root_uri, + &self.inner, + ) + .await + } + pub(crate) fn graph_branch(&self) -> Option<&str> { self.inner.graph_branch() } @@ -225,6 +242,15 @@ impl Snapshot { self.inner.graph_head(branch) } + /// The identity of the `schema_contract` row in this snapshot's own pinned + /// graph-manifest version; `None` only for a version written before the + /// row existed. + pub(crate) fn schema_contract(&self) -> Option { + self.inner + .schema_contract() + .map(SchemaContractIdentity::from) + } + pub(crate) fn bind_catalog_aliases(&mut self, catalog: &Catalog) -> Result<()> { self.inner.bind_catalog_aliases(catalog) } diff --git a/crates/omnigraph/src/db/upgrade.rs b/crates/omnigraph/src/db/upgrade.rs index b015bbc46..03ef234bf 100644 --- a/crates/omnigraph/src/db/upgrade.rs +++ b/crates/omnigraph/src/db/upgrade.rs @@ -28,6 +28,7 @@ use crate::seams::{decide_seam, fail}; #[path = "upgrade/detached_only.rs"] mod detached_only; +pub(crate) mod legacy_schema_files; pub(crate) mod legacy_sidecars; #[cfg(all(test, feature = "failpoints"))] @@ -41,13 +42,44 @@ const HANDLER: &str = "registration-clocks-v6-to-v7"; const RETIREMENT_HANDLER: &str = "native-retirement-v7-to-v8"; const DETACHED_PINS_HANDLER: &str = "detached-pins-v8-v9-to-v10"; const DETACHED_ONLY_HANDLER: &str = "detached-only-v10-to-v11"; -const DEFAULT_TARGET: u32 = 11; +const SCHEMA_CONTRACT_HANDLER: &str = "schema-contract-v11-v12-to-v13"; +const DEFAULT_TARGET: u32 = 13; const RETIREMENT_KEY: &str = "omnigraph.retired_manifest_branch"; const MAX_VERSIONS: usize = 100_000; const MAX_APPENDED_UPGRADE_VERSIONS: u64 = 3; const MAX_ROWS: usize = 1_000_000; const MAX_METADATA_BYTES: usize = 64 * 1024 * 1024; +/// Prepare a flat-v11 main manifest and return its legacy source, IR, and state text. +/// The DST harness persists these files before installing or enabling read weather. +#[cfg(feature = "dst")] +#[doc(hidden)] +pub async fn dst_prepare_legacy_upgrade_fixture(root: &str) -> Result<(String, String, String)> { + let mut main = Dataset::open(&format!("{root}/__manifest")) + .await + .map_err(OmniError::storage)?; + let row = crate::db::manifest::ManifestCoordinator::read_schema_contract_at( + root, + None, + main.version().version, + ) + .await?; + let ir: omnigraph_compiler::SchemaIR = + serde_json::from_str(&row.ir).map_err(|error| invalid(error.to_string()))?; + let shape_hash = omnigraph_compiler::schema_shape_hash_from_ir(&ir) + .map_err(|error| invalid(error.to_string()))?; + let state = serde_json::json!({ + "format_version": 2, + "schema_shape_hash": shape_hash, + "schema_ir_hash": row.head.schema_ir_hash, + "schema_identity_version": row.head.schema_identity_version, + "schema_identity_domain": row.head.schema_identity_domain, + }) + .to_string(); + crate::db::manifest::migrations::restamp_flat_for_test(&mut main, 11).await?; + Ok((row.source, row.ir, state)) +} + /// Explicit storage conversion options. Execution requires exclusive operator control. #[derive(Debug, Clone, Copy, Default)] pub struct UpgradeOptions { @@ -199,13 +231,13 @@ async fn run( policy: Option<&dyn omnigraph_policy::PolicyChecker>, report: &mut UpgradeReport, ) -> Result<()> { - if !matches!(report.target_format, 7 | 8 | 10 | 11) { + if !matches!(report.target_format, 7 | 8 | 10 | 11 | 13) { if let Ok(main) = open(root, None).await { report.observed_format = read_stamp(&main); } report.finding( "unsupported_target", - "this binary has registered routes to formats 7, 8, 10 and 11 only; v9 was the system-column vintage, which since v10 is converted on a served graph by `omnigraph schema upgrade-system-columns`", + "this binary has registered routes to formats 7, 8, 10, 11 and 13 only; v9 was the system-column vintage, which since v10 is converted on a served graph by `omnigraph schema upgrade-system-columns`", ); return Ok(()); } @@ -214,7 +246,7 @@ async fn run( } else { Some(crate::db::reserve_export_root_exclusion(root)?) }; - for _ in 0..4 { + for _ in 0..5 { let main = open(root, None).await?; let stamp = read_stamp(&main); if report.observed_format.is_none() { @@ -252,7 +284,8 @@ async fn run( Some(6) => 7, Some(7) => report.target_format.min(8), Some(8) | Some(9) => report.target_format.min(10), - Some(stamp) if stamp >= 10 => report.target_format, + Some(10) => report.target_format.min(11), + Some(stamp) if stamp >= 11 => report.target_format, _ => report.target_format.min(8), }); let initial_step = pending @@ -274,7 +307,7 @@ async fn run( report.route.push(DETACHED_PINS_HANDLER.into()); } if matches!(initial_step, 6..=10) - && report.target_format == 11 + && report.target_format >= 11 && !report .route .iter() @@ -282,6 +315,15 @@ async fn run( { report.route.push(DETACHED_ONLY_HANDLER.into()); } + if matches!(initial_step, 6..=12) + && report.target_format == 13 + && !report + .route + .iter() + .any(|entry| entry == SCHEMA_CONTRACT_HANDLER) + { + report.route.push(SCHEMA_CONTRACT_HANDLER.into()); + } run_step(root, options, actor, policy, report, step_target).await?; if options.check { if step_target < report.target_format && report.success() { @@ -291,9 +333,12 @@ async fn run( if step_target < 10 && report.target_format >= 10 { report.work.deferred_checks.insert("the v10 stamp step validates the converted v8 output's live branches before it has effects".into()); } - if report.target_format == 11 { + if step_target < 11 && report.target_format >= 11 { report.work.deferred_checks.insert("the v11 step judges the converted v10 output's pins on every live branch before it has effects".into()); } + if report.target_format == 13 { + report.work.deferred_checks.insert("the v13 step validates legacy contract bytes and each branch's v11/v12 layout before it has effects".into()); + } } return Ok(()); } @@ -303,7 +348,7 @@ async fn run( report.work.deferred_checks.clear(); } Err(invalid( - "storage upgrade route exceeded its four registered steps", + "storage upgrade route exceeded its five registered steps", )) } @@ -323,9 +368,48 @@ decide_seam! { pub static UPGRADE_AFTER_ACTIVATION = ("upgrade.after_activation", Unreachable, [Fail]); } +decide_seam! { + pub static UPGRADE_AFTER_SCHEMA_FILE_DELETE = ("upgrade.after_schema_file_delete", Unreachable, [Fail]); +} + +fn refuse_legacy_sentinel(native: &str) -> Result<()> { + if crate::branch_names::logical_branch_name(native).trim_start_matches('/') + == "__schema_apply_lock__" + { + return Err(invalid( + "legacy schema-apply sentinel requires the source-compatible executable before conversion", + )); + } + Ok(()) +} + +async fn validated_manifest_contract( + dataset: &Dataset, +) -> Result { + let row = crate::db::manifest::migrations::read_upgrade_schema_contract(dataset).await?; + crate::db::schema_state::refuse_unsupported_schema_versions(&row.ir)?; + let (ir, _) = crate::db::schema_state::validate_schema_contract_row(&row)?; + let state = read_manifest_state(dataset).await?; + crate::db::schema_state::validate_schema_ir_against_entries( + &ir, + state.entries.iter(), + state.version, + )?; + Ok(row) +} + +async fn validate_converted_contract(dataset: &Dataset, intent: &UpgradeIntent) -> Result<()> { + let contract = validated_manifest_contract(dataset).await?; + intent + .schema_contract + .as_ref() + .ok_or_else(|| invalid("schema-contract upgrade has no exact contract ownership"))? + .validate_row(&contract) +} + /// Whether a live branch at `branch_stamp` belongs to a current graph whose -/// main is at `main_stamp`: any served stamp when main is served (a v11 graph -/// converts to v12 one branch per publish), else main's exact stamp. +/// main is at `main_stamp`: a served stamp when main is served, otherwise +/// main's exact intermediate target stamp. fn branch_stamp_is_current(main_stamp: u32, branch_stamp: Option) -> bool { if is_served_stamp(main_stamp) { branch_stamp.is_some_and(is_served_stamp) @@ -357,10 +441,7 @@ async fn run_step( "an owned storage conversion requires explicit recovery", ); } - let storage = storage_for_uri(root)?; - let (_, schema_state) = - crate::db::schema_state::load_validated_schema_contract(root, Arc::clone(&storage)).await?; - report.graph_identity = Some(schema_state.schema_identity_domain.clone()); + let storage = crate::storage::decorate(storage_for_uri(root)?); let sidecars = crate::db::upgrade::legacy_sidecars::pending_legacy_sidecars(root, storage.as_ref()) .await?; @@ -390,6 +471,26 @@ async fn run_step( crate::db::manifest::migrations::guard_stamp(&main)?; } read_manifest_state(&main).await?; + let current_contract = if expected == 13 { + Some(validated_manifest_contract(&main).await?) + } else { + legacy_schema_files::refuse_staging(root, storage.as_ref()).await?; + None + }; + let legacy = if current_contract.is_none() { + Some( + legacy_schema_files::load_validated_schema_contract(root, Arc::clone(&storage)) + .await? + .into_row(), + ) + } else { + None + }; + let contract = current_contract + .as_ref() + .or(legacy.as_ref()) + .ok_or_else(|| invalid("current graph has no schema contract"))?; + report.graph_identity = Some(contract.head.schema_identity_domain.clone()); let branches = if expected >= 8 { crate::branch_control::list_live_manifest_branch_contents(&main).await? } else { @@ -400,16 +501,14 @@ async fn run_step( } let mut logical_names = HashSet::from(["main".to_string()]); for native in branches.keys() { + if expected < 13 { + refuse_legacy_sentinel(native)?; + } if !logical_names.insert(crate::branch_names::logical_branch_name(native).to_string()) { return Err(invalid( "current graph contains duplicate live logical branch names", )); } - if crate::db::is_internal_system_branch(native) { - return Err(invalid( - "resolve internal branch recovery before storage upgrade", - )); - } let branch = main .checkout_branch(native) .await @@ -424,6 +523,11 @@ async fn run_step( ))); } read_manifest_state(&branch).await?; + if let Some(contract) = ¤t_contract { + if validated_manifest_contract(&branch).await? != *contract { + return Err(invalid("current branch schema contract differs from main")); + } + } } report.outcome = if report.completed_handlers.is_empty() { UpgradeOutcome::AlreadyCurrent @@ -462,27 +566,66 @@ async fn run_step( if pending.is_none() && !matches!( (read_stamp(&main), step_target), - (Some(6), 7) | (Some(7), 8) | (Some(8), 10) | (Some(9), 10) | (Some(10), 11) + (Some(6), 7) + | (Some(7), 8) + | (Some(8), 10) + | (Some(9), 10) + | (Some(10), 11) + | (Some(11), 13) + | (Some(12), 13) ) { - report.finding("unsupported_source", "only validated v6, v7, v8, v9 and v10 graphs have conversion handlers; preserve the source and use its executable for export/import"); + report.finding("unsupported_source", "only validated v6, v7, v8, v9, v10, v11 and v12 graphs have conversion handlers; preserve the source and use its executable for export/import"); return Ok(()); } let resumed = pending.is_some(); + legacy_schema_files::refuse_staging(root, storage.as_ref()).await?; + let contract = if let Some(intent) = pending.as_ref().filter(|intent| intent.protocol == 5) { + let mut complete = true; + for branch in &intent.branches { + let current = open(root, branch.native.as_deref()).await?; + if branch_completed(¤t, branch, intent)? { + validate_converted_contract(¤t, intent).await?; + } else { + complete = false; + } + } + if complete { + verify_inventory(root, intent, true).await?; + validated_manifest_contract(&main).await? + } else { + legacy_schema_files::load_validated_schema_contract(root, Arc::clone(&storage)) + .await? + .into_row() + } + } else { + legacy_schema_files::load_validated_schema_contract(root, Arc::clone(&storage)) + .await? + .into_row() + }; + report.graph_identity = Some(contract.head.schema_identity_domain.clone()); let mut intent = match pending { Some(intent) => { - if intent.graph_identity != schema_state.schema_identity_domain { + if intent.graph_identity != contract.head.schema_identity_domain { return Err(invalid("upgrade schema identity changed")); } + if let Some(expected) = &intent.schema_contract { + expected.validate_row(&contract)?; + } intent } - None => inventory(&main, schema_state.schema_identity_domain.clone()).await?, + None => inventory(&main, contract.head.schema_identity_domain.clone()).await?, }; + if intent.protocol == 5 && intent.schema_contract.is_none() { + intent.schema_contract = + Some(crate::db::manifest::migrations::UpgradeSchemaContract::from_row(&contract)); + } let handler = match intent.source_format { 6 => HANDLER, 7 => RETIREMENT_HANDLER, 8 | 9 => DETACHED_PINS_HANDLER, - _ => DETACHED_ONLY_HANDLER, + 10 => DETACHED_ONLY_HANDLER, + _ => SCHEMA_CONTRACT_HANDLER, }; if !report.route.iter().any(|entry| entry == handler) { report.route.push(handler.into()); @@ -510,9 +653,9 @@ async fn run_step( } if handler == DETACHED_ONLY_HANDLER && !resumed { let blocked = if options.check { - detached_only::preflight(root).await? + detached_only::preflight(root, &contract).await? } else { - let work = detached_only::execute(root).await?; + let work = detached_only::execute(root, &contract).await?; tracing::info!( promoted = work.promoted, reaped = work.reaped, @@ -530,11 +673,11 @@ async fn run_step( } if !options.check { let main = open(root, None).await?; - intent = inventory(&main, schema_state.schema_identity_domain.clone()).await?; + intent = inventory(&main, contract.head.schema_identity_domain.clone()).await?; } } verify_inventory(root, &intent, report.recovery.is_some()).await?; - preflight(root, &intent, &mut report.work).await?; + preflight(root, &intent, &contract, &mut report.work).await?; if options.check { if report.recovery.is_none() { report.outcome = UpgradeOutcome::CheckPassed; @@ -575,7 +718,7 @@ async fn run_step( .checkout_version(branch.version) .await .map_err(OmniError::storage)?; - publish_conversion(current, source, branch, &intent).await?; + publish_conversion(current, source, branch, &intent, &contract).await?; report.last_durable_completed_boundary = Some(format!( "converted:{}", branch.native.as_deref().unwrap_or("main") @@ -592,12 +735,19 @@ async fn run_step( .await .map_err(OmniError::storage)?; equivalent(&source, ¤t).await?; + if intent.protocol == 5 { + validate_converted_contract(¤t, &intent).await?; + } } verify_inventory(root, &intent, true).await?; let main = open(root, None).await?; if intent_from(&main)?.as_ref() != Some(&intent) { return Err(invalid("activation ownership changed")); } + if intent.protocol == 5 { + legacy_schema_files::cleanup(root, storage.as_ref(), &contract).await?; + verify_inventory(root, &intent, true).await?; + } fail(&UPGRADE_BEFORE_ACTIVATION)?; publish_activation(main).await?; report.last_durable_completed_boundary = Some("activated".into()); @@ -648,11 +798,7 @@ async fn inventory(main: &Dataset, graph_identity: String) -> Result Result 1, 7 => 2, 8 | 9 => 3, - _ => 4, + 10 => 4, + _ => 5, }, attempt: ulid::Ulid::new().to_string(), source_format, @@ -685,10 +832,12 @@ async fn inventory(main: &Dataset, graph_identity: String) -> Result 7, 7 => 8, 8 | 9 => 10, + 10 => 11, _ => DEFAULT_TARGET, }, graph_identity, branches: sources, + schema_contract: None, }) } @@ -705,13 +854,15 @@ fn verify_source_head( if dataset.version().version != expected { return Err(invalid("source branch changed; refusing foreign movement")); } - if read_stamp(dataset) - != Some(if fenced { - intent.target_format - } else { - intent.source_format - }) - { + let stamp = read_stamp(dataset); + let valid_stamp = if fenced { + stamp == Some(intent.target_format) + } else if intent.protocol == 5 { + matches!(stamp, Some(11 | 12)) + } else { + stamp == Some(intent.source_format) + }; + if !valid_stamp { return Err(invalid("source branch has an incompatible format")); } Ok(()) @@ -775,7 +926,12 @@ fn confined(root: &str, dataset: &Dataset) -> Result<()> { Ok(()) } -async fn preflight(root: &str, intent: &UpgradeIntent, work: &mut UpgradeWork) -> Result<()> { +async fn preflight( + root: &str, + intent: &UpgradeIntent, + contract: &crate::db::manifest::SchemaContractRow, + work: &mut UpgradeWork, +) -> Result<()> { let mut refs = HashSet::new(); let mut logical_names = HashSet::from(["main"]); for native in intent @@ -797,7 +953,11 @@ async fn preflight(root: &str, intent: &UpgradeIntent, work: &mut UpgradeWork) - .checkout_version(branch.version) .await .map_err(OmniError::storage)?; - if read_stamp(&source) != Some(intent.source_format) { + if if intent.protocol == 5 { + !matches!(read_stamp(&source), Some(11 | 12)) + } else { + read_stamp(&source) != Some(intent.source_format) + } { return Err(invalid("source history has an unsupported format")); } if source @@ -814,6 +974,19 @@ async fn preflight(root: &str, intent: &UpgradeIntent, work: &mut UpgradeWork) - validate_metadata_budget(&source).await?; let (old, lineage) = crate::db::manifest::state::read_manifest_state_and_lineage(&source).await?; + if intent.protocol == 5 { + if old.schema_contract.is_some() { + return Err(invalid( + "legacy source unexpectedly carries a schema contract row", + )); + } + let (ir, _) = crate::db::schema_state::validate_schema_contract_row(contract)?; + crate::db::schema_state::validate_schema_ir_against_entries( + &ir, + old.entries.iter(), + old.version, + )?; + } if intent.source_format == 6 { validate_source_branch_identity(branch, &old, &lineage)?; } else if branch.native.is_some() @@ -828,7 +1001,9 @@ async fn preflight(root: &str, intent: &UpgradeIntent, work: &mut UpgradeWork) - } let schema: Schema = source.schema().into(); let expected = flat_manifest_schema(); - if schema.fields().len() != expected.fields().len() + if intent.protocol == 5 { + crate::db::manifest::migrations::validate_schema_contract_source(&source)?; + } else if schema.fields().len() != expected.fields().len() || schema .fields() .iter() @@ -1134,6 +1309,7 @@ async fn publish_conversion( source: Dataset, branch: &SourceBranch, intent: &UpgradeIntent, + contract: &crate::db::manifest::SchemaContractRow, ) -> Result<()> { let mut metadata = source.schema().metadata.clone(); metadata.insert( @@ -1152,7 +1328,7 @@ async fn publish_conversion( serde_json::to_string(&receipt(branch, intent)).map_err(|e| invalid(e.to_string()))?, ); let destination = Arc::new(current); - let transaction = if intent.source_format != 6 { + let transaction = if intent.source_format != 6 && intent.protocol != 5 { let operation = Operation::UpdateConfig { config_updates: None, table_metadata_updates: None, @@ -1167,30 +1343,44 @@ async fn publish_conversion( }; Transaction::new(destination.version().version, operation, None) } else { - let schema: Schema = source.schema().into(); - let schema = Arc::new(schema.with_metadata(metadata)); - let mut scan = source.scan(); - let mut columns: Vec = schema - .fields() - .iter() - .map(|field| field.name().clone()) - .collect(); - columns.push("_row_last_updated_at_version".into()); - scan.project(&columns).map_err(OmniError::storage)?; - scan.batch_size(1024); - let batches = scan.try_into_stream().await.map_err(OmniError::storage)?; - let output_schema = Arc::clone(&schema); - let mut seen = HashSet::new(); - let version = source.version().version; - let converted = batches - .map_err(datafusion::error::DataFusionError::from) - .and_then(move |batch| { - let result = convert_batch(batch, Arc::clone(&output_schema), version, &mut seen) - .map_err(|error| datafusion::error::DataFusionError::External(Box::new(error))); - futures::future::ready(result) - }); - let stream: datafusion::physical_plan::SendableRecordBatchStream = - Box::pin(RecordBatchStreamAdapter::new(schema, converted)); + let stream: datafusion::physical_plan::SendableRecordBatchStream = if intent.protocol == 5 { + let (schema, batches) = + crate::db::manifest::migrations::schema_contract_conversion_batches( + &source, metadata, contract, + ) + .await?; + Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::iter(batches.into_iter().map(Ok)), + )) + } else { + let schema: Schema = source.schema().into(); + let schema = Arc::new(schema.with_metadata(metadata)); + let mut scan = source.scan(); + let mut columns: Vec = schema + .fields() + .iter() + .map(|field| field.name().clone()) + .collect(); + columns.push("_row_last_updated_at_version".into()); + scan.project(&columns).map_err(OmniError::storage)?; + scan.batch_size(1024); + let batches = scan.try_into_stream().await.map_err(OmniError::storage)?; + let output_schema = Arc::clone(&schema); + let mut seen = HashSet::new(); + let version = source.version().version; + let converted = batches + .map_err(datafusion::error::DataFusionError::from) + .and_then(move |batch| { + let result = + convert_batch(batch, Arc::clone(&output_schema), version, &mut seen) + .map_err(|error| { + datafusion::error::DataFusionError::External(Box::new(error)) + }); + futures::future::ready(result) + }); + Box::pin(RecordBatchStreamAdapter::new(schema, converted)) + }; let params = WriteParams { mode: WriteMode::Overwrite, enable_stable_row_ids: true, @@ -1218,7 +1408,11 @@ async fn publish_conversion( "storage conversion publication did not carry its receipt", )); } - equivalent(&source, &target).await + equivalent(&source, &target).await?; + if intent.protocol == 5 { + validate_converted_contract(&target, intent).await?; + } + Ok(()) } fn convert_batch( diff --git a/crates/omnigraph/src/db/upgrade/detached_only.rs b/crates/omnigraph/src/db/upgrade/detached_only.rs index 1da54922e..0aec36ef6 100644 --- a/crates/omnigraph/src/db/upgrade/detached_only.rs +++ b/crates/omnigraph/src/db/upgrade/detached_only.rs @@ -35,17 +35,27 @@ pub(super) struct DetachedOnlyWork { } /// Judge every pin of every live branch without writing. -pub(super) async fn preflight(root: &str) -> Result> { +pub(super) async fn preflight( + root: &str, + contract: &crate::db::manifest::SchemaContractRow, +) -> Result> { let _admission = crate::db::manifest::migrations::admit_conversion_source(root, SOURCE_STAMP); - let db = Omnigraph::open_read_only(root).await?; + let db = + Omnigraph::open_for_storage_upgrade(root, crate::db::OpenMode::ReadOnly, contract.clone()) + .await?; blocked_pins(&db).await } /// Promote, reap and record on every live branch; refuse before any write /// when a pin is blocked. -pub(super) async fn execute(root: &str) -> Result { +pub(super) async fn execute( + root: &str, + contract: &crate::db::manifest::SchemaContractRow, +) -> Result { let _admission = crate::db::manifest::migrations::admit_conversion_source(root, SOURCE_STAMP); - let db = Omnigraph::open(root).await?; + let db = + Omnigraph::open_for_storage_upgrade(root, crate::db::OpenMode::ReadWrite, contract.clone()) + .await?; let mut work = DetachedOnlyWork { blocked: blocked_pins(&db).await?, ..DetachedOnlyWork::default() @@ -135,15 +145,7 @@ pub(super) async fn execute(root: &str) -> Result { } async fn live_branches(db: &Omnigraph) -> Result>> { - Ok(crate::db::omnigraph::optimize::cleanup_graph_branches(db) - .await? - .into_iter() - .filter(|branch| { - branch - .as_deref() - .is_none_or(|branch| !crate::db::is_internal_system_branch(branch)) - }) - .collect()) + crate::db::omnigraph::optimize::cleanup_graph_branches(db).await } /// One pin as several branches can share it: table, lineage and target. diff --git a/crates/omnigraph/src/db/upgrade/legacy_schema_files.rs b/crates/omnigraph/src/db/upgrade/legacy_schema_files.rs new file mode 100644 index 000000000..aa008c8b7 --- /dev/null +++ b/crates/omnigraph/src/db/upgrade/legacy_schema_files.rs @@ -0,0 +1,217 @@ +//! The three root schema files a graph carried before the contract lived in +//! main's `__manifest` (`schema_contract` row, internal schema v13): read by +//! the storage-upgrade route over pre-row roots and by nothing served. +//! Conversion removes these files only after every branch carries its contract. + +use std::sync::Arc; + +use omnigraph_compiler::{SchemaIR, schema_ir_hash, schema_shape_hash_from_ir, validate_schema_ir}; +use serde::Deserialize; + +use crate::db::manifest::{SchemaContractHead, SchemaContractRow}; +use crate::db::schema_state::{ + SchemaContractIdentity, compile_schema_source, schema_lock_conflict, + validate_current_source_matches, validate_identity_envelope, +}; +use crate::error::{OmniError, Result}; +use crate::storage::{StorageAdapter, join_uri}; + +pub(crate) const SCHEMA_SOURCE_FILENAME: &str = "_schema.pg"; +pub(crate) const SCHEMA_IR_FILENAME: &str = "_schema.ir.json"; +pub(crate) const SCHEMA_STATE_FILENAME: &str = "__schema_state.json"; + +const SCHEMA_STATE_FORMAT_VERSION: u32 = 2; + +const MISSING_SCHEMA_CONTRACT_MESSAGE: &str = "graph is missing the mandatory identity-bearing schema contract (_schema.ir.json and __schema_state.json); automatic bootstrap is not supported"; +const INCOMPLETE_SCHEMA_CONTRACT_MESSAGE: &str = "graph schema contract is incomplete: _schema.ir.json and __schema_state.json must both be present"; + +pub(crate) fn schema_source_uri(root_uri: &str) -> String { + join_uri(root_uri, SCHEMA_SOURCE_FILENAME) +} + +pub(crate) fn schema_ir_uri(root_uri: &str) -> String { + join_uri(root_uri, SCHEMA_IR_FILENAME) +} + +pub(crate) fn schema_state_uri(root_uri: &str) -> String { + join_uri(root_uri, SCHEMA_STATE_FILENAME) +} + +/// The identity fields of `__schema_state.json`. Completed legacy applies +/// can retain a `publication` receipt; conversion ignores that retired field. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +struct LegacySchemaState { + format_version: u32, + schema_shape_hash: String, + schema_ir_hash: String, + schema_identity_version: u32, + schema_identity_domain: String, +} + +/// A pre-row root's contract as its three files carry it, validated: the +/// texts the upgrade route writes into the `schema_contract` row and the +/// identity that row records. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct LegacySchemaContract { + pub(crate) source: String, + pub(crate) ir_json: String, + pub(crate) ir: SchemaIR, + pub(crate) identity: SchemaContractIdentity, +} + +impl LegacySchemaContract { + pub(super) fn into_row(self) -> SchemaContractRow { + SchemaContractRow { + source: self.source, + ir: self.ir_json, + head: SchemaContractHead { + schema_ir_hash: self.identity.schema_ir_hash, + schema_identity_version: self.identity.schema_identity_version, + schema_identity_domain: self.identity.schema_identity_domain, + }, + } + } +} + +pub(super) async fn refuse_staging(root: &str, storage: &dyn StorageAdapter) -> Result<()> { + for name in [ + SCHEMA_SOURCE_FILENAME, + SCHEMA_IR_FILENAME, + SCHEMA_STATE_FILENAME, + ] { + if storage + .exists(&join_uri(root, &format!("{name}.staging"))) + .await? + { + return Err(schema_lock_conflict( + "legacy schema staging requires the source-compatible executable before conversion", + )); + } + } + Ok(()) +} + +/// Main's converted row survives interruption after any subset of these deletes. +pub(super) async fn cleanup( + root: &str, + storage: &dyn StorageAdapter, + contract: &SchemaContractRow, +) -> Result<()> { + let (ir, _) = crate::db::schema_state::validate_schema_contract_row(contract)?; + for name in [ + SCHEMA_SOURCE_FILENAME, + SCHEMA_IR_FILENAME, + SCHEMA_STATE_FILENAME, + ] { + let path = join_uri(root, name); + if !storage.exists(&path).await? { + continue; + } + let text = storage.read_text(&path).await?; + let matches = match name { + SCHEMA_SOURCE_FILENAME => text == contract.source, + SCHEMA_IR_FILENAME => text == contract.ir, + _ => { + let state: LegacySchemaState = serde_json::from_str(&text) + .map_err(|error| invalid_contract_file("graph schema state", name, error))?; + state.format_version == SCHEMA_STATE_FORMAT_VERSION + && state.schema_ir_hash == contract.head.schema_ir_hash + && state.schema_identity_version == contract.head.schema_identity_version + && state.schema_identity_domain == contract.head.schema_identity_domain + && state.schema_shape_hash + == schema_shape_hash_from_ir(&ir) + .map_err(|error| schema_lock_conflict(error.to_string()))? + } + }; + if !matches { + return Err(schema_lock_conflict(format!( + "legacy contract file {name} changed before cleanup" + ))); + } + storage.delete(&path).await?; + crate::seams::fail(&super::UPGRADE_AFTER_SCHEMA_FILE_DELETE)?; + } + Ok(()) +} + +/// Load the accepted IR and its schema identity from the three legacy root +/// files (three `read_text`, two `exists`). A missing or incomplete contract +/// reports an unparseable source first, and a storage error before a parse +/// error. +pub(crate) async fn load_validated_schema_contract( + root_uri: &str, + storage: Arc, +) -> Result { + let storage = storage.as_ref(); + let source = storage.read_text(&schema_source_uri(root_uri)).await?; + let ir_uri = schema_ir_uri(root_uri); + let state_uri = schema_state_uri(root_uri); + let ir_exists = storage.exists(&ir_uri).await?; + let state_exists = storage.exists(&state_uri).await?; + let (ir_json, state_json) = match (ir_exists, state_exists) { + (true, true) => ( + storage.read_text(&ir_uri).await?, + storage.read_text(&state_uri).await?, + ), + (false, false) => { + compile_schema_source(&source)?; + return Err(schema_lock_conflict(MISSING_SCHEMA_CONTRACT_MESSAGE)); + } + _ => { + compile_schema_source(&source)?; + return Err(schema_lock_conflict(INCOMPLETE_SCHEMA_CONTRACT_MESSAGE)); + } + }; + let current_source_shape = compile_schema_source(&source)?; + let ir = serde_json::from_str::(&ir_json).map_err(|err| { + invalid_contract_file("accepted compiled schema contract", SCHEMA_IR_FILENAME, err) + })?; + let state = serde_json::from_str::(&state_json) + .map_err(|err| invalid_contract_file("graph schema state", SCHEMA_STATE_FILENAME, err))?; + if state.format_version != SCHEMA_STATE_FORMAT_VERSION { + return Err(schema_lock_conflict(format!( + "graph schema state format {} is unsupported", + state.format_version + ))); + } + let head = SchemaContractHead { + schema_ir_hash: state.schema_ir_hash.clone(), + schema_identity_version: state.schema_identity_version, + schema_identity_domain: state.schema_identity_domain.clone(), + }; + validate_identity_envelope(&head)?; + validate_schema_ir(&ir).map_err(|error| { + schema_lock_conflict(format!( + "accepted compiled schema is not a valid identity-bearing IR: {error}" + )) + })?; + let actual_hash = schema_ir_hash(&ir).map_err(|err| schema_lock_conflict(err.to_string()))?; + if actual_hash != state.schema_ir_hash { + return Err(schema_lock_conflict( + "accepted compiled schema does not match the recorded schema state", + )); + } + let projected_shape_hash = + schema_shape_hash_from_ir(&ir).map_err(|err| schema_lock_conflict(err.to_string()))?; + if projected_shape_hash != state.schema_shape_hash { + return Err(schema_lock_conflict( + "accepted compiled schema's semantic projection does not match the recorded schema shape", + )); + } + if ir.schema_identity_domain.as_str() != state.schema_identity_domain { + return Err(schema_lock_conflict( + "accepted compiled schema identity domain does not match the recorded schema state", + )); + } + validate_current_source_matches(&ir, ¤t_source_shape)?; + Ok(LegacySchemaContract { + source, + ir_json, + ir, + identity: SchemaContractIdentity::from(&head), + }) +} + +fn invalid_contract_file(subject: &str, file: &str, err: impl std::fmt::Display) -> OmniError { + schema_lock_conflict(format!("{subject} in {file} is invalid: {err}")) +} diff --git a/crates/omnigraph/src/db/upgrade/legacy_sidecars.rs b/crates/omnigraph/src/db/upgrade/legacy_sidecars.rs index b6b326b6e..5b0f5db91 100644 --- a/crates/omnigraph/src/db/upgrade/legacy_sidecars.rs +++ b/crates/omnigraph/src/db/upgrade/legacy_sidecars.rs @@ -1,8 +1,8 @@ //! The `__recovery/` directory of graphs written before RFC 0067. //! //! No writer arms a recovery sidecar any more: every table effect is a -//! detached commit published as a pin, and a staged schema contract names the -//! graph commit that publishes it. A sidecar can therefore only come from a +//! detached commit published as a pin beside its schema contract row. A sidecar +//! can therefore only come from a //! build that predates detached table commits and stopped mid-write. This //! binary cannot interpret one, so a read-write open and the storage upgrade //! refuse the graph until the build that wrote the sidecar has resolved it. @@ -64,10 +64,10 @@ pub(crate) async fn refuse_legacy_sidecars( /// Nonmutating absence proof for callers without recovery authority (the /// cluster admission probe): every JSON object under `__recovery/` blocks, -/// including malformed ones — absence is the only provable state — and so -/// does any staged schema contract file, since only a read-write open may -/// settle one. Listing is bounded; exceeding the bound refuses rather than -/// walking an unbounded directory. +/// including malformed ones — absence is the only provable state. Listing is +/// bounded; exceeding the bound refuses rather than walking an unbounded +/// directory. A served (stamp 13) root carries no schema staging file; the +/// upgrade route judges those on a pre-row root. pub(crate) async fn refuse_pending_recovery( root_uri: &str, storage: &dyn StorageAdapter, @@ -89,18 +89,6 @@ pub(crate) async fn refuse_pending_recovery( "graph has pending recovery; resolve it with explicit recovery authority before retrying", )); } - for staging in [ - crate::db::schema_state::schema_source_staging_uri(root_uri), - crate::db::schema_state::schema_ir_staging_uri(root_uri), - crate::db::schema_state::schema_state_staging_uri(root_uri), - ] { - if storage.exists(&staging).await? { - return Err(OmniError::recovery_required( - "pending-schema-recovery", - "graph has staged schema recovery; resolve it with explicit recovery authority before retrying", - )); - } - } Ok(()) } diff --git a/crates/omnigraph/src/db/upgrade/tests.rs b/crates/omnigraph/src/db/upgrade/tests.rs index 67ca27c2a..53679eca2 100644 --- a/crates/omnigraph/src/db/upgrade/tests.rs +++ b/crates/omnigraph/src/db/upgrade/tests.rs @@ -35,6 +35,7 @@ async fn synthetic_v6_fixture_with_branch(root: &str, create_branch: bool) { .unwrap(); drop(db); settle_fixture_pins(root).await; + persist_legacy_schema_contract(root).await; replay_manifest_as_merge_writer(root).await; let mut dataset = open(root, None).await.unwrap(); dataset @@ -203,7 +204,7 @@ async fn settle_fixture_pins(root: &str) { /// The default route on a branch-free synthetic v6 graph runs all four /// steps and lands at v11; `--check` names the deferred preflights first. #[tokio::test] -async fn storage_upgrade_default_route_takes_a_synthetic_v6_graph_to_v11() { +async fn storage_upgrade_default_route_takes_a_synthetic_v6_graph_to_v13() { #[cfg(feature = "failpoints")] let _scenario = crate::seams::FailScenario::setup(); let dir = tempfile::tempdir().unwrap(); @@ -226,10 +227,11 @@ async fn storage_upgrade_default_route_takes_a_synthetic_v6_graph_to_v11() { HANDLER, RETIREMENT_HANDLER, DETACHED_PINS_HANDLER, - DETACHED_ONLY_HANDLER + DETACHED_ONLY_HANDLER, + SCHEMA_CONTRACT_HANDLER ] ); - assert_eq!(check.work.deferred_checks.len(), 3, "{check:?}"); + assert_eq!(check.work.deferred_checks.len(), 4, "{check:?}"); assert_eq!(stored_files(dir.path()), before); let upgraded = upgrade_storage(root, UpgradeOptions::default()) @@ -242,7 +244,8 @@ async fn storage_upgrade_default_route_takes_a_synthetic_v6_graph_to_v11() { HANDLER, RETIREMENT_HANDLER, DETACHED_PINS_HANDLER, - DETACHED_ONLY_HANDLER + DETACHED_ONLY_HANDLER, + SCHEMA_CONTRACT_HANDLER ] ); let reopened = Omnigraph::open(root).await.unwrap(); @@ -251,7 +254,7 @@ async fn storage_upgrade_default_route_takes_a_synthetic_v6_graph_to_v11() { .internal_schema_version_of(crate::db::ReadTarget::branch("main")) .await .unwrap(), - 11 + 13 ); // The storage route keeps the legacy spellings; the vintage is the // schema IR's, and `omnigraph schema upgrade-system-columns` converts it @@ -270,6 +273,7 @@ async fn storage_upgrade_default_route_takes_a_synthetic_v6_graph_to_v11() { /// of a graph an older binary left behind; fresh graphs of either vintage are /// born packed at the current stamp. async fn restamp_all_manifests(root: &str, stamp: u32) { + persist_legacy_schema_contract(root).await; let main = open(root, None).await.unwrap(); let refs: Vec = crate::branch_control::list_branch_contents(&main) .await @@ -288,6 +292,80 @@ async fn restamp_all_manifests(root: &str, stamp: u32) { .unwrap(); } +fn legacy_schema_names() -> [&'static str; 3] { + [ + legacy_schema_files::SCHEMA_SOURCE_FILENAME, + legacy_schema_files::SCHEMA_IR_FILENAME, + legacy_schema_files::SCHEMA_STATE_FILENAME, + ] +} + +async fn persist_legacy_schema_contract(root: &str) -> crate::db::manifest::SchemaContractRow { + let row = validated_manifest_contract(&open(root, None).await.unwrap()) + .await + .unwrap(); + let (ir, _) = crate::db::schema_state::validate_schema_contract_row(&row).unwrap(); + std::fs::write( + Path::new(root).join(legacy_schema_files::SCHEMA_SOURCE_FILENAME), + &row.source, + ) + .unwrap(); + std::fs::write( + Path::new(root).join(legacy_schema_files::SCHEMA_IR_FILENAME), + &row.ir, + ) + .unwrap(); + std::fs::write( + Path::new(root).join(legacy_schema_files::SCHEMA_STATE_FILENAME), + serde_json::to_vec(&serde_json::json!({ + "format_version": 2, + "schema_shape_hash": omnigraph_compiler::schema_shape_hash_from_ir(&ir).unwrap(), + "schema_ir_hash": row.head.schema_ir_hash, + "schema_identity_version": row.head.schema_identity_version, + "schema_identity_domain": row.head.schema_identity_domain, + })) + .unwrap(), + ) + .unwrap(); + row +} + +async fn restamp_schema_contract_source(dataset: &mut Dataset, stamp: u32) { + if stamp == 12 { + crate::db::manifest::migrations::restamp_packed_v12_for_test(dataset) + .await + .unwrap(); + } else { + crate::db::manifest::migrations::restamp_flat_for_test(dataset, stamp) + .await + .unwrap(); + } +} + +async fn schema_contract_fixture( + root: &str, + main_stamp: u32, + branch_stamp: u32, +) -> crate::db::manifest::SchemaContractRow { + let db = Omnigraph::init(root, "node Person { name: String }") + .await + .unwrap(); + db.branch_create("feature").await.unwrap(); + drop(db); + let row = persist_legacy_schema_contract(root).await; + let mut main = open(root, None).await.unwrap(); + for native in crate::branch_control::list_live_manifest_branch_contents(&main) + .await + .unwrap() + .into_keys() + { + let mut branch = main.checkout_branch(&native).await.unwrap(); + restamp_schema_contract_source(&mut branch, branch_stamp).await; + } + restamp_schema_contract_source(&mut main, main_stamp).await; + row +} + fn stored_files(root: &Path) -> BTreeMap, std::time::SystemTime)> { fn collect(path: &Path, files: &mut BTreeMap, std::time::SystemTime)>) { for entry in std::fs::read_dir(path).unwrap() { @@ -309,6 +387,565 @@ fn stored_files(root: &Path) -> BTreeMap, std::time::SystemTim files } +#[tokio::test] +async fn storage_upgrade_current_contract_ignores_legacy_orphans_and_former_sentinel_name() { + #[cfg(feature = "failpoints")] + let _scenario = crate::seams::FailScenario::setup(); + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + let db = Omnigraph::init(root, "node Person { name: String }") + .await + .unwrap(); + db.branch_create("__schema_apply_lock__").await.unwrap(); + for name in legacy_schema_names() { + std::fs::write(dir.path().join(name), "orphan, not authority").unwrap(); + std::fs::write(dir.path().join(format!("{name}.staging")), "orphan staging").unwrap(); + } + let before = stored_files(dir.path()); + for check in [true, false] { + let report = upgrade_storage( + root, + UpgradeOptions { + check, + to_format: None, + }, + ) + .await + .unwrap(); + assert_eq!(report.outcome, UpgradeOutcome::AlreadyCurrent, "{report:?}"); + assert_eq!(stored_files(dir.path()), before); + } + assert!(Omnigraph::open_read_only(root).await.is_ok()); +} + +async fn append_flat_manifest_extension( + root: &str, + object_id: &str, + object_type: &str, + metadata: &str, +) { + let main = open(root, None).await.unwrap(); + let schema = Arc::new(arrow_schema::Schema::from(main.schema())); + let columns = schema + .fields() + .iter() + .map(|field| -> arrow_array::ArrayRef { + let value = match field.name().as_str() { + "object_id" => object_id, + "object_type" => object_type, + "table_key" => "", + "metadata" => metadata, + _ => return arrow_array::new_null_array(field.data_type(), 1), + }; + Arc::new(arrow_array::StringArray::from(vec![value])) + }) + .collect(); + let batch = arrow_array::RecordBatch::try_new(schema.clone(), columns).unwrap(); + let input = arrow_array::RecordBatchIterator::new([Ok(batch)], schema); + Dataset::write( + input, + &format!("{root}/__manifest"), + Some(lance::dataset::WriteParams { + mode: lance::dataset::WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); +} + +#[tokio::test] +async fn storage_upgrade_refuses_restamped_current_layout_and_injected_legacy_contract() { + #[cfg(feature = "failpoints")] + let _scenario = crate::seams::FailScenario::setup(); + for (stamp, inject) in [ + (11, None), + (12, None), + (11, Some("schema_contract")), + (11, Some("unknown_extension")), + ] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + Omnigraph::init(root, "node Person { name: String }") + .await + .unwrap(); + let row = persist_legacy_schema_contract(root).await; + let mut main = open(root, None).await.unwrap(); + if let Some(object_type) = inject { + restamp_schema_contract_source(&mut main, stamp).await; + let head = serde_json::to_string(&row.head).unwrap(); + append_flat_manifest_extension(root, "schema_contract", object_type, &head).await; + } else { + main.update_schema_metadata([( + INTERNAL_SCHEMA_VERSION_KEY, + stamp.to_string().as_str(), + )]) + .await + .unwrap(); + } + let before = stored_files(dir.path()); + for check in [true, false] { + let report = upgrade_storage( + root, + UpgradeOptions { + check, + to_format: None, + }, + ) + .await + .unwrap(); + assert_eq!( + report.outcome, + UpgradeOutcome::CheckFailed, + "stamp {stamp}, inject {inject:?}: {report:?}" + ); + if inject == Some("unknown_extension") { + assert!( + format!("{report:?}").contains("has object_type 'unknown_extension'"), + "{report:?}" + ); + } + assert_eq!(stored_files(dir.path()), before); + } + } +} + +#[cfg(feature = "failpoints")] +#[tokio::test] +async fn storage_upgrade_schema_contract_cleanup_retries_after_each_delete() { + #[cfg(feature = "failpoints")] + let _scenario = crate::seams::FailScenario::setup(); + for deleted in 1..=3 { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + let contract = schema_contract_fixture(root, 11, 12).await; + let failed = { + let _fault = catalog::UPGRADE_AFTER_SCHEMA_FILE_DELETE.fire_once_at(deleted); + upgrade_storage(root, UpgradeOptions::default()) + .await + .unwrap() + }; + assert_eq!( + failed.outcome, + UpgradeOutcome::RecoveryRequired, + "{failed:?}" + ); + assert!( + intent_from(&open(root, None).await.unwrap()) + .unwrap() + .is_some() + ); + for (index, name) in legacy_schema_names().iter().enumerate() { + assert_eq!( + dir.path().join(name).exists(), + u64::try_from(index).unwrap() >= deleted + ); + } + assert_eq!( + validated_manifest_contract(&open(root, None).await.unwrap()) + .await + .unwrap(), + contract + ); + assert!(Omnigraph::open(root).await.is_err()); + assert!(Omnigraph::open_read_only(root).await.is_err()); + let resumed = upgrade_storage(root, UpgradeOptions::default()) + .await + .unwrap(); + assert_eq!(resumed.outcome, UpgradeOutcome::Completed, "{resumed:?}"); + let before = stored_files(dir.path()); + assert_eq!( + upgrade_storage(root, UpgradeOptions::default()) + .await + .unwrap() + .outcome, + UpgradeOutcome::AlreadyCurrent + ); + assert_eq!(stored_files(dir.path()), before); + assert!(Omnigraph::open_read_only(root).await.is_ok()); + } +} + +#[cfg(feature = "failpoints")] +#[tokio::test] +async fn storage_upgrade_schema_contract_resumes_every_partial_cleanup_state() { + let _scenario = crate::seams::FailScenario::setup(); + for absent_mask in 0..8 { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + let row = schema_contract_fixture(root, 12, 11).await; + { + let _fault = catalog::UPGRADE_AFTER_BRANCH.fire_once_at(2); + let report = upgrade_storage(root, UpgradeOptions::default()) + .await + .unwrap(); + assert_eq!( + report.outcome, + UpgradeOutcome::RecoveryRequired, + "{report:?}" + ); + } + assert_eq!( + validated_manifest_contract(&open(root, None).await.unwrap()) + .await + .unwrap(), + row + ); + for (index, name) in legacy_schema_names().iter().enumerate() { + if absent_mask & (1 << index) != 0 { + std::fs::remove_file(dir.path().join(name)).unwrap(); + } + } + let before = stored_files(dir.path()); + let check = upgrade_storage( + root, + UpgradeOptions { + check: true, + to_format: None, + }, + ) + .await + .unwrap(); + assert_eq!(check.outcome, UpgradeOutcome::RecoveryRequired, "{check:?}"); + assert_eq!(stored_files(dir.path()), before); + let resumed = upgrade_storage(root, UpgradeOptions::default()) + .await + .unwrap(); + assert_eq!( + resumed.outcome, + UpgradeOutcome::Completed, + "mask {absent_mask}: {resumed:?}" + ); + for name in legacy_schema_names() { + assert!(!dir.path().join(name).exists()); + } + assert_eq!( + validated_manifest_contract(&open(root, None).await.unwrap()) + .await + .unwrap(), + row + ); + } +} + +#[cfg(feature = "failpoints")] +#[tokio::test] +async fn storage_upgrade_schema_contract_interruption_boundaries_retry() { + #[cfg(feature = "failpoints")] + let _scenario = crate::seams::FailScenario::setup(); + for main_stamp in [11, 12] { + for seam in [ + &catalog::UPGRADE_AFTER_FENCE, + &catalog::UPGRADE_AFTER_STAGE, + &catalog::UPGRADE_AFTER_BRANCH, + &catalog::UPGRADE_BEFORE_ACTIVATION, + &catalog::UPGRADE_AFTER_ACTIVATION, + ] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + let row = schema_contract_fixture(root, main_stamp, 11).await; + let original = open(root, None).await.unwrap(); + let fence_version = original.version().version + 1; + let failed = { + let _fault = seam.fire_always(); + upgrade_storage(root, UpgradeOptions::default()) + .await + .unwrap() + }; + assert_eq!( + failed.outcome, + UpgradeOutcome::RecoveryRequired, + "{}: {failed:?}", + seam.name() + ); + let resumed = upgrade_storage(root, UpgradeOptions::default()) + .await + .unwrap(); + assert!(resumed.success(), "{}: {resumed:?}", seam.name()); + assert_eq!( + validated_manifest_contract(&open(root, None).await.unwrap()) + .await + .unwrap(), + row + ); + let historical = + crate::db::manifest::ManifestCoordinator::snapshot_at(root, None, fence_version) + .await + .unwrap(); + assert!(historical.dataset("node:Person").is_some()); + let fence = open(root, None) + .await + .unwrap() + .checkout_version(fence_version) + .await + .unwrap(); + equivalent(&original, &fence).await.unwrap(); + } + } +} + +#[tokio::test] +async fn storage_upgrade_schema_contract_refuses_forged_historical_fence() { + #[cfg(feature = "failpoints")] + let _scenario = crate::seams::FailScenario::setup(); + for stamp in [11, 12] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + let row = schema_contract_fixture(root, stamp, stamp).await; + let mut source = open(root, None).await.unwrap(); + let mut intent = inventory(&source, row.head.schema_identity_domain.clone()) + .await + .unwrap(); + intent.schema_contract = + Some(crate::db::manifest::migrations::UpgradeSchemaContract::from_row(&row)); + source + .update_schema_metadata([ + (INTERNAL_SCHEMA_VERSION_KEY.to_string(), "13".to_string()), + ( + UPGRADE_PENDING_KEY.to_string(), + serde_json::to_string(&intent).unwrap(), + ), + ( + "test:foreign-fence-effect".to_string(), + "unowned".to_string(), + ), + ]) + .await + .unwrap(); + let forged_version = source.version().version; + let mut source = publish_activation(source).await.unwrap(); + source + .update_schema_metadata([(INTERNAL_SCHEMA_VERSION_KEY, stamp.to_string().as_str())]) + .await + .unwrap(); + assert!( + upgrade_storage(root, UpgradeOptions::default()) + .await + .unwrap() + .success() + ); + let error = + crate::db::manifest::ManifestCoordinator::snapshot_at(root, None, forged_version) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("does not match its exact source"), + "{error}" + ); + } +} + +#[cfg(feature = "failpoints")] +#[tokio::test] +async fn storage_upgrade_schema_contract_refuses_contract_drift_on_retry() { + #[cfg(feature = "failpoints")] + let _scenario = crate::seams::FailScenario::setup(); + for filename in [ + legacy_schema_files::SCHEMA_SOURCE_FILENAME, + legacy_schema_files::SCHEMA_IR_FILENAME, + ] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + schema_contract_fixture(root, 11, 12).await; + { + let _fault = catalog::UPGRADE_AFTER_BRANCH.fire_once_at(1); + let report = upgrade_storage(root, UpgradeOptions::default()) + .await + .unwrap(); + assert_eq!( + report.outcome, + UpgradeOutcome::RecoveryRequired, + "{report:?}" + ); + } + let path = dir.path().join(filename); + let original = std::fs::read_to_string(&path).unwrap(); + std::fs::write(&path, format!("{original}\n")).unwrap(); + let before = stored_files(dir.path()); + let refused = upgrade_storage(root, UpgradeOptions::default()) + .await + .unwrap(); + assert_eq!( + refused.outcome, + UpgradeOutcome::RecoveryRequired, + "{refused:?}" + ); + assert!( + refused + .findings + .iter() + .any(|finding| finding.message.contains("exact text changed")), + "{refused:?}" + ); + assert_eq!(stored_files(dir.path()), before); + std::fs::write(path, original).unwrap(); + assert!( + upgrade_storage(root, UpgradeOptions::default()) + .await + .unwrap() + .success() + ); + } +} + +#[tokio::test] +async fn storage_upgrade_schema_contract_refuses_legacy_artifacts_without_effects() { + #[cfg(feature = "failpoints")] + let _scenario = crate::seams::FailScenario::setup(); + for artifact in [ + "_schema.pg.staging", + "_schema.ir.json.staging", + "__schema_state.json.staging", + "__schema_apply_lock__", + "__schema_apply_lock__.01ARZ3NDEKTSV4RRFFQ69G5FAV", + ] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + schema_contract_fixture(root, 11, 12).await; + if artifact.ends_with(".staging") { + std::fs::write(dir.path().join(artifact), "unfinished").unwrap(); + } else { + let mut main = open(root, None).await.unwrap(); + let version = main.version().version; + crate::storage_layer::lance_clone::create_branch(&mut main, artifact, version) + .await + .unwrap(); + } + let before = stored_files(dir.path()); + for check in [true, false] { + let result = upgrade_storage( + root, + UpgradeOptions { + check, + to_format: None, + }, + ) + .await + .unwrap(); + assert_eq!( + result.outcome, + UpgradeOutcome::CheckFailed, + "{artifact}: {result:?}" + ); + assert_eq!(stored_files(dir.path()), before); + } + } +} + +#[tokio::test] +async fn storage_upgrade_schema_contract_refuses_invalid_contract_without_effects() { + #[cfg(feature = "failpoints")] + let _scenario = crate::seams::FailScenario::setup(); + for case in [ + "missing", "source", "ir", "state", "identity", "hash", "shape", "domain", "manifest", + ] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + schema_contract_fixture(root, 12, 11).await; + match case { + "missing" => std::fs::remove_file(dir.path().join("_schema.ir.json")).unwrap(), + "source" => std::fs::write(dir.path().join("_schema.pg"), "invalid schema").unwrap(), + "ir" => std::fs::write(dir.path().join("_schema.ir.json"), "{}").unwrap(), + "manifest" => { + let other = tempfile::tempdir().unwrap(); + let other_root = other.path().to_str().unwrap(); + Omnigraph::init(other_root, "node Alien { name: String }") + .await + .unwrap(); + persist_legacy_schema_contract(other_root).await; + for name in legacy_schema_names() { + std::fs::copy(other.path().join(name), dir.path().join(name)).unwrap(); + } + } + field => { + let path = dir.path().join("__schema_state.json"); + let mut state: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + match field { + "state" => state["format_version"] = serde_json::json!(99), + "identity" => state["schema_identity_version"] = serde_json::json!(99), + "hash" => state["schema_ir_hash"] = serde_json::json!("sha256:bad"), + "shape" => state["schema_shape_hash"] = serde_json::json!("sha256:bad"), + "domain" => { + state["schema_identity_domain"] = + serde_json::json!("01ARZ3NDEKTSV4RRFFQ69G5FAV") + } + _ => unreachable!(), + } + std::fs::write(path, serde_json::to_vec(&state).unwrap()).unwrap(); + } + } + let before = stored_files(dir.path()); + let result = upgrade_storage(root, UpgradeOptions::default()) + .await + .unwrap(); + assert_eq!( + result.outcome, + UpgradeOutcome::CheckFailed, + "{case}: {result:?}" + ); + assert_eq!(stored_files(dir.path()), before); + } +} + +#[tokio::test] +async fn storage_upgrade_legacy_constructor_does_not_admit_ordinary_opens() { + #[cfg(feature = "failpoints")] + let _scenario = crate::seams::FailScenario::setup(); + for keep_row in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + Omnigraph::init(root, "node Person { name: String }") + .await + .unwrap(); + let contract = persist_legacy_schema_contract(root).await; + let mut main = open(root, None).await.unwrap(); + if keep_row { + main.update_schema_metadata([(INTERNAL_SCHEMA_VERSION_KEY, "10")]) + .await + .unwrap(); + } else { + restamp_schema_contract_source(&mut main, 10).await; + } + let _admission = crate::db::manifest::migrations::admit_conversion_source(root, 10); + let converted = + Omnigraph::open_for_storage_upgrade(root, crate::db::OpenMode::ReadOnly, contract) + .await; + assert_eq!(converted.is_ok(), !keep_row); + let identity = crate::storage::write_queue_root_identity( + &crate::storage::normalize_root_uri(root).unwrap(), + ) + .unwrap(); + let queue = crate::db::write_queue::WriteQueueManager::for_root(&identity); + let gate = queue.acquire_schema_exclusive().await; + for read_only in [false, true] { + let probes = crate::instrumentation::QueryIoProbes::default(); + let scans = Arc::clone(&probes.manifest_scan_count); + let opened = crate::instrumentation::with_query_io_probes(probes, async { + if read_only { + Omnigraph::open_read_only(root).await + } else { + Omnigraph::open(root).await + } + }); + let error = tokio::time::timeout(std::time::Duration::from_secs(2), opened) + .await + .expect("ordinary format refusal must precede the schema queue") + .err() + .unwrap() + .to_string(); + assert!(error.contains("reads only v13"), "{error}"); + assert_eq!(scans.load(std::sync::atomic::Ordering::Relaxed), 0); + } + drop(gate); + if let Ok(db) = converted { + assert!(db.fresh_snapshot_for_branch(None).await.is_ok()); + } + } +} + #[tokio::test] async fn storage_upgrade_check_has_no_local_store_effects() { #[cfg(feature = "failpoints")] @@ -490,7 +1127,7 @@ async fn storage_upgrade_interruption_boundaries_retry_without_mixed_visibility( assert!(Omnigraph::open_read_only(root).await.is_ok(), "{boundary}"); for branch in [None, Some("feature")] { let dataset = open(root, branch).await.unwrap(); - assert_eq!(read_stamp(&dataset), Some(11), "{boundary}"); + assert_eq!(read_stamp(&dataset), Some(13), "{boundary}"); } } } @@ -501,56 +1138,72 @@ async fn storage_upgrade_interruption_boundaries_retry_without_mixed_visibility( async fn storage_upgrade_recovery_refuses_foreign_head_movement() { use crate::seams::FailScenario; let _scenario = FailScenario::setup(); - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().to_str().unwrap(); - synthetic_v6_fixture(root).await; - { - let _fault = catalog::UPGRADE_AFTER_FENCE.fire_always(); - let interrupted = upgrade_storage( + for source_stamp in [6, 11, 12] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + let target = if source_stamp == 6 { + synthetic_v6_fixture(root).await; + 8 + } else { + schema_contract_fixture(root, source_stamp, source_stamp).await; + 13 + }; + { + let _fault = catalog::UPGRADE_AFTER_FENCE.fire_always(); + let interrupted = upgrade_storage( + root, + UpgradeOptions { + check: false, + to_format: Some(target), + }, + ) + .await + .unwrap(); + assert_eq!( + interrupted.outcome, + UpgradeOutcome::RecoveryRequired, + "{interrupted:?}" + ); + } + let pending = intent_from(&open(root, None).await.unwrap()) + .unwrap() + .unwrap(); + let native = pending + .branches + .iter() + .find_map(|branch| branch.native.as_deref()) + .unwrap(); + let mut foreign = open(root, Some(native)).await.unwrap(); + foreign + .update_schema_metadata([("test:foreign", "movement")]) + .await + .unwrap(); + let before_retry = stored_files(dir.path()); + let refused = upgrade_storage( root, UpgradeOptions { check: false, - to_format: Some(8), + to_format: Some(target), }, ) .await .unwrap(); assert_eq!( - interrupted.outcome, + refused.outcome, UpgradeOutcome::RecoveryRequired, - "{interrupted:?}" + "{refused:?}" + ); + assert!( + refused + .findings + .iter() + .any(|finding| finding.message.contains("foreign movement")), + "{refused:?}" ); + assert_eq!(stored_files(dir.path()), before_retry); + assert!(Omnigraph::open(root).await.is_err()); + assert!(Omnigraph::open_read_only(root).await.is_err()); } - let mut foreign = open(root, Some("feature")).await.unwrap(); - foreign - .update_schema_metadata([("test:foreign", "movement")]) - .await - .unwrap(); - let before_retry = stored_files(dir.path()); - let refused = upgrade_storage( - root, - UpgradeOptions { - check: false, - to_format: Some(8), - }, - ) - .await - .unwrap(); - assert_eq!( - refused.outcome, - UpgradeOutcome::RecoveryRequired, - "{refused:?}" - ); - assert!( - refused - .findings - .iter() - .any(|finding| finding.message.contains("foreign movement")), - "{refused:?}" - ); - assert_eq!(stored_files(dir.path()), before_retry); - assert!(Omnigraph::open(root).await.is_err()); - assert!(Omnigraph::open_read_only(root).await.is_err()); } #[tokio::test] @@ -960,7 +1613,11 @@ async fn storage_upgrade_v7_to_v8_preserves_manifest_fragments_and_history() { assert_eq!(finished.outcome, UpgradeOutcome::Completed, "{finished:?}"); assert_eq!( finished.completed_handlers, - [DETACHED_PINS_HANDLER, DETACHED_ONLY_HANDLER] + [ + DETACHED_PINS_HANDLER, + DETACHED_ONLY_HANDLER, + SCHEMA_CONTRACT_HANDLER + ] ); assert!(Omnigraph::open(root).await.is_ok()); let before = stored_files(dir.path()); @@ -1100,6 +1757,7 @@ async fn storage_upgrade_preserves_prior_v6_to_v7_pending_intent_before_continui .await .unwrap(), ); + persist_legacy_schema_contract(root).await; let mut dataset = open(root, None).await.unwrap(); crate::db::manifest::migrations::restamp_flat_for_test(&mut dataset, 6) .await @@ -1130,6 +1788,12 @@ async fn storage_upgrade_preserves_prior_v6_to_v7_pending_intent_before_continui .await .unwrap(); drop(dataset); + assert!( + upgrade_storage(root, UpgradeOptions::default()) + .await + .unwrap() + .success() + ); for _ in 0..2 { let db = Omnigraph::open(root).await.unwrap(); assert!( @@ -1225,7 +1889,11 @@ async fn storage_upgrade_current_v8_preserves_retired_ancestry_and_recreated_nam assert_eq!(upgraded.outcome, UpgradeOutcome::Completed, "{upgraded:?}"); assert_eq!( upgraded.completed_handlers, - [DETACHED_PINS_HANDLER, DETACHED_ONLY_HANDLER] + [ + DETACHED_PINS_HANDLER, + DETACHED_ONLY_HANDLER, + SCHEMA_CONTRACT_HANDLER + ] ); let reopened = Omnigraph::open(root).await.unwrap(); assert_eq!( @@ -1238,7 +1906,7 @@ async fn storage_upgrade_current_v8_preserves_retired_ancestry_and_recreated_nam .internal_schema_version_of(crate::db::ReadTarget::branch(branch)) .await .unwrap(), - 11, + 13, "live branch {branch} is restamped" ); } @@ -1309,7 +1977,7 @@ async fn storage_upgrade_current_vintage_is_already_current_without_a_route() { Some(crate::db::manifest::INTERNAL_MANIFEST_SCHEMA_VERSION) ); assert_eq!(stored_files(dir.path()), before); - for to_format in [11] { + for to_format in [13] { let explicit_served = upgrade_storage( root, UpgradeOptions { @@ -1331,6 +1999,7 @@ async fn storage_upgrade_current_vintage_is_already_current_without_a_route() { (8, "target_below_stamp"), (9, "unsupported_target"), (10, "target_below_stamp"), + (11, "target_below_stamp"), (12, "unsupported_target"), ] { let refused = upgrade_storage( @@ -1360,7 +2029,7 @@ async fn storage_upgrade_current_vintage_is_already_current_without_a_route() { } let mut dataset = open(root, None).await.unwrap(); dataset - .update_schema_metadata([(INTERNAL_SCHEMA_VERSION_KEY, "13")]) + .update_schema_metadata([(INTERNAL_SCHEMA_VERSION_KEY, "14")]) .await .unwrap(); drop(dataset); @@ -1444,7 +2113,7 @@ async fn storage_upgrade_legacy_source_refuses_reserved_retirement_metadata_with /// v10 restamp and the v11 step, `--to-format 8` stays already current before /// and after, and a v8 graph with another branch is refused before any effect. #[tokio::test] -async fn storage_upgrade_default_route_takes_a_legacy_v8_graph_to_v11() { +async fn storage_upgrade_default_route_takes_a_legacy_v8_graph_to_v13() { #[cfg(feature = "failpoints")] let _scenario = crate::seams::FailScenario::setup(); let dir = tempfile::tempdir().unwrap(); @@ -1483,9 +2152,16 @@ async fn storage_upgrade_default_route_takes_a_legacy_v8_graph_to_v11() { .await .unwrap(); assert_eq!(check.outcome, UpgradeOutcome::CheckPassed, "{check:?}"); - assert_eq!(check.target_format, 11); + assert_eq!(check.target_format, 13); assert!(check.target_defaulted); - assert_eq!(check.route, [DETACHED_PINS_HANDLER, DETACHED_ONLY_HANDLER]); + assert_eq!( + check.route, + [ + DETACHED_PINS_HANDLER, + DETACHED_ONLY_HANDLER, + SCHEMA_CONTRACT_HANDLER + ] + ); assert_eq!(stored_files(dir.path()), before, "check writes nothing"); let upgraded = upgrade_storage( @@ -1500,7 +2176,11 @@ async fn storage_upgrade_default_route_takes_a_legacy_v8_graph_to_v11() { assert_eq!(upgraded.outcome, UpgradeOutcome::Completed, "{upgraded:?}"); assert_eq!( upgraded.completed_handlers, - [DETACHED_PINS_HANDLER, DETACHED_ONLY_HANDLER] + [ + DETACHED_PINS_HANDLER, + DETACHED_ONLY_HANDLER, + SCHEMA_CONTRACT_HANDLER + ] ); assert_eq!( upgraded.last_durable_completed_boundary.as_deref(), @@ -1512,7 +2192,7 @@ async fn storage_upgrade_default_route_takes_a_legacy_v8_graph_to_v11() { .internal_schema_version_of(crate::db::ReadTarget::branch("main")) .await .unwrap(), - 11 + 13 ); let snapshot = reopened.snapshot().await; let person = reopened @@ -1525,7 +2205,7 @@ async fn storage_upgrade_default_route_takes_a_legacy_v8_graph_to_v11() { "the storage route keeps the legacy spellings" ); drop(reopened); - for to_format in [None, Some(11)] { + for to_format in [None, Some(13)] { let again = upgrade_storage( root, UpgradeOptions { @@ -1592,107 +2272,131 @@ async fn storage_upgrade_default_route_takes_a_legacy_v8_graph_to_v11() { .internal_schema_version_of(crate::db::ReadTarget::branch(branch)) .await .unwrap(), - 11 + 13 ); } } -/// Lazy conversion leaves a graph with v11 and v12 branches until every branch -/// has published; `upgrade --check` and execute report it already current, -/// whichever branch converted first. #[tokio::test] -async fn storage_upgrade_accepts_a_graph_converting_branch_by_branch() { +async fn storage_upgrade_schema_contract_converts_mixed_branch_formats() { #[cfg(feature = "failpoints")] let _scenario = crate::seams::FailScenario::setup(); - for published in ["main", "feature"] { - let idle = if published == "main" { - "feature" - } else { - "main" - }; + for (main_stamp, branch_stamp) in [(11, 12), (12, 11), (11, 11), (12, 12)] { let dir = tempfile::tempdir().unwrap(); let root = dir.path().to_str().unwrap(); - let db = Omnigraph::init(root, "node Person { name: String }") + let contract = schema_contract_fixture(root, main_stamp, branch_stamp).await; + if main_stamp == 11 { + append_flat_manifest_extension( + root, + "unknown_extension", + "unknown_extension", + "extension retained verbatim", + ) + .await; + } + let state_path = dir.path().join("__schema_state.json"); + let mut legacy_state: serde_json::Value = + serde_json::from_slice(&std::fs::read(&state_path).unwrap()).unwrap(); + legacy_state["publication"] = + serde_json::json!({"graph_manifest_version": 1, "graph_commit_id": "retired-receipt"}); + std::fs::write(&state_path, serde_json::to_vec(&legacy_state).unwrap()).unwrap(); + let main = open(root, None).await.unwrap(); + let branch = crate::branch_control::list_live_manifest_branch_contents(&main) .await + .unwrap() + .into_keys() + .next() .unwrap(); - db.branch_create("feature").await.unwrap(); - drop(db); - restamp_all_manifests(root, 11).await; - let db = crate::Session::from_defaults( - std::sync::Arc::new(Omnigraph::open(root).await.unwrap()), - omnigraph_compiler::settings::SessionSettings::default(), - ); - db.mutate( - published, - "query seed($name: String) { insert Person { name: $name } }", - "seed", - &HashMap::from([( - "name".to_string(), - omnigraph_compiler::query::ast::Literal::String("converting".to_string()), - )]), + let mut sources = Vec::new(); + for native in [Some(branch.as_str()), None] { + let source = open(root, native).await.unwrap(); + crate::db::manifest::migrations::validate_schema_contract_source(&source).unwrap(); + sources.push((native.map(str::to_owned), source)); + } + assert!(Omnigraph::open(root).await.is_err()); + let before = stored_files(dir.path()); + let check = upgrade_storage( + root, + UpgradeOptions { + check: true, + to_format: None, + }, ) .await .unwrap(); - drop(db); - let converting = Omnigraph::open(root).await.unwrap(); - for (branch, stamp) in [(published, 12), (idle, 11)] { + assert_eq!(check.outcome, UpgradeOutcome::CheckPassed, "{check:?}"); + assert_eq!(check.route, [SCHEMA_CONTRACT_HANDLER]); + assert_eq!(stored_files(dir.path()), before); + let report = upgrade_storage(root, UpgradeOptions::default()) + .await + .unwrap(); + assert_eq!(report.outcome, UpgradeOutcome::Completed, "{report:?}"); + assert_eq!(report.completed_handlers, [SCHEMA_CONTRACT_HANDLER]); + for (native, source) in sources { + let target = open(root, native.as_deref()).await.unwrap(); + assert_eq!(read_stamp(&target), Some(13)); assert_eq!( - converting - .internal_schema_version_of(crate::db::ReadTarget::branch(branch)) - .await - .unwrap(), - stamp, - "after one publish on {published}, {branch}" + validated_manifest_contract(&target).await.unwrap(), + contract ); - } - drop(converting); - let before = stored_files(dir.path()); - for check in [true, false] { - for to_format in [None, Some(11)] { - let report = upgrade_storage(root, UpgradeOptions { check, to_format }) - .await + equivalent(&source, &target).await.unwrap(); + if native.is_none() && main_stamp == 11 { + let batch = target.scan().try_into_batch().await.unwrap(); + let ids = batch + .column_by_name("object_id") + .unwrap() + .as_any() + .downcast_ref::() .unwrap(); - assert_eq!( - report.outcome, - UpgradeOutcome::AlreadyCurrent, - "publish on {published}, check {check}, target {to_format:?}: {report:?}" - ); - assert_eq!(stored_files(dir.path()), before); + let extension_rows = (0..batch.num_rows()) + .filter(|&row| ids.value(row) == "unknown_extension") + .collect::>(); + assert_eq!(extension_rows.len(), 1); + let record = batch + .column_by_name("record") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for (column, expected) in [ + ( + batch.column_by_name("object_type").unwrap(), + "unknown_extension", + ), + ( + record.column_by_name("metadata").unwrap(), + "extension retained verbatim", + ), + ] { + let values = column.as_any().downcast_ref::().unwrap(); + assert_eq!(values.value(extension_rows[0]), expected); + } } - for (to_format, expected_code) in - [(10, "target_below_stamp"), (12, "unsupported_target")] - { - let refused = upgrade_storage( - root, - UpgradeOptions { - check, - to_format: Some(to_format), - }, - ) + let retained = target + .checkout_version(source.version().version) .await .unwrap(); - assert_eq!( - refused.outcome, - UpgradeOutcome::CheckFailed, - "publish on {published}, check {check}, target {to_format}: {refused:?}" - ); - assert!( - refused - .findings - .iter() - .any(|finding| finding.code == expected_code), - "{refused:?}" - ); - assert_eq!(stored_files(dir.path()), before); - } + assert_eq!(read_stamp(&retained), read_stamp(&source)); + equivalent(&source, &retained).await.unwrap(); } + for name in legacy_schema_names() { + assert!(!dir.path().join(name).exists()); + } + let db = Omnigraph::open_read_only(root).await.unwrap(); + assert_eq!(db.branch_list().await.unwrap(), ["main", "feature"]); + let before = stored_files(dir.path()); + let again = upgrade_storage(root, UpgradeOptions::default()) + .await + .unwrap(); + assert_eq!(again.outcome, UpgradeOutcome::AlreadyCurrent); + assert_eq!(stored_files(dir.path()), before); } } /// A v9 graph, the 0.11.x current vintage, takes the v10 restamp and then the /// v11 step, with every branch restamped and no payload copied or rewritten. #[tokio::test] -async fn storage_upgrade_default_route_takes_a_v9_graph_to_v11() { +async fn storage_upgrade_default_route_takes_a_v9_graph_to_v13() { #[cfg(feature = "failpoints")] let _scenario = crate::seams::FailScenario::setup(); let dir = tempfile::tempdir().unwrap(); @@ -1719,8 +2423,15 @@ async fn storage_upgrade_default_route_takes_a_v9_graph_to_v11() { .unwrap(); assert_eq!(check.outcome, UpgradeOutcome::CheckPassed, "{check:?}"); assert_eq!(check.observed_format, Some(9)); - assert_eq!(check.target_format, 11); - assert_eq!(check.route, [DETACHED_PINS_HANDLER, DETACHED_ONLY_HANDLER]); + assert_eq!(check.target_format, 13); + assert_eq!( + check.route, + [ + DETACHED_PINS_HANDLER, + DETACHED_ONLY_HANDLER, + SCHEMA_CONTRACT_HANDLER + ] + ); assert_eq!(stored_files(dir.path()), before, "check writes nothing"); let unsupported = upgrade_storage( root, @@ -1745,7 +2456,11 @@ async fn storage_upgrade_default_route_takes_a_v9_graph_to_v11() { assert_eq!(upgraded.outcome, UpgradeOutcome::Completed, "{upgraded:?}"); assert_eq!( upgraded.completed_handlers, - [DETACHED_PINS_HANDLER, DETACHED_ONLY_HANDLER] + [ + DETACHED_PINS_HANDLER, + DETACHED_ONLY_HANDLER, + SCHEMA_CONTRACT_HANDLER + ] ); assert_eq!(upgraded.work.payload_bytes_copied, 0); assert_eq!(upgraded.work.payload_bytes_rewritten, 0); @@ -1757,7 +2472,7 @@ async fn storage_upgrade_default_route_takes_a_v9_graph_to_v11() { .internal_schema_version_of(crate::db::ReadTarget::branch(branch)) .await .unwrap(), - 11 + 13 ); } drop(reopened); @@ -1771,7 +2486,7 @@ async fn storage_upgrade_default_route_takes_a_v9_graph_to_v11() { /// pin and writes nothing; execution promotes the pending pin once, reaps its /// copy, records the last linear version on every live branch and restamps. #[tokio::test] -async fn storage_upgrade_default_route_takes_a_v10_graph_to_v11() { +async fn storage_upgrade_default_route_takes_a_v10_graph_to_v13() { use crate::db::omnigraph::promotion::{open_at, table_location, walk_chain}; #[cfg(feature = "failpoints")] let _scenario = crate::seams::FailScenario::setup(); @@ -1829,7 +2544,7 @@ async fn storage_upgrade_default_route_takes_a_v10_graph_to_v11() { Err(error) => error.to_string(), }; assert!( - refused.contains("reads only v11 to v12"), + refused.contains("reads only v13 to v13"), "a v10 graph is refused by normal open: {refused}" ); @@ -1845,9 +2560,12 @@ async fn storage_upgrade_default_route_takes_a_v10_graph_to_v11() { .unwrap(); assert_eq!(check.outcome, UpgradeOutcome::CheckPassed, "{check:?}"); assert_eq!(check.observed_format, Some(10)); - assert_eq!(check.target_format, 11); + assert_eq!(check.target_format, 13); assert!(check.target_defaulted); - assert_eq!(check.route, [DETACHED_ONLY_HANDLER]); + assert_eq!( + check.route, + [DETACHED_ONLY_HANDLER, SCHEMA_CONTRACT_HANDLER] + ); assert_eq!(stored_files(dir.path()), before, "check writes nothing"); #[cfg(feature = "failpoints")] @@ -1884,7 +2602,10 @@ async fn storage_upgrade_default_route_takes_a_v10_graph_to_v11() { .await .unwrap(); assert_eq!(upgraded.outcome, UpgradeOutcome::Completed, "{upgraded:?}"); - assert_eq!(upgraded.completed_handlers, [DETACHED_ONLY_HANDLER]); + assert_eq!( + upgraded.completed_handlers, + [DETACHED_ONLY_HANDLER, SCHEMA_CONTRACT_HANDLER] + ); assert_eq!( upgraded.last_durable_completed_boundary.as_deref(), Some("activated") @@ -1898,7 +2619,7 @@ async fn storage_upgrade_default_route_takes_a_v10_graph_to_v11() { .internal_schema_version_of(crate::db::ReadTarget::branch(branch)) .await .unwrap(), - 11, + 13, "live branch {branch} is restamped" ); let snapshot = reopened.snapshot_for_branch(Some(branch)).await.unwrap(); @@ -1960,7 +2681,7 @@ async fn storage_upgrade_default_route_takes_a_v10_graph_to_v11() { ); drop(reopened); - for to_format in [None, Some(11)] { + for to_format in [None, Some(13)] { let again = upgrade_storage( root, UpgradeOptions { diff --git a/crates/omnigraph/src/db/write_queue.rs b/crates/omnigraph/src/db/write_queue.rs index 89e0d9e6b..b0a15658d 100644 --- a/crates/omnigraph/src/db/write_queue.rs +++ b/crates/omnigraph/src/db/write_queue.rs @@ -333,8 +333,8 @@ pub(crate) struct ExportDestructivePermit { /// /// Every `Omnigraph` handle for one canonical root identity shares the same /// manager via a process-global weak registry. This matters beyond HTTP's usual -/// `Arc` shape: a separately-opened handle can settle a staged -/// schema contract or delete refs, which must serialize with a live writer +/// `Arc` shape: a separately-opened handle can apply a schema +/// or delete refs, which must serialize with a live writer /// owned by the first handle. The registry deliberately keys only by the queue root /// identity; custom storage adapters for the same URI conservatively serialize /// too. @@ -426,18 +426,6 @@ impl WriteQueueManager { scheduled_schema_shared(Arc::clone(&self.schema_gate)).await } - /// The shared side without waiting: `None` while an exclusive permit is - /// held or queued in this process. A standing schema-apply sentinel with - /// a free gate is therefore another process's apply (or a dead one). - pub(crate) fn try_acquire_schema_shared(&self) -> Option { - let _turn = crate::dst_gate::turn(); - let guard = Arc::clone(&self.schema_gate.lock).try_read_owned().ok()?; - Some(SchemaSharedPermit { - inner: Some(guard), - slot: Arc::clone(&self.schema_gate), - }) - } - /// Take the schema gate's exclusive side: sole ownership of the /// accepted-view transition. Shared holders drain first; new shared /// permits queue behind this acquisition in plain mode. Same @@ -670,7 +658,7 @@ mod tests { let writer = tokio::spawn(async move { qm_writer.acquire_schema_exclusive().await }); let qm_probe = Arc::clone(&qm); timeout(Duration::from_secs(2), async move { - while let Some(permit) = qm_probe.try_acquire_schema_shared() { + while let Ok(permit) = Arc::clone(&qm_probe.schema_gate.lock).try_read_owned() { drop(permit); tokio::task::yield_now().await; } diff --git a/crates/omnigraph/src/exec/merge.rs b/crates/omnigraph/src/exec/merge.rs index 6f95d1d05..f3f15a6db 100644 --- a/crates/omnigraph/src/exec/merge.rs +++ b/crates/omnigraph/src/exec/merge.rs @@ -5156,9 +5156,6 @@ impl Session { }, actor_id, )?; - self.ensure_schema_apply_idle("branch_merge") - .await - .map_err(OmniError::before_effect)?; // Keep the planning/publication future out of the public API's callers; // deeply composed loads and merges otherwise retain large debug // construction frames throughout execution. Poll it in the same task. @@ -5292,12 +5289,6 @@ impl Omnigraph { let outer_prepare_timing = crate::instrumentation::start_merge_timing( crate::instrumentation::MergeTimingPhase::OuterPrepare, ); - if is_internal_system_branch(source) || is_internal_system_branch(target) { - return Err(OmniError::manifest(format!( - "branch_merge does not allow internal system refs ('{}' -> '{}')", - source, target - ))); - } let source_branch = Omnigraph::normalize_branch_name(source)?; let target_branch = Omnigraph::normalize_branch_name(target)?; if source_branch == target_branch { @@ -5307,28 +5298,11 @@ impl Omnigraph { } let relevant_branches = [source_branch.as_deref(), target_branch.as_deref()]; - // Branch merge is still a legacy per-table publisher, but its graph-ref - // authority must be stable for the complete prepare -> publish window. - // First install any pending schema contract of this handle, then join the - // same root-shared schema -> branch order used by native branch controls. - // Holding both branch gates through publication prevents a target - // delete/recreate from reusing the branch name underneath a plan (ABA). - let completed_prior_work = self.settle_pending_schema_install().await?; - let preparation_error = |error: OmniError| { - if completed_prior_work { - error.without_pre_effect_evidence() - } else { - error.before_effect() - } - }; let _schema_permit = self.write_queue().acquire_schema_shared().await; let _branch_guards = self .write_queue() .acquire_branches(&[source_branch.clone(), target_branch.clone()]) .await; - self.ensure_schema_apply_not_locked("branch_merge") - .await - .map_err(preparation_error)?; // Capture each branch as one coherent RFC-022 authority token plus // immutable snapshot. The target token is the coarse publish read set; // the source token pins the exact merge input without requiring the @@ -5336,7 +5310,7 @@ impl Omnigraph { let (source_txn, target_txn, source_commits, target_commits) = self .open_merge_write_txns(source_branch.as_deref(), target_branch.as_deref()) .await - .map_err(preparation_error)?; + .map_err(OmniError::before_effect)?; let source_head_commit_id = source_txn .effective_graph_head .clone() @@ -5354,7 +5328,7 @@ impl Omnigraph { &relevant_branches, ) .await - .map_err(preparation_error)?; + .map_err(OmniError::before_effect)?; if source_head_commit_id == target_head_commit_id || base_commit.graph_commit_id == source_head_commit_id diff --git a/crates/omnigraph/src/exec/mod.rs b/crates/omnigraph/src/exec/mod.rs index 624eb2219..9e4146771 100644 --- a/crates/omnigraph/src/exec/mod.rs +++ b/crates/omnigraph/src/exec/mod.rs @@ -29,7 +29,7 @@ use time::format_description::well_known::Rfc3339; use crate::db::Snapshot; use crate::db::manifest::ManifestCoordinator; -use crate::db::{MergeOutcome, MergeResult, Omnigraph, WriteTxn, is_internal_system_branch}; +use crate::db::{MergeOutcome, MergeResult, Omnigraph, WriteTxn}; use crate::error::{MergeConflict, MergeConflictKind, OmniError, Result}; use crate::storage_layer::SnapshotHandle; use tempfile::{Builder as TempDirBuilder, TempDir}; diff --git a/crates/omnigraph/src/exec/mutation.rs b/crates/omnigraph/src/exec/mutation.rs index 3d37509bf..1c5efbee6 100644 --- a/crates/omnigraph/src/exec/mutation.rs +++ b/crates/omnigraph/src/exec/mutation.rs @@ -882,7 +882,7 @@ impl Omnigraph { "prepared mutation authority changed before effects; repreparing" ); crate::instrumentation::record_mutation_reprepare(); - self.refresh_for_reprepare().await?; + self.refresh_coordinator_only().await?; } result => return result, } @@ -903,165 +903,140 @@ impl Omnigraph { retryable: &mut bool, ) -> Result { let requested = Self::normalize_branch_name(branch)?; - // Reject internal `__run__*` / system-prefixed branches at the - // public write boundary. Direct-publish paths assert this - // explicitly so a caller can't write to legacy or system - // staging branches by passing the prefix verbatim. - if let Some(name) = requested.as_deref() { - crate::db::ensure_public_branch_ref(name, "mutate")?; - } - // Install this handle's published-but-uninstalled schema contract, if - // any. This MUST run before `open_write_txn`, which captures the - // accepted schema identity and catalog. - let completed_prior_work = self.settle_pending_schema_install().await?; - let result = async { - // Capture one branch-wide write authority: native branch identity, - // exact optional graph head, accepted schema identity/catalog, and the - // base table snapshot. Execution, validation, staging, and publication - // all use this immutable attempt. `commit_all` revalidates the complete - // token under the root-shared schema → branch → sorted-table gates - // before its first detached commit. - let mut txn = self - .open_write_txn(requested.as_deref()) - .await - .map_err(|error| { - if first_attempt && !completed_prior_work { - error.before_effect() - } else { - error.without_pre_effect_evidence() - } - })?; - // Caller CAS gate against the pinned view this attempt executes with — - // a separate head lookup would reopen the race. Re-checked per - // reprepare; not `ReadSetChanged`, so the retry loop never replays it. - if let Some(expected) = expected_head { - let actual = txn.effective_graph_head.as_deref(); - if actual != Some(expected) { - return Err(OmniError::precondition_failed( - requested.as_deref().unwrap_or("main"), - expected, - actual.map(str::to_string), - )); + // Capture one branch-wide write authority: native branch identity, + // exact optional graph head, accepted schema identity/catalog, and the + // base table snapshot. Execution, validation, staging, and publication + // all use this immutable attempt. `commit_all` revalidates the complete + // token under the root-shared schema → branch → sorted-table gates + // before its first detached commit. + let mut txn = self + .open_write_txn(requested.as_deref()) + .await + .map_err(|error| { + if first_attempt { + error.before_effect() + } else { + error.without_pre_effect_evidence() } - txn.caller_expected_graph_head = Some(expected.to_string()); + })?; + // Caller CAS gate against the pinned view this attempt executes with — + // a separate head lookup would reopen the race. Re-checked per + // reprepare; not `ReadSetChanged`, so the retry loop never replays it. + if let Some(expected) = expected_head { + let actual = txn.effective_graph_head.as_deref(); + if actual != Some(expected) { + return Err(OmniError::precondition_failed( + requested.as_deref().unwrap_or("main"), + expected, + actual.map(str::to_string), + )); } - let resolved_params = params.clone(); - - // Per-query staging accumulator. Inserts and updates push batches into - // `pending`; deletes push predicates into `delete_predicates`. At the - // boundary, `stage_all` prepares one exact transaction per touched table - // and `commit_all` commits each as a detached version of its pinned - // base (RFC 0067). The publisher then makes the complete result - // graph-visible in one manifest CAS. Branch is threaded explicitly — no - // coordinator swap. - let mut staging = MutationStaging::default(); - - // Lower + validate up front so the touched-dataset set is known before - // execution. A lowering/validation error returns exactly as it did - // when this happened inside execute_named_mutation. - let ir = self.lower_named_mutation(&txn.catalog, query_source, query_name)?; - check_param_date_literals(params, &ir.params)?; - // Only an insert-only mutation is safe to replay automatically after a - // pre-effect authority mismatch. Update/Delete keep strict caller-visible - // `ReadSetChanged`; replaying their stale read-modify-write plan would be - // a semantic rebase rather than a fresh execution contract. - *retryable = ir - .ops - .iter() - .all(|op| matches!(op, MutationOpIR::Insert { .. })); - - let exec_result = self - .execute_named_mutation( - &ir, - &resolved_params, - requested.as_deref(), - &mut staging, - &txn, - ) - .await; - - match exec_result { - Err(e) => Err(e), - Ok(total) if staging.is_empty() => { - if txn.caller_expected_graph_head.is_some() { - fail(&MUTATION_POST_NO_EFFECT_PRE_GATE)?; - // A no-op has no table transaction, so it never reaches - // `commit_all`. It still needs a linearization point for - // the caller's CAS promise: under the same schema -> branch - // ordering as effectful writes (shared permit — this pass - // only reads the accepted view), revalidate the complete - // authority and map a moved caller head to terminal 412. - let _schema_permit = self.write_queue().acquire_schema_shared().await; - let _branch_guard = self - .write_queue() - .acquire_branch(requested.as_deref()) - .await; - self.revalidate_write_txn(&txn).await?; - } - Ok(crate::MutationReceipt { - result: total, - commit: None, - }) - } - Ok(total) => { - self.validate_staged_mutation(&staging, &txn).await?; - let staged = staging - .stage_all_with_concurrency( - self, - requested.as_deref(), - stage_write_concurrency, - ) - .await?; - fail(&MUTATION_POST_STAGE_PRE_EFFECT_GATE)?; - let lineage_intent = self - .new_lineage_intent_for_branch(requested.as_deref(), actor_id) - .await?; - // `_held_gates` holds the shared schema permit, branch - // effect gate, and sorted table gates acquired by `commit_all`. - // They remain held through manifest publication, covering the - // complete same-process effect lifetime. They are a local - // serialization aid; the exact publisher precondition remains - // the correctness authority. - let super::staging::CommittedMutation { - updates, - expected_versions, - gates: _held_gates, - } = staged.commit_all(self, requested.as_deref(), &txn).await?; - // Failpoint for the detached-effects → publisher boundary: - // every table effect is committed detached but nothing is - // graph-visible. A failure here leaves the graph unchanged and - // the detached versions as reclaimable garbage. See - // `tests/failpoints.rs::finalize_publisher_residual_does_not_drift_untouched_tables`. - fail(&MUTATION_POST_FINALIZE_PRE_PUBLISHER)?; - let publish_result = self - .commit_updates_on_branch_with_expected( - requested.as_deref(), - &updates, - &expected_versions, - actor_id, - &txn, - lineage_intent, - ) + txn.caller_expected_graph_head = Some(expected.to_string()); + } + let resolved_params = params.clone(); + + // Per-query staging accumulator. Inserts and updates push batches into + // `pending`; deletes push predicates into `delete_predicates`. At the + // boundary, `stage_all` prepares one exact transaction per touched table + // and `commit_all` commits each as a detached version of its pinned + // base (RFC 0067). The publisher then makes the complete result + // graph-visible in one manifest CAS. Branch is threaded explicitly — no + // coordinator swap. + let mut staging = MutationStaging::default(); + + // Lower + validate up front so the touched-dataset set is known before + // execution. A lowering/validation error returns exactly as it did + // when this happened inside execute_named_mutation. + let ir = self.lower_named_mutation(&txn.catalog, query_source, query_name)?; + check_param_date_literals(params, &ir.params)?; + // Only an insert-only mutation is safe to replay automatically after a + // pre-effect authority mismatch. Update/Delete keep strict caller-visible + // `ReadSetChanged`; replaying their stale read-modify-write plan would be + // a semantic rebase rather than a fresh execution contract. + *retryable = ir + .ops + .iter() + .all(|op| matches!(op, MutationOpIR::Insert { .. })); + + let exec_result = self + .execute_named_mutation( + &ir, + &resolved_params, + requested.as_deref(), + &mut staging, + &txn, + ) + .await; + + match exec_result { + Err(e) => Err(e), + Ok(total) if staging.is_empty() => { + if txn.caller_expected_graph_head.is_some() { + fail(&MUTATION_POST_NO_EFFECT_PRE_GATE)?; + // A no-op has no table transaction, so it never reaches + // `commit_all`. It still needs a linearization point for + // the caller's CAS promise: under the same schema -> branch + // ordering as effectful writes (shared permit — this pass + // only reads the accepted view), revalidate the complete + // authority and map a moved caller head to terminal 412. + let _schema_permit = self.write_queue().acquire_schema_shared().await; + let _branch_guard = self + .write_queue() + .acquire_branch(requested.as_deref()) .await; - // RFC 0067: every effect is a detached commit of its pinned base, - // so a publish failure leaves the graph unchanged; the error - // is returned as is (a moved head is `ReadSetChanged`). - let commit = publish_result?; - Ok(crate::MutationReceipt { - result: total, - commit: Some(commit), - }) + self.revalidate_write_txn(&txn).await?; } + Ok(crate::MutationReceipt { + result: total, + commit: None, + }) } - } - .await; - result.map_err(|error: OmniError| { - if completed_prior_work { - error.without_pre_effect_evidence() - } else { - error + Ok(total) => { + self.validate_staged_mutation(&staging, &txn).await?; + let staged = staging + .stage_all_with_concurrency(self, requested.as_deref(), stage_write_concurrency) + .await?; + fail(&MUTATION_POST_STAGE_PRE_EFFECT_GATE)?; + let lineage_intent = self + .new_lineage_intent_for_branch(requested.as_deref(), actor_id) + .await?; + // `_held_gates` holds the shared schema permit, branch + // effect gate, and sorted table gates acquired by `commit_all`. + // They remain held through manifest publication, covering the + // complete same-process effect lifetime. They are a local + // serialization aid; the exact publisher precondition remains + // the correctness authority. + let super::staging::CommittedMutation { + updates, + expected_versions, + gates: _held_gates, + } = staged.commit_all(self, requested.as_deref(), &txn).await?; + // Failpoint for the detached-effects → publisher boundary: + // every table effect is committed detached but nothing is + // graph-visible. A failure here leaves the graph unchanged and + // the detached versions as reclaimable garbage. See + // `tests/failpoints.rs::finalize_publisher_residual_does_not_drift_untouched_tables`. + fail(&MUTATION_POST_FINALIZE_PRE_PUBLISHER)?; + let publish_result = self + .commit_updates_on_branch_with_expected( + requested.as_deref(), + &updates, + &expected_versions, + actor_id, + &txn, + lineage_intent, + ) + .await; + // RFC 0067: every effect is a detached commit of its pinned base, + // so a publish failure leaves the graph unchanged; the error + // is returned as is (a moved head is `ReadSetChanged`). + let commit = publish_result?; + Ok(crate::MutationReceipt { + result: total, + commit: Some(commit), + }) } - }) + } } /// Lower + validate a named mutation query into its IR. diff --git a/crates/omnigraph/src/loader/mod.rs b/crates/omnigraph/src/loader/mod.rs index c52bdbe3d..c86d32b39 100644 --- a/crates/omnigraph/src/loader/mod.rs +++ b/crates/omnigraph/src/loader/mod.rs @@ -115,7 +115,6 @@ impl Omnigraph { branch: &str, base: Option<&str>, ) -> Result<(Option, Option)> { - crate::db::ensure_public_branch_ref(branch, "load")?; // Branch convention: `None` represents `main`. A requested base keeps // the explicit "main" spelling because it is also returned in the DTO. let requested = Self::normalize_branch_name(branch)?; @@ -356,10 +355,6 @@ impl Omnigraph { input_shape: LoadInputShape, stage_write_concurrency: usize, ) -> Result { - // The pending schema-contract install precedes both an implicit - // target-branch fork and data staging. - let completed_prior_work = self.settle_pending_schema_install().await?; - // Schema/catalog authority is captured once via the `WriteTxn` (plus its // cheap trailing identity-marker fence); the only second full validation // is the required pre-effect recheck under gates. Per-table resolution @@ -371,13 +366,7 @@ impl Omnigraph { let exists = self .branch_list() .await - .map_err(|error| { - if completed_prior_work { - error.without_pre_effect_evidence() - } else { - error.before_effect() - } - })? + .map_err(OmniError::before_effect)? .iter() .any(|name| name == target); if !exists { @@ -392,14 +381,7 @@ impl Omnigraph { target, actor_id, ) - .await - .map_err(|error| { - if completed_prior_work { - error.without_pre_effect_evidence() - } else { - error - } - })?; + .await?; branch_created = true; // DST window (loader walk D1 → D2): the implicit fork is // durable, the load has not begun. @@ -420,7 +402,7 @@ impl Omnigraph { ) .await .map_err(|error| { - if completed_prior_work || branch_created { + if branch_created { error.without_pre_effect_evidence() } else { error @@ -574,7 +556,7 @@ async fn load_jsonl_data( branch = branch.unwrap_or("main"), "prepared load authority changed before effects; repreparing" ); - db.refresh_for_reprepare().await?; + db.refresh_coordinator_only().await?; } result => return result, } diff --git a/crates/omnigraph/src/runtime_cache.rs b/crates/omnigraph/src/runtime_cache.rs index 50f497844..d6a4169ed 100644 --- a/crates/omnigraph/src/runtime_cache.rs +++ b/crates/omnigraph/src/runtime_cache.rs @@ -10,7 +10,7 @@ use omnigraph_core::handle_cache::LruMap; use sha2::{Digest, Sha256}; use tokio::sync::Mutex; -use crate::db::{ResolvedTarget, Snapshot}; +use crate::db::{ResolvedTarget, SchemaContractIdentity, Snapshot}; use crate::error::Result; use crate::graph_index::{GraphIndex, persist}; @@ -276,43 +276,57 @@ pub struct ReadCaches { pub session: Arc, pub handles: Arc, /// The accepted catalog built by - /// `Omnigraph::build_accepted_catalog_with_schema_gate_held`, memoized on - /// the contract's exact bytes. + /// `Omnigraph::accepted_catalog_for_snapshot`, memoized on the + /// exact captured manifest image and validated contract row. pub accepted_catalog: AcceptedCatalogMemo, /// Named queries compiled against an accepted catalog. pub compiled_queries: CompiledQueryCache, } -/// Memo of the last accepted catalog, keyed on the contract's byte-exact text -/// (`SchemaContractText`), which the caller reads on every call, so a -/// rewritten contract misses. One entry: a graph has one accepted contract at -/// a time. +/// One accepted row and catalog, with the immutable image that was validated. #[derive(Default)] pub struct AcceptedCatalogMemo { inner: std::sync::Mutex>>, } -struct AcceptedCatalogEntry { - text: crate::db::SchemaContractText, - catalog: Arc, +pub(crate) struct AcceptedCatalogEntry { + pub snapshot: Snapshot, + pub row: omnigraph_catalog::SchemaContractRow, + pub identity: SchemaContractIdentity, + pub catalog: Arc, } impl AcceptedCatalogMemo { - pub(crate) fn get(&self, text: &crate::db::SchemaContractText) -> Option> { - let entry = self - .inner + pub(crate) fn current(&self) -> Option> { + self.inner .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .clone()?; - (entry.text == *text).then(|| Arc::clone(&entry.catalog)) + .clone() + } + + #[cfg(test)] + pub(crate) fn get(&self, identity: &SchemaContractIdentity) -> Option> { + let entry = self.current()?; + (entry.identity == *identity).then(|| Arc::clone(&entry.catalog)) } - pub(crate) fn memoize(&self, text: crate::db::SchemaContractText, catalog: Arc) { + pub(crate) fn memoize( + &self, + snapshot: Snapshot, + row: omnigraph_catalog::SchemaContractRow, + catalog: Arc, + ) -> Arc { + let entry = Arc::new(AcceptedCatalogEntry { + identity: SchemaContractIdentity::from(&row.head), + snapshot, + row, + catalog, + }); *self .inner .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = - Some(Arc::new(AcceptedCatalogEntry { text, catalog })); + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::clone(&entry)); + entry } } diff --git a/crates/omnigraph/src/seams/catalog.rs b/crates/omnigraph/src/seams/catalog.rs index 961629003..3d9092481 100644 --- a/crates/omnigraph/src/seams/catalog.rs +++ b/crates/omnigraph/src/seams/catalog.rs @@ -28,15 +28,13 @@ omnigraph_seams::catalog! { crate::db::upgrade::UPGRADE_AFTER_BRANCH, crate::db::upgrade::UPGRADE_AFTER_FENCE, crate::db::upgrade::UPGRADE_AFTER_STAGE, + crate::db::upgrade::UPGRADE_AFTER_SCHEMA_FILE_DELETE, crate::db::upgrade::UPGRADE_DETACHED_ONLY_BETWEEN_REAPS, crate::db::upgrade::UPGRADE_BEFORE_ACTIVATION, crate::db::omnigraph::BRANCH_CONTROL_PRE_GATES, crate::db::omnigraph::BRANCH_DELETE_POST_TABLE_GATES, crate::db::omnigraph::CHANGE_FEED_POST_CAPTURE, crate::db::omnigraph::INIT_AFTER_COORDINATOR_INIT, - crate::db::omnigraph::INIT_AFTER_SCHEMA_CONTRACT_WRITTEN, - crate::db::omnigraph::INIT_AFTER_SCHEMA_PG_WRITTEN, - crate::db::omnigraph::INIT_SCHEMA_CLEANUP_DELETE, crate::db::omnigraph::LOCAL_CREATE_IF_ABSENT_PROBE, crate::db::omnigraph::OPEN_BEFORE_SCHEMA_CONTRACT_READ, crate::db::omnigraph::SCHEMA_RELOAD_BEFORE_CONTRACT_READ, @@ -50,10 +48,7 @@ omnigraph_seams::catalog! { crate::db::omnigraph::optimize::OPTIMIZE_POST_TABLE_EFFECT, crate::db::omnigraph::optimize::OPTIMIZE_POST_PHASE_B_PRE_MANIFEST_COMMIT, crate::db::omnigraph::schema_apply::SCHEMA_APPLY_AFTER_MANIFEST_COMMIT, - crate::db::omnigraph::schema_apply::SCHEMA_APPLY_AFTER_STAGING_WRITE, - crate::db::omnigraph::schema_apply::SCHEMA_APPLY_BEFORE_STAGING_WRITE, crate::db::omnigraph::schema_apply::SCHEMA_APPLY_POST_LOCK_PRE_EFFECT, - crate::db::omnigraph::schema_apply::SCHEMA_APPLY_POST_SENTINEL, crate::db::omnigraph::schema_apply::SCHEMA_APPLY_POST_TABLE_COMMIT, crate::db::omnigraph::table_ops::ENSURE_INDICES_POST_PHASE_B_PRE_MANIFEST_COMMIT, crate::db::omnigraph::table_ops::ENSURE_INDICES_POST_STAGE_PRE_COMMIT_BTREE, diff --git a/crates/omnigraph/src/storage_layer.rs b/crates/omnigraph/src/storage_layer.rs index 6eae83b5c..d0401ddc3 100644 --- a/crates/omnigraph/src/storage_layer.rs +++ b/crates/omnigraph/src/storage_layer.rs @@ -401,6 +401,13 @@ pub trait TableStorage: sealed::Sealed + Send + Sync + Debug { branch: Option<&str>, ) -> Result; + /// Validate an original empty create and compare its complete physical schema. + async fn validate_initial_empty_table( + &self, + snapshot: &SnapshotHandle, + desired: &SchemaRef, + ) -> Result; + /// Native identity of the branch backing an already-open snapshot. Branch /// merge uses it to bind a proven source interval and its target to the /// exact native ref incarnation, closing delete/recreate ABA. @@ -854,6 +861,14 @@ impl TableStorage for TableStore { .map(SnapshotHandle::new) } + async fn validate_initial_empty_table( + &self, + snapshot: &SnapshotHandle, + desired: &SchemaRef, + ) -> Result { + TableStore::validate_initial_empty_table(self, snapshot.dataset(), desired).await + } + async fn branch_identifier( &self, snapshot: &SnapshotHandle, diff --git a/crates/omnigraph/src/table_store.rs b/crates/omnigraph/src/table_store.rs index ee6441539..5e5710df0 100644 --- a/crates/omnigraph/src/table_store.rs +++ b/crates/omnigraph/src/table_store.rs @@ -1439,6 +1439,42 @@ impl TableStore { } } + /// Validate an original empty create for a non-destructive schema-apply retry. + /// Returns whether its complete Arrow schema already matches the desired one. + pub async fn validate_initial_empty_table( + &self, + ds: &Dataset, + desired: &SchemaRef, + ) -> Result { + if ds.version().version != 1 + || !ds.manifest().uses_stable_row_ids() + || ds.manifest().data_storage_format.lance_file_format() + != LanceFileVersion::V2_2.resolve() + || ds + .manifest() + .fragments + .iter() + .any(|fragment| fragment.physical_rows != Some(0)) + { + return Err(OmniError::manifest_conflict(format!( + "schema apply cannot reuse '{}': expected an original empty version-one table", + ds.uri() + ))); + } + let transaction = ds.read_transaction().await.map_err(OmniError::storage)?; + if !transaction.is_some_and(|transaction| { + transaction.read_version == 0 + && matches!(transaction.operation, Operation::Overwrite { .. }) + }) { + return Err(OmniError::manifest_conflict(format!( + "schema apply cannot reuse '{}': version one is not an original create", + ds.uri() + ))); + } + let actual = arrow_schema::Schema::from(ds.schema()); + Ok(&actual == desired.as_ref()) + } + /// List the named Lance branches present on the dataset at `dataset_uri`. /// The `cleanup` orphan reconciler diffs this against the manifest branch /// set to find orphaned per-table forks. `main`/default is not a named @@ -6339,6 +6375,62 @@ mod tests { RecordBatch::try_new(schema, vec![col]).unwrap() } + #[tokio::test] + async fn schema_retry_refuses_noninitial_or_nonempty_tables_without_mutation() { + for case in ["nonempty", "advanced", "unstable", "wrong-format"] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + let uri = format!("{root}/candidate"); + let store = TableStore::new(root, Arc::new(lance::session::Session::default())); + let batch = batch_with_ids(if case == "nonempty" { + &["retained"] + } else { + &[] + }); + let desired = batch.schema(); + let params = WriteParams { + enable_stable_row_ids: case != "unstable", + data_storage_version: Some(if case == "wrong-format" { + LanceFileVersion::V2_1 + } else { + LanceFileVersion::V2_2 + }), + auto_cleanup: None, + skip_auto_cleanup: true, + ..Default::default() + }; + let reader = arrow_array::RecordBatchIterator::new(vec![Ok(batch)], desired.clone()); + let mut dataset = Dataset::write(reader, &uri, Some(params)).await.unwrap(); + if case == "advanced" { + dataset + .update_config([("retry-probe", Some("1"))]) + .await + .unwrap(); + } + let manifest_path = format!("/{}", dataset.manifest_location().path); + let bytes = std::fs::read(&manifest_path).unwrap(); + let version = dataset.version().version; + let error = store + .validate_initial_empty_table(&dataset, &desired) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("original empty version-one table"), + "{case}: {error}" + ); + assert_eq!(std::fs::read(&manifest_path).unwrap(), bytes, "{case}"); + let reopened = Dataset::open(&uri).await.unwrap(); + assert_eq!(reopened.version().version, version, "{case}"); + assert_eq!( + reopened.count_rows(None).await.unwrap(), + usize::from(case == "nonempty"), + "{case}" + ); + } + } + #[test] fn staging_witness_requires_complete_canonical_authority() { use lance::dataset::refs::BranchIdentifier; diff --git a/crates/omnigraph/tests/detached_commit_matrix.rs b/crates/omnigraph/tests/detached_commit_matrix.rs index 81f037d36..8eac38b8f 100644 --- a/crates/omnigraph/tests/detached_commit_matrix.rs +++ b/crates/omnigraph/tests/detached_commit_matrix.rs @@ -63,7 +63,7 @@ enum Writer { /// a pointer switch with no table effect, so only its pre-publish window. Merge, /// Schema apply adding a nullable Person property: one detached rewrite - /// of Person, no row change, the contract staged and installed. + /// of Person, no row change, the contract row published atomically. SchemaApply, /// Optimize over a Person table with four small fragments: one detached /// compaction rewrite, no row change, published with an exact CAS on the @@ -167,7 +167,7 @@ impl Window { Window::PostDetached(n) if schema => { (catalog::SCHEMA_APPLY_POST_TABLE_COMMIT.name(), n as u64) } - Window::PrePublish if schema => (catalog::SCHEMA_APPLY_AFTER_STAGING_WRITE.name(), 1), + Window::PrePublish if schema => (catalog::GRAPH_PUBLISH_BEFORE_COMMIT_APPEND.name(), 1), Window::PrePublish if merge => ( catalog::BRANCH_MERGE_POST_PHASE_B_PRE_MANIFEST_COMMIT.name(), 1, @@ -667,22 +667,23 @@ async fn run_cell( } if fault == Fault::Race { let race_name = format!("m{index}_race"); + insert(&db, &race_name).await.unwrap(); + model.names.insert(race_name); + std::fs::write(barrier.join("go"), b"1").unwrap(); + let out = child.wait_with_output().unwrap(); + acknowledged = out.status.success(); if matches!(writer, Writer::SchemaApply | Writer::SystemColumnUpgrade) { - // The apply's durable sentinel refuses every concurrent - // writer of the graph while it is in flight (the upgrade - // runs through the same sentinel). - let refused = insert(&db, &race_name).await.unwrap_err(); assert!( - refused.to_string().contains("schema apply"), - "{cell}: the sentinel must refuse the racer: {refused}" + !acknowledged, + "{cell}: the schema writer must lose to the concurrent publication" + ); + assert!( + String::from_utf8_lossy(&out.stdout).contains("write authority"), + "{cell}: expected a stale graph-head refusal, stdout: {}", + String::from_utf8_lossy(&out.stdout) ); - } else { - insert(&db, &race_name).await.unwrap(); - model.names.insert(race_name); } - std::fs::write(barrier.join("go"), b"1").unwrap(); - let out = child.wait_with_output().unwrap(); - acknowledged = out.status.success(); + let stdout = String::from_utf8_lossy(&out.stdout).to_string(); if let Some(err) = stdout.lines().find(|line| line.starts_with("CHILD_ERR")) { note = err.chars().take(90).collect(); diff --git a/crates/omnigraph/tests/end_to_end.rs b/crates/omnigraph/tests/end_to_end.rs index 85ade5a79..0ccea210e 100644 --- a/crates/omnigraph/tests/end_to_end.rs +++ b/crates/omnigraph/tests/end_to_end.rs @@ -36,7 +36,7 @@ async fn init_creates_schema_file_and_manifest() { let db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - assert!(dir.path().join("_schema.pg").exists()); + assert!(!dir.path().join("_schema.pg").exists()); assert!(dir.path().join("__manifest").exists()); assert_eq!(db.catalog().node_types.len(), 2); assert_eq!(db.catalog().edge_types.len(), 2); @@ -1499,13 +1499,7 @@ async fn blob_read_on_upgraded_unmarked_v6_table_fails_closed_for_old_snapshots( .unwrap() .published_dataset_version; drop(db); - let mut manifest = lance::Dataset::open(&format!("{uri}/__manifest")) - .await - .unwrap(); - omnigraph_catalog::migrations::restamp_flat_for_test(&mut manifest, 10) - .await - .unwrap(); - drop(manifest); + helpers::make_legacy_flat_manifest_fixture(dir.path(), 10).await; let upgraded = upgrade_storage(uri, UpgradeOptions::default()) .await .unwrap(); diff --git a/crates/omnigraph/tests/failpoints.rs b/crates/omnigraph/tests/failpoints.rs index cbc119cf5..ba990ea82 100644 --- a/crates/omnigraph/tests/failpoints.rs +++ b/crates/omnigraph/tests/failpoints.rs @@ -123,7 +123,9 @@ fn run_rfc023_external_writer_action( LoadMode::Merge => "merge", LoadMode::Overwrite => "overwrite", }; - let output = Command::new(std::env::current_exe().map_err(|error| error.to_string())?) + let stdout = tempfile::NamedTempFile::new().map_err(|error| error.to_string())?; + let stderr = tempfile::NamedTempFile::new().map_err(|error| error.to_string())?; + let mut child = Command::new(std::env::current_exe().map_err(|error| error.to_string())?) .arg("--exact") .arg("rfc023_external_writer_process") .arg("--ignored") @@ -133,16 +135,43 @@ fn run_rfc023_external_writer_action( .env(RFC023_EXTERNAL_MODE_ENV, mode) .env(RFC023_EXTERNAL_PAYLOAD_ENV, payload) .env(RFC023_EXTERNAL_ACTION_ENV, action) - .output() + .stdout(stdout.reopen().map_err(|error| error.to_string())?) + .stderr(stderr.reopen().map_err(|error| error.to_string())?) + .spawn() .map_err(|error| error.to_string())?; - if output.status.success() { + let deadline = std::time::Instant::now() + + std::time::Duration::from_secs(if action == "schema_apply_company" { + 15 + } else { + 45 + }); + loop { + if child + .try_wait() + .map_err(|error| error.to_string())? + .is_some() + { + break; + } + if std::time::Instant::now() >= deadline { + child.kill().map_err(|error| error.to_string())?; + child.wait().map_err(|error| error.to_string())?; + return Err(format!( + "external writer timed out: {}", + std::fs::read_to_string(stderr.path()).unwrap_or_default() + )); + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + let status = child.wait().map_err(|error| error.to_string())?; + if status.success() { return Ok(()); } Err(format!( "external writer failed with {}\nstdout:\n{}\nstderr:\n{}", - output.status, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), + status, + std::fs::read_to_string(stdout.path()).unwrap_or_default(), + std::fs::read_to_string(stderr.path()).unwrap_or_default(), )) } @@ -181,6 +210,15 @@ fn rfc023_external_writer_process() { "load" => { db.load("main", &payload, mode).await.unwrap(); } + "schema_apply_company" => { + db.apply_schema(&payload).await.unwrap(); + let data = if payload.contains("slug: String") { + r#"{"type":"Company","data":{"slug":"winner"}}"# + } else { + r#"{"type":"Company","data":{"name":"winner"}}"# + }; + db.load_jsonl(data, LoadMode::Append).await.unwrap(); + } "collector_late_live" | "collector_late_retired" => { db.branch_create("collector-late").await.unwrap(); db.load("collector-late", &payload, mode).await.unwrap(); @@ -2070,25 +2108,20 @@ async fn read_capture_proceeds_while_writer_parked() { assert_eq!(count_rows(&db_b, "node:Person").await, 5); } -// Atomic schema apply: schema apply writes staging files first, then commits -// the manifest, then renames staging → final. Tests below inject crashes at -// the two boundaries and assert that reopening the graph yields a consistent -// state. - #[tokio::test] #[serial] -async fn schema_apply_pre_commit_crash_discards_staging_on_reopen() { +async fn schema_apply_pre_commit_crash_preserves_published_contract() { let _scenario = FailScenario::setup(); let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap().to_string(); { let db = helpers::session(Omnigraph::init(&uri, SCHEMA_V1).await.unwrap()); - let _failpoint = catalog::SCHEMA_APPLY_AFTER_STAGING_WRITE.fire_always(); + let _failpoint = catalog::GRAPH_PUBLISH_BEFORE_COMMIT_APPEND.fire_always(); let err = db.apply_schema(SCHEMA_V2_ADDED_TYPE).await.unwrap_err(); assert!( err.to_string() - .contains("injected failpoint triggered: schema_apply.after_staging_write"), + .contains("injected failpoint triggered: graph_publish.before_commit_append"), "got: {}", err ); @@ -2097,15 +2130,9 @@ async fn schema_apply_pre_commit_crash_discards_staging_on_reopen() { "a failure before publication needs no recovery: {err}" ); } - assert!( - dir.path().join("__schema_state.json.staging").exists(), - "the staged contract outlives the writer" - ); + assert_no_staging_files(dir.path()); assert_no_recovery_sidecars(dir.path()); - // RFC 0067: the staged contract names a graph commit that never landed, - // so the next read-write open discards it; the created Company dataset - // is unregistered garbage at the path the retry creates at. let db = helpers::session(Omnigraph::open(&uri).await.unwrap()); assert_eq!(db.schema_source().as_str(), SCHEMA_V1); assert_no_staging_files(dir.path()); @@ -2120,32 +2147,51 @@ async fn schema_apply_pre_commit_crash_discards_staging_on_reopen() { let company_uri = unregistered_node_table_uri(&db).await; assert!( std::path::Path::new(&company_uri).exists(), - "the abandoned create stays as unregistered garbage until the retry reclaims it" + "the abandoned create remains available for a non-destructive retry" ); db.apply_schema(SCHEMA_V2_ADDED_TYPE) .await - .expect("the retry reclaims the leftover and publishes"); + .expect("the retry reuses the leftover and publishes"); assert_eq!(helpers::count_rows(&db, "node:Company").await, 0); assert_eq!(node_table_uri(&db, "Company").await, company_uri); assert_no_staging_files(dir.path()); assert_no_recovery_sidecars(dir.path()); } -#[tokio::test] -#[serial] -async fn schema_apply_recovers_partial_schema_promotion_after_commit_crash() { - let _scenario = FailScenario::setup(); - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap().to_string(); +/// A published contract needs neither root schema files nor recovery sidecars. +async fn assert_no_schema_apply_residue(uri: &str) { + let storage = omnigraph_storage::storage_for_uri(uri).unwrap(); + for name in [ + "_schema.pg", + "_schema.ir.json", + "__schema_state.json", + "_schema.pg.staging", + "_schema.ir.json.staging", + "__schema_state.json.staging", + ] { + let path = omnigraph_storage::join_uri(uri, name); + assert!( + !storage.exists(&path).await.unwrap(), + "atomic schema publication must not leave {path}" + ); + } + let recovery = omnigraph_storage::join_uri(uri, "__recovery"); + assert!( + storage.list_dir(&recovery).await.unwrap().is_empty(), + "atomic schema publication must not leave recovery sidecars" + ); +} +/// Both backend wrappers exercise publication, fresh admission and live-handle progress. +async fn assert_schema_apply_post_commit_failure_leaves_readable_contract(uri: &str) { + let db = helpers::session(Omnigraph::init(uri, SCHEMA_V1).await.unwrap()); { - let db = helpers::session(Omnigraph::init(&uri, SCHEMA_V1).await.unwrap()); let _failpoint = catalog::SCHEMA_APPLY_AFTER_MANIFEST_COMMIT.fire_always(); let err = db.apply_schema(SCHEMA_V2_ADDED_TYPE).await.unwrap_err(); assert!( matches!(err, OmniError::RecoveryRequired { .. }), - "a failure after publication reports the pending contract installation: {err}" + "a failure after publication identifies the committed operation: {err}" ); assert!( err.to_string() @@ -2154,60 +2200,57 @@ async fn schema_apply_recovers_partial_schema_promotion_after_commit_crash() { err ); } - assert_no_recovery_sidecars(dir.path()); + assert_no_schema_apply_residue(uri).await; - // ReadOnly must remain non-mutating, but it also must not combine the - // already-published manifest delta with the old live schema contract. - // It fails closed until a read-write open performs the promotion. - let read_only_error = match Omnigraph::open_read_only(&uri).await { - Ok(_) => panic!("read-only open must refuse a committed-but-unpromoted SchemaApply"), - Err(error) => error, - }; + let read_only = Omnigraph::open_read_only(uri).await.unwrap(); + assert_eq!(read_only.schema_source().as_str(), SCHEMA_V2_ADDED_TYPE); assert!( - matches!(read_only_error, OmniError::RecoveryRequired { .. }), - "{read_only_error}" - ); - assert!( - read_only_error - .to_string() - .contains("schema contract promotion is pending"), - "{read_only_error}" - ); - assert!(dir.path().join("_schema.pg.staging").exists()); - assert!(dir.path().join("_schema.ir.json.staging").exists()); - assert!(dir.path().join("__schema_state.json.staging").exists()); - assert_eq!( - std::fs::read_to_string(dir.path().join("_schema.pg")).unwrap(), - SCHEMA_V1, - "the read-only coherence guard must not promote schema files" + read_only + .snapshot_of(ReadTarget::branch("main")) + .await + .unwrap() + .dataset("node:Company") + .is_some() ); + assert_eq!(helpers::count_rows(&read_only, "node:Company").await, 0); + let fresh = helpers::session(Omnigraph::open(uri).await.unwrap()); + assert_eq!(fresh.schema_source().as_str(), SCHEMA_V2_ADDED_TYPE); + assert_eq!(helpers::count_rows(&fresh, "node:Company").await, 0); + assert_no_schema_apply_residue(uri).await; - // Simulate a crash partway through promotion: source reached its final - // name, while the IR/state contract remains staged. Recovery must - // validate the mixed state as one target identity and finish it. - std::fs::rename( - dir.path().join("_schema.pg.staging"), - dir.path().join("_schema.pg"), + db.load_jsonl( + r#"{"type":"Company","data":{"name":"SameHandle"}}"#, + LoadMode::Merge, ) - .unwrap(); - assert!(!dir.path().join("_schema.pg.staging").exists()); - assert!(dir.path().join("_schema.ir.json.staging").exists()); - assert!(dir.path().join("__schema_state.json.staging").exists()); - - // Reopen: the publishing commit is in lineage, so recovery completes the - // remaining promotion and the live schema matches v2. - let db = helpers::session(Omnigraph::open(&uri).await.unwrap()); - assert_eq!(db.schema_source().as_str(), SCHEMA_V2_ADDED_TYPE); - assert_no_staging_files(dir.path()); - assert_eq!(helpers::count_rows(&db, "node:Company").await, 0); - db.apply_schema(SCHEMA_V2_ADDED_TYPE) + .await + .expect("the same handle writes the published new type after the fault clears"); + fresh + .load_jsonl( + r#"{"type":"Company","data":{"name":"FreshHandle"}}"#, + LoadMode::Merge, + ) .await - .expect("the reclaimed sentinel admits the next apply, a no-op here"); + .expect("a fresh handle writes the published new type after the fault clears"); + let mut names = collect_column_strings(&read_table(&read_only, "node:Company").await, "name"); + names.sort(); + assert_eq!(names, vec!["FreshHandle", "SameHandle"]); + fresh + .apply_schema(SCHEMA_V2_ADDED_TYPE) + .await + .expect("the next apply recognizes the published contract"); + assert_no_schema_apply_residue(uri).await; +} + +#[tokio::test] +#[serial] +async fn schema_apply_post_commit_failure_leaves_readable_contract() { + let _scenario = FailScenario::setup(); + let dir = tempfile::tempdir().unwrap(); + assert_schema_apply_post_commit_failure_leaves_readable_contract(dir.path().to_str().unwrap()) + .await; } -/// The applying handle's coordinator observes the fixed manifest commit before -/// schema files and the catalog ArcSwap are promoted. Query capture joins the -/// schema gate so it cannot pair that new snapshot with the old catalog. +/// Query capture joins the schema gate through coherent publication. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn schema_apply_live_query_waits_for_coherent_schema_publication() { @@ -2295,96 +2338,7 @@ edge WorksAt: Person -> Company #[tokio::test] #[serial] -async fn schema_apply_recovers_partial_rename() { - // Construct a partial-rename state: _schema.pg has been renamed in - // (matching v2), but _schema.ir.json.staging and __schema_state.json.staging - // were never renamed. Recovery should detect that the live source matches - // the staging state's hash and complete the remaining renames. - let _scenario = FailScenario::setup(); - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap().to_string(); - - { - let db = helpers::session(Omnigraph::init(&uri, SCHEMA_V1).await.unwrap()); - db.apply_schema(SCHEMA_V2_ADDED_TYPE).await.unwrap(); - } - - // Simulate: one of the renames (the IR or state file) didn't complete by - // copying the live ir/state files back to their staging names. - std::fs::copy( - dir.path().join("_schema.ir.json"), - dir.path().join("_schema.ir.json.staging"), - ) - .unwrap(); - std::fs::copy( - dir.path().join("__schema_state.json"), - dir.path().join("__schema_state.json.staging"), - ) - .unwrap(); - - // Reopen — recovery should complete the rename (overwriting final files - // with identical staging content) and remove the staging files. - let db = helpers::session(Omnigraph::open(&uri).await.unwrap()); - assert_eq!(db.schema_source().as_str(), SCHEMA_V2_ADDED_TYPE); - assert_no_staging_files(dir.path()); -} - -/// Azure implements schema-contract promotion as GET -> completed PUT -> -/// DELETE. Model a crash after the destination PUT by leaving the identical -/// live and staging objects together, then prove the ordinary open-time -/// recovery completes the remaining source deletions. -#[tokio::test] -#[serial] -async fn azure_schema_apply_recovers_source_and_destination_after_partial_rename() { - let Ok(container) = std::env::var("OMNIGRAPH_AZURE_TEST_CONTAINER") else { - eprintln!( - "skipping Azure schema rename recovery: OMNIGRAPH_AZURE_TEST_CONTAINER is not set" - ); - return; - }; - let _scenario = FailScenario::setup(); - let uri = format!( - "az://{container}/engine-failpoints/schema-rename-{}", - ulid::Ulid::new() - ); - let storage = omnigraph_storage::storage_for_uri(&uri).unwrap(); - - { - let db = helpers::session(Omnigraph::init(&uri, SCHEMA_V1).await.unwrap()); - db.apply_schema(SCHEMA_V2_ADDED_TYPE).await.unwrap(); - } - - for name in ["_schema.ir.json", "__schema_state.json"] { - let live = format!("{uri}/{name}"); - let staging = format!("{live}.staging"); - let body = storage.read_text(&live).await.unwrap(); - storage.write_text(&staging, &body).await.unwrap(); - assert!(storage.exists(&live).await.unwrap()); - assert!(storage.exists(&staging).await.unwrap()); - } - - let reopened = helpers::session( - Omnigraph::open(&uri) - .await - .expect("Azure open must complete the interrupted schema-contract rename"), - ); - assert_eq!(reopened.schema_source().as_str(), SCHEMA_V2_ADDED_TYPE); - for name in ["_schema.ir.json", "__schema_state.json"] { - assert!(storage.exists(&format!("{uri}/{name}")).await.unwrap()); - assert!( - !storage - .exists(&format!("{uri}/{name}.staging")) - .await - .unwrap() - ); - } - drop(reopened); - storage.delete_prefix(&uri).await.unwrap(); -} - -#[tokio::test] -#[serial] -async fn schema_apply_retries_over_its_own_unpublished_staging_without_reopen() { +async fn schema_apply_retries_after_unpublished_effects_without_reopen() { let _scenario = FailScenario::setup(); let dir = tempfile::tempdir().unwrap(); let db = init_and_load(&dir).await; @@ -2394,7 +2348,7 @@ async fn schema_apply_retries_over_its_own_unpublished_staging_without_reopen() helpers::TEST_SCHEMA ); { - let _failpoint = catalog::SCHEMA_APPLY_AFTER_STAGING_WRITE.fire_always(); + let _failpoint = catalog::GRAPH_PUBLISH_BEFORE_COMMIT_APPEND.fire_always(); let err = db.apply_schema(&desired).await.unwrap_err(); assert!( !matches!(err, OmniError::RecoveryRequired { .. }), @@ -2402,7 +2356,7 @@ async fn schema_apply_retries_over_its_own_unpublished_staging_without_reopen() ); } assert_no_recovery_sidecars(dir.path()); - assert!(dir.path().join("__schema_state.json.staging").exists()); + assert_no_staging_files(dir.path()); db.apply_schema(&desired) .await @@ -2436,11 +2390,11 @@ async fn load_after_schema_apply_pre_publish_failure_keeps_the_accepted_catalog( schema_with_person_city() ); { - let _failpoint = catalog::SCHEMA_APPLY_AFTER_STAGING_WRITE.fire_always(); + let _failpoint = catalog::GRAPH_PUBLISH_BEFORE_COMMIT_APPEND.fire_always(); let err = db.apply_schema(&v2_schema).await.unwrap_err(); assert!( err.to_string() - .contains("injected failpoint triggered: schema_apply.after_staging_write"), + .contains("injected failpoint triggered: graph_publish.before_commit_append"), "unexpected error: {err}" ); } @@ -2469,18 +2423,9 @@ async fn load_after_schema_apply_pre_publish_failure_keeps_the_accepted_catalog( assert_eq!(helpers::count_rows(&reopened, "node:Person").await, 2); } -/// A concurrent write's entry heal must NOT promote a LIVE schema -/// apply's staging files. The apply pauses just after writing its -/// staging files (sidecar on disk from Phase A, staging on disk, -/// manifest not yet committed); a load on the same handle fires the -/// heal in that window. If the heal's schema-staging reconcile runs -/// unserialized, it promotes the staging files from under the live -/// apply — putting the NEW catalog live against the OLD manifest — and -/// the resumed apply's own renames then fail on the missing sources: -/// an error (and a corrupted catalog) for an otherwise-healthy apply. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] -async fn heal_does_not_promote_live_schema_apply_staging() { +async fn load_waits_for_schema_apply_publication() { use omnigraph::loader::LoadMode; use std::sync::Arc; @@ -2492,9 +2437,8 @@ async fn heal_does_not_promote_live_schema_apply_staging() { Omnigraph::init(&uri, helpers::TEST_SCHEMA).await.unwrap(), )); - // Park the apply right after its staging files land (its sidecar is - // already on disk from Phase A; the manifest commit has not run). - let rv = helpers::failpoint::Rendezvous::park_first(&catalog::SCHEMA_APPLY_AFTER_STAGING_WRITE); + let rv = + helpers::failpoint::Rendezvous::park_first(&catalog::GRAPH_PUBLISH_BEFORE_COMMIT_APPEND); let apply_db = Arc::clone(&db); let desired = format!( @@ -2503,17 +2447,9 @@ async fn heal_does_not_promote_live_schema_apply_staging() { ); let apply = tokio::spawn(async move { apply_db.apply_schema(&desired).await }); - // Wait until the apply is parked in the window (staging files written). rv.wait_until_reached().await; - let staging_pg = dir.path().join("_schema.pg.staging"); - assert!( - staging_pg.exists(), - "schema apply never reached the paused window" - ); + assert_no_staging_files(dir.path()); - // Concurrent load on the same handle: its entry heal runs while the - // apply is paused. The load itself may fail (schema apply in - // progress) — what matters is what its heal does to the live apply. let load_db = Arc::clone(&db); let load = tokio::spawn(async move { load_db @@ -2527,22 +2463,17 @@ async fn heal_does_not_promote_live_schema_apply_staging() { .await }); - // Give the load's heal time to act inside the window. Broken code - // completes the load here (its heal promoted the staging files and - // stole the apply's commit); fixed code leaves the load blocked on - // the schema-apply serialization key until the apply finishes. - tokio::time::sleep(std::time::Duration::from_millis(500)).await; + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert!(!load.is_finished(), "load waits for the schema gate"); rv.release(); let apply_result = apply.await.unwrap(); - let _ = tokio::time::timeout(std::time::Duration::from_secs(30), load) + tokio::time::timeout(std::time::Duration::from_secs(30), load) .await .expect("load must complete once the apply releases its guards") + .unwrap() .unwrap(); - apply_result.expect( - "a concurrent write's heal must not promote the live schema \ - apply's staging files out from under it", - ); + apply_result.expect("the apply publishes while the load waits for its schema gate"); // The migration landed and nothing recovery-shaped remains. assert_eq!(helpers::count_rows(&db, "node:Tag").await, 0); @@ -2749,11 +2680,11 @@ async fn schema_apply_pre_staging_failure_leaves_no_residue() { ); { let db = helpers::session(Omnigraph::open(&uri).await.unwrap()); - let _failpoint = catalog::SCHEMA_APPLY_BEFORE_STAGING_WRITE.fire_always(); + let _failpoint = catalog::GRAPH_PUBLISH_BEFORE_COMMIT_APPEND.fire_always(); let err = db.apply_schema(&v2_schema).await.unwrap_err(); assert!( err.to_string() - .contains("injected failpoint triggered: schema_apply.before_staging_write"), + .contains("injected failpoint triggered: graph_publish.before_commit_append"), "unexpected error: {err}" ); } @@ -2787,7 +2718,7 @@ async fn schema_apply_pre_staging_failure_leaves_no_residue() { "a detached rewrite never moves the linear HEAD" ); assert!(snapshot.dataset("node:Tag").is_none()); - let live_schema = std::fs::read_to_string(dir.path().join("_schema.pg")).unwrap(); + let live_schema = db.schema_source(); assert!(!live_schema.contains("city: String?"), "{live_schema}"); assert!(!live_schema.contains("node Tag"), "{live_schema}"); @@ -2808,11 +2739,11 @@ async fn metadata_only_schema_apply_before_staging_leaves_no_residue() { let indexed_schema = helpers::TEST_SCHEMA.replace("age: I32?", "age: I32? @index"); { let db = helpers::session(Omnigraph::init(&uri, helpers::TEST_SCHEMA).await.unwrap()); - let _failpoint = catalog::SCHEMA_APPLY_BEFORE_STAGING_WRITE.fire_always(); + let _failpoint = catalog::GRAPH_PUBLISH_BEFORE_COMMIT_APPEND.fire_always(); let err = db.apply_schema(&indexed_schema).await.unwrap_err(); assert!( err.to_string() - .contains("injected failpoint triggered: schema_apply.before_staging_write"), + .contains("injected failpoint triggered: graph_publish.before_commit_append"), "unexpected error: {err}" ); } @@ -2842,16 +2773,16 @@ async fn metadata_only_schema_apply_after_staging_discards_on_next_open() { let indexed_schema = helpers::TEST_SCHEMA.replace("age: I32?", "age: I32? @index"); { let db = helpers::session(Omnigraph::init(&uri, helpers::TEST_SCHEMA).await.unwrap()); - let _failpoint = catalog::SCHEMA_APPLY_AFTER_STAGING_WRITE.fire_always(); + let _failpoint = catalog::GRAPH_PUBLISH_BEFORE_COMMIT_APPEND.fire_always(); let err = db.apply_schema(&indexed_schema).await.unwrap_err(); assert!( err.to_string() - .contains("injected failpoint triggered: schema_apply.after_staging_write"), + .contains("injected failpoint triggered: graph_publish.before_commit_append"), "unexpected error: {err}" ); } assert_no_recovery_sidecars(dir.path()); - assert!(dir.path().join("__schema_state.json.staging").exists()); + assert_no_staging_files(dir.path()); // Metadata-only applies have no table effect: the staged contract is // their only durable state, and its recorded commit never landed. @@ -2884,58 +2815,17 @@ async fn metadata_only_schema_apply_post_publish_failure_heals_on_next_write() { let err = db.apply_schema(&indexed_schema).await.unwrap_err(); assert!( matches!(err, OmniError::RecoveryRequired { .. }), - "a failure after publication reports the pending contract installation: {err}" + "a failure after publication identifies the committed operation: {err}" + ); + assert_eq!( + err.completion_evidence(), + Some(CompletionEvidence::Uncertain), + "the post-publication failure retains uncertain completion: {err}" ); } - assert!(dir.path().join("__schema_state.json.staging").exists()); - assert_no_recovery_sidecars(dir.path()); - - // A validation-shaped error while completing this already-published - // contract is not a fresh mutation's ordinary pre-effect refusal. - let staging_path = dir.path().join("_schema.pg.staging"); - let staged_source = std::fs::read(&staging_path).unwrap(); - std::fs::write(&staging_path, "node {").unwrap(); - let blocked = mutate_main( - &db, - MUTATION_QUERIES, - "insert_person", - &mixed_params(&[("$name", "blocked")], &[("$age", 12)]), - ) - .await - .unwrap_err(); - std::fs::write(&staging_path, staged_source).unwrap(); - assert_eq!( - blocked.completion_evidence(), - Some(CompletionEvidence::Uncertain), - "published schema completion dominates a validation-shaped cause: {blocked}" - ); - - // Completion succeeds before native branch-name validation. The whole - // command cannot claim no effects, but its validation cause stays nonfatal. - let invalid_branch = db.branch_create_from("main", "bad?name").await.unwrap_err(); - assert_eq!(invalid_branch.completion_evidence(), None); - assert!( - matches!(invalid_branch, OmniError::Manifest(ref error) - if error.kind == omnigraph::error::ManifestErrorKind::BadRequest), - "completed pending work must leave an ordinary name refusal: {invalid_branch}" - ); - - // Completion succeeds before the next mutation's ordinary validation - // refusal. Prior successful work is neither an effect-free command nor - // evidence of uncertain completion. - let invalid = mutate_main(&db, MUTATION_QUERIES, "missing_mutation", ¶ms(&[])) - .await - .unwrap_err(); - assert_eq!( - invalid.completion_evidence(), - None, - "successful completion does not poison later validation: {invalid}" - ); assert_no_staging_files(dir.path()); - assert_eq!(helpers::count_rows(&db, "node:Person").await, 0); + assert_no_recovery_sidecars(dir.path()); - // The same handle now admits the ordinary write against the installed - // contract, with the dead apply's sentinel released. db.load_jsonl( "{\"type\":\"Person\",\"data\":{\"name\":\"alice\",\"age\":30}}\n", LoadMode::Append, @@ -2950,96 +2840,200 @@ async fn metadata_only_schema_apply_post_publish_failure_heals_on_next_write() { assert_eq!(helpers::count_rows(&db, "node:Person").await, 1); db.apply_schema(&indexed_schema) .await - .expect("the released sentinel admits the next apply, a no-op here"); + .expect("the next apply recognizes the published contract"); - let before_refusal = db.list_commits(None).await.unwrap(); - let refused = { - let _release_failure = catalog::BRANCH_DELETE_POST_ARCHIVE.fire_always(); - db.apply_schema("node {").await.unwrap_err() - }; - assert_eq!( - refused.completion_evidence(), - Some(CompletionEvidence::Uncertain), - "failed sentinel completion must dominate the primary validation refusal: {refused}" + let invalid_branch = db.branch_create_from("main", "bad?name").await.unwrap_err(); + let (invalid_branch, evidence) = invalid_branch.into_completion_evidence(); + assert_eq!(evidence, Some(CompletionEvidence::BeforeEffect)); + assert!( + matches!(invalid_branch, OmniError::Manifest(ref error) + if error.kind == ManifestErrorKind::BadRequest), + "the accepted contract leaves an ordinary name refusal: {invalid_branch}" ); - assert_eq!(db.list_commits(None).await.unwrap(), before_refusal); - db.apply_schema(&indexed_schema) + let invalid = mutate_main(&db, MUTATION_QUERIES, "missing_mutation", ¶ms(&[])) .await - .expect("the next apply completes the failed sentinel release on the same handle"); - - db.branch_create("blocks-schema").await.unwrap(); - let refused = { - let _release_failure = catalog::BRANCH_DELETE_POST_ARCHIVE.fire_always(); - db.apply_schema(&indexed_schema).await.unwrap_err() - }; + .unwrap_err(); assert_eq!( - refused.completion_evidence(), - Some(CompletionEvidence::Uncertain), - "failed sentinel completion must dominate the mono-branch refusal: {refused}" + invalid.completion_evidence(), + None, + "successful completion does not poison later validation: {invalid}" ); - assert_eq!(db.list_commits(None).await.unwrap(), before_refusal); - db.branch_delete("blocks-schema").await.unwrap(); - db.apply_schema(&indexed_schema) - .await - .expect("the mono-branch refusal's pending release also heals on the same handle"); + assert_no_staging_files(dir.path()); + assert_no_recovery_sidecars(dir.path()); + assert_eq!(helpers::count_rows(&db, "node:Person").await, 1); } #[tokio::test] #[serial] -async fn schema_apply_retry_reclaims_an_abandoned_add_type_dataset() { +async fn schema_apply_retry_preserves_the_original_empty_create() { let _scenario = FailScenario::setup(); - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap().to_string(); - let db = helpers::session(Omnigraph::init(&uri, SCHEMA_V1).await.unwrap()); - - { - let _failpoint = catalog::SCHEMA_APPLY_BEFORE_STAGING_WRITE.fire_always(); - db.apply_schema(SCHEMA_V2_ADDED_TYPE) - .await - .expect_err("the pre-staging failpoint must stop the apply after the create"); - } - let company_uri = unregistered_node_table_uri(&db).await; - assert!( - std::path::Path::new(&company_uri).exists(), - "the version-one create is durable before the failure" - ); - drop(db); - let recovered = helpers::session( - Omnigraph::open(&uri) - .await - .expect("an unregistered dataset is not recovery state"), - ); - assert!( + for changed in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap().to_string(); + let db = helpers::session(Omnigraph::init(&uri, SCHEMA_V1).await.unwrap()); + { + let _failpoint = catalog::GRAPH_PUBLISH_BEFORE_COMMIT_APPEND.fire_always(); + db.apply_schema(SCHEMA_V2_ADDED_TYPE).await.unwrap_err(); + } + let company_uri = unregistered_node_table_uri(&db).await; + let original = lance::Dataset::open(&company_uri).await.unwrap(); + let original_uuid = original.read_transaction().await.unwrap().unwrap().uuid; + let manifest_path = format!("/{}", original.manifest_location().path); + let original_bytes = std::fs::read(&manifest_path).unwrap(); + drop(db); + let recovered = helpers::session(Omnigraph::open(&uri).await.unwrap()); + let desired = if changed { + SCHEMA_V2_ADDED_TYPE.replace( + "node Company { name: String @key }", + "node Company { slug: String @key }", + ) + } else { + SCHEMA_V2_ADDED_TYPE.to_string() + }; + recovered.apply_schema(&desired).await.unwrap(); + assert_eq!(helpers::count_rows(&recovered, "node:Company").await, 0); + assert_eq!(node_table_uri(&recovered, "Company").await, company_uri); + let company = lance::Dataset::open(&company_uri).await.unwrap(); + assert_eq!(company.version().version, 1); + assert_eq!( + company.read_transaction().await.unwrap().unwrap().uuid, + original_uuid + ); + assert_eq!(std::fs::read(&manifest_path).unwrap(), original_bytes); + let snapshot = helpers::snapshot_main(&recovered).await.unwrap(); + let entry = snapshot.dataset("node:Company").unwrap(); + assert_eq!(entry.published_dataset_version, if changed { 2 } else { 1 }); + assert_eq!(entry.version_metadata.staged_version().is_some(), changed); + assert_eq!(entry.version_metadata.last_linear_version(), Some(1)); recovered - .snapshot_of(omnigraph::db::ReadTarget::branch("main")) + .repair(omnigraph::db::RepairOptions { + confirm: true, + force: false, + }) .await - .unwrap() - .dataset("node:Company") - .is_none(), - "nothing registers the orphan target" - ); - assert!( - std::path::Path::new(&company_uri).exists(), - "the open leaves the unregistered leftover for the retry" - ); - assert_no_recovery_sidecars(dir.path()); + .unwrap(); + recovered.ensure_indices().await.unwrap(); + let cleaned = recovered.cleanup(keep_one()).await.unwrap(); + assert!(cleaned.iter().all(|row| row.error.is_none()), "{cleaned:?}"); + drop(recovered); + let reopened = helpers::session(Omnigraph::open(&uri).await.unwrap()); + assert_eq!(reopened.schema_source().as_str(), desired); + let data = if changed { + r#"{"type":"Company","data":{"slug":"winner"}}"# + } else { + r#"{"type":"Company","data":{"name":"winner"}}"# + }; + reopened.load_jsonl(data, LoadMode::Append).await.unwrap(); + assert_eq!(helpers::count_rows(&reopened, "node:Company").await, 1); + let export = reopened.export_jsonl("main", &[]).await.unwrap(); + assert!(export.contains("winner")); + assert_eq!(std::fs::read(&manifest_path).unwrap(), original_bytes); + } +} - recovered - .apply_schema(SCHEMA_V2_ADDED_TYPE) +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn schema_apply_added_type_loser_preserves_a_concurrent_process_winner() { + let _scenario = FailScenario::setup(); + for changed in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap().to_string(); + let db = std::sync::Arc::new(helpers::session( + Omnigraph::init(&uri, SCHEMA_V1).await.unwrap(), + )); + let original_paths = std::fs::read_dir(dir.path().join("nodes")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .collect::>(); + let parked = + helpers::failpoint::Rendezvous::park_first(&catalog::SCHEMA_APPLY_POST_LOCK_PRE_EFFECT); + let loser = std::sync::Arc::clone(&db); + let apply = tokio::spawn(async move { loser.apply_schema(SCHEMA_V2_ADDED_TYPE).await }); + parked.wait_until_reached().await; + let winner_source = if changed { + SCHEMA_V2_ADDED_TYPE.replace( + "node Company { name: String @key }", + "node Company { slug: String @key }", + ) + } else { + SCHEMA_V2_ADDED_TYPE.to_string() + }; + let child_uri = uri.clone(); + let child_source = winner_source.clone(); + tokio::task::spawn_blocking(move || { + run_rfc023_external_writer_action( + child_uri, + LoadMode::Append, + child_source, + "schema_apply_company", + ) + }) .await - .expect("the retry reclaims the leftover under the sentinel and publishes"); - assert_eq!( - helpers::count_rows(&recovered, "node:Company").await, - 0, - "the retried AddType must be registered and queryable" - ); - assert_eq!(node_table_uri(&recovered, "Company").await, company_uri); - let company = lance::Dataset::open(&company_uri).await.unwrap(); - assert_eq!( - company.version().version, - 1, - "the reclaimed path holds the retry's own version-one create" - ); + .unwrap() + .unwrap(); + let company_path = std::fs::read_dir(dir.path().join("nodes")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| !original_paths.contains(path)) + .unwrap(); + let company_uri = company_path.to_str().unwrap(); + let before = lance::Dataset::open(company_uri).await.unwrap(); + let manifest_path = format!("/{}", before.manifest_location().path); + let original_bytes = std::fs::read(&manifest_path).unwrap(); + let paths_before_loser = std::fs::read_dir(dir.path().join("nodes")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .collect::>(); + parked.release(); + let error = tokio::time::timeout(std::time::Duration::from_secs(15), apply) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert!( + !parked.timed_out(), + "the winner must finish before the loser resumes" + ); + assert!( + !matches!(error, OmniError::RecoveryRequired { .. }), + "{error}" + ); + assert!(error.is_read_set_changed(), "{error}"); + let paths_after_loser = std::fs::read_dir(dir.path().join("nodes")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .collect::>(); + assert_eq!( + paths_after_loser, paths_before_loser, + "both applies must use the same allocated table path" + ); + assert_eq!(std::fs::read(&manifest_path).unwrap(), original_bytes); + assert_eq!( + lance::Dataset::open(company_uri) + .await + .unwrap() + .version() + .version, + 1 + ); + let reopened = helpers::session(Omnigraph::open(&uri).await.unwrap()); + assert_eq!(reopened.schema_source().as_str(), winner_source); + assert_eq!(helpers::count_rows(&reopened, "node:Company").await, 1); + assert!( + reopened + .export_jsonl("main", &[]) + .await + .unwrap() + .contains("winner") + ); + let data = if changed { + r#"{"type":"Company","data":{"slug":"after"}}"# + } else { + r#"{"type":"Company","data":{"name":"after"}}"# + }; + reopened.load_jsonl(data, LoadMode::Append).await.unwrap(); + assert_eq!(helpers::count_rows(&reopened, "node:Company").await, 2); + } } #[tokio::test] @@ -3191,10 +3185,7 @@ async fn schema_apply_partial_table_effect_leaves_no_residue() { ); } -/// A concurrent publication on main between the staged effects and the -/// manifest commit: the apply loses its graph-head CAS, which is a plain -/// refusal before publication. Its detached rewrite and staged contract are -/// garbage, the winner is untouched, and the retry plans from the winner. +/// A concurrent graph publication makes the schema writer lose its head CAS. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn schema_apply_loses_the_manifest_cas_to_a_concurrent_publication_without_residue() { @@ -3208,7 +3199,7 @@ async fn schema_apply_loses_the_manifest_cas_to_a_concurrent_publication_without let schema_db = std::sync::Arc::new(helpers::session(Omnigraph::open(&uri).await.unwrap())); let winner_db = helpers::session(Omnigraph::open(&uri).await.unwrap()); let rendezvous = - helpers::failpoint::Rendezvous::park_first(&catalog::SCHEMA_APPLY_AFTER_STAGING_WRITE); + helpers::failpoint::Rendezvous::park_first(&catalog::GRAPH_PUBLISH_BEFORE_COMMIT_APPEND); let desired = schema_with_person_city(); let apply_handle = std::sync::Arc::clone(&schema_db); let apply_task = tokio::spawn(async move { apply_handle.apply_schema(&desired).await }); @@ -4342,91 +4333,6 @@ async fn ensure_indices_phase_b_failure_does_not_leak_sidecar_when_no_work_neede ); } -// ─── MR-668 PR 2a: Omnigraph::init cleanup on partial failure ────────────── -// -// `init_with_storage` writes three schema artifacts before invoking -// `GraphCoordinator::init`. Without cleanup, a failure between any of those -// steps left orphan files behind, making the URI unusable for a retry of -// `init` (it would refuse because `_schema.pg` already exists). The tests -// below pin: on failpoint trigger at the two pre-commit phase boundaries, -// the three schema files are removed before the error is returned. -// -// The third boundary (`init.after_coordinator_init`) sits past the graph's -// commit point, where the cleanup must not run (issue #495 — deleting the -// schema files there left a graph that could neither open nor re-init). -// Its test asserts the graph survives an error at that window. -// -// Coverage note: orphan Lance directories after a failure DURING -// `GraphCoordinator::init` are a known limitation — see the coverage-gap -// comment in `init_with_storage`. - -#[tokio::test] -#[serial] -async fn init_failpoint_after_schema_pg_written_cleans_up_schema_file() { - let _scenario = FailScenario::setup(); - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap(); - let _failpoint = catalog::INIT_AFTER_SCHEMA_PG_WRITTEN.fire_always(); - - let err = match Omnigraph::init(uri, helpers::TEST_SCHEMA).await { - Ok(_) => panic!("expected Omnigraph::init to fail at the configured failpoint"), - Err(e) => e, - }; - assert!( - err.to_string() - .contains("injected failpoint triggered: init.after_schema_pg_written"), - "got: {err}" - ); - - // Only `_schema.pg` was written at this phase boundary, but the - // cleanup attempts all three — `delete` treats not-found as Ok, - // so the other two deletes are no-ops. - assert!( - !dir.path().join("_schema.pg").exists(), - "_schema.pg must be cleaned up after init failure" - ); - assert!( - !dir.path().join("__init_claim.json").exists(), - "pre-create failure must release the init claim after cleanup" - ); -} - -#[tokio::test] -#[serial] -async fn init_failpoint_after_schema_contract_written_cleans_up_all_schema_files() { - let _scenario = FailScenario::setup(); - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap(); - let _failpoint = catalog::INIT_AFTER_SCHEMA_CONTRACT_WRITTEN.fire_always(); - - let err = match Omnigraph::init(uri, helpers::TEST_SCHEMA).await { - Ok(_) => panic!("expected Omnigraph::init to fail at the configured failpoint"), - Err(e) => e, - }; - assert!( - err.to_string() - .contains("injected failpoint triggered: init.after_schema_contract_written"), - "got: {err}" - ); - - assert!( - !dir.path().join("_schema.pg").exists(), - "_schema.pg must be cleaned up" - ); - assert!( - !dir.path().join("_schema.ir.json").exists(), - "_schema.ir.json must be cleaned up" - ); - assert!( - !dir.path().join("__schema_state.json").exists(), - "__schema_state.json must be cleaned up" - ); - assert!( - !dir.path().join("__init_claim.json").exists(), - "pre-create failure must release the init claim after cleanup" - ); -} - #[tokio::test] #[serial] async fn init_failpoint_after_coordinator_init_leaves_completed_store_intact() { @@ -4457,8 +4363,8 @@ async fn init_failpoint_after_coordinator_init_leaves_completed_store_intact() { // files must survive the failed init. for schema_file in ["_schema.pg", "_schema.ir.json", "__schema_state.json"] { assert!( - dir.path().join(schema_file).exists(), - "{schema_file} must survive a post-commit-point init failure" + !dir.path().join(schema_file).exists(), + "{schema_file} is not part of a row-contract graph" ); } @@ -4510,8 +4416,8 @@ async fn init_failpoint_post_manifest_create_leaves_completed_graph_intact() { for schema_file in ["_schema.pg", "_schema.ir.json", "__schema_state.json"] { assert!( - dir.path().join(schema_file).exists(), - "{schema_file} must survive a post-commit-point init failure" + !dir.path().join(schema_file).exists(), + "{schema_file} is not part of a row-contract graph" ); } @@ -4546,10 +4452,7 @@ async fn init_manifest_create_lost_ack_recovers_exact_genesis() { ); drop(db); for artifact in ["_schema.pg", "_schema.ir.json", "__schema_state.json"] { - assert!( - dir.path().join(artifact).exists(), - "lost-ack recovery must preserve {artifact}" - ); + assert!(!dir.path().join(artifact).exists()); } assert!( !dir.path().join("__init_claim.json").exists(), @@ -4598,17 +4501,10 @@ async fn init_table_create_lost_ack_preserves_claim_and_schema() { }; assert_eq!(error_uri, uri); assert!(source.to_string().contains("init.table_create_post_native")); - for artifact in [ - "_schema.pg", - "_schema.ir.json", - "__schema_state.json", - "__init_claim.json", - ] { - assert!( - dir.path().join(artifact).exists(), - "physical-init ambiguity must preserve {artifact}" - ); - } + assert!( + dir.path().join("__init_claim.json").exists(), + "physical-init ambiguity must preserve __init_claim.json" + ); assert!( dir.path().join("nodes").exists() || dir.path().join("edges").exists(), "the injected error must follow a real durable table Create" @@ -4647,17 +4543,10 @@ async fn init_manifest_create_unknown_and_probe_failure_preserves_claim_and_sche .contains("init.manifest_create_post_native") ); assert!(probe.to_string().contains("init.manifest_create_probe")); - for artifact in [ - "_schema.pg", - "_schema.ir.json", - "__schema_state.json", - "__init_claim.json", - ] { - assert!( - dir.path().join(artifact).exists(), - "indeterminate initialization must preserve {artifact}" - ); - } + assert!( + dir.path().join("__init_claim.json").exists(), + "indeterminate initialization must preserve __init_claim.json" + ); drop(probe_failure); drop(lost_ack); @@ -4670,44 +4559,30 @@ async fn init_manifest_create_unknown_and_probe_failure_preserves_claim_and_sche ); } -// The floor under the schema-files-gone damage state: however a graph loses -// its schema files while keeping its data and `__manifest`, `open` must -// diagnose the state by name. #[tokio::test] #[serial] -async fn open_missing_schema_pg_reports_schema_files_missing() { +async fn open_serves_the_row_contract_without_the_schema_files() { let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap().to_string(); Omnigraph::init(&uri, helpers::TEST_SCHEMA) .await .expect("init must succeed"); - std::fs::remove_file(dir.path().join("_schema.pg")).unwrap(); - - let err = match Omnigraph::open(&uri).await { - Ok(_) => panic!("open must fail without _schema.pg"), - Err(e) => e, - }; - let OmniError::Manifest(manifest) = &err else { - panic!("missing schema source must remain a typed manifest error"); - }; - assert_eq!(manifest.kind, ManifestErrorKind::NotFound); - let msg = err.to_string(); - assert!( - msg.contains("missing its schema files") && msg.contains("_schema.pg"), - "open must name the missing schema files, got: {msg}" - ); + for name in ["_schema.pg", "_schema.ir.json", "__schema_state.json"] { + assert!(!dir.path().join(name).exists()); + } - let read_only_err = match Omnigraph::open_read_only(&uri).await { - Ok(_) => panic!("read-only open must also fail without _schema.pg"), - Err(err) => err, - }; - let OmniError::Manifest(manifest) = read_only_err else { - panic!("read-only missing schema source must remain a typed manifest error"); - }; - assert_eq!(manifest.kind, ManifestErrorKind::NotFound); + let db = Omnigraph::open(&uri) + .await + .expect("open reads the contract from main's __manifest, not the files"); + assert_eq!(db.schema_source().as_str(), helpers::TEST_SCHEMA); + assert!(db.catalog().type_id("Person").is_some()); + drop(db); + let read_only = Omnigraph::open_read_only(&uri) + .await + .expect("read-only open reads the contract from main's __manifest too"); + assert_eq!(read_only.schema_source().as_str(), helpers::TEST_SCHEMA); + drop(read_only); - std::fs::remove_file(dir.path().join("_schema.ir.json")).unwrap(); - std::fs::remove_file(dir.path().join("__schema_state.json")).unwrap(); let reinit = match Omnigraph::init(&uri, "node Replacement { key: String @key }\n").await { Ok(_) => panic!("strict init must not rebind a readable manifest with missing schema"), Err(err) => err, @@ -4774,37 +4649,6 @@ async fn init_crash_after_manifest_create_leaves_openable_store() { assert_eq!(count_rows(&db, "node:Person").await, 1); } -#[tokio::test] -#[serial] -async fn init_failpoint_returns_original_error_not_cleanup_error() { - // A delete failure is outcome-unknown: retain the claim so a delayed - // delete cannot race a later initializer, but still return the original - // pre-physical init error rather than masking it with cleanup failure. - let _scenario = FailScenario::setup(); - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap(); - let _failpoint = catalog::INIT_AFTER_SCHEMA_PG_WRITTEN.fire_always(); - let _delete_failure = catalog::INIT_SCHEMA_CLEANUP_DELETE.fire_always(); - - let err = match Omnigraph::init(uri, helpers::TEST_SCHEMA).await { - Ok(_) => panic!("expected Omnigraph::init to fail at the configured failpoint"), - Err(e) => e, - }; - let msg = err.to_string(); - assert!( - msg.contains("init.after_schema_pg_written"), - "init error must surface the failpoint cause, got: {msg}" - ); - assert!( - dir.path().join("_schema.pg").exists(), - "the injected delete failure must leave the owned schema artifact" - ); - assert!( - dir.path().join("__init_claim.json").exists(), - "an indeterminate schema delete must retain the init claim" - ); -} - // Local roots probe create-if-absent on init and on read-write open (issue // #453: no hard_link(2) breaks every write). The failpoint stands in for a // refusing filesystem; real refusals are classified by storage-crate tests. @@ -6431,6 +6275,21 @@ async fn azure_optimize_detached_pin_survives_next_write() { assert_optimize_detached_pin_survives_next_write(&uri).await; delete_azure_graph(&uri).await; } +/// Atomic schema publication on configured Azure/Azurite, followed by fixture cleanup. +#[tokio::test] +#[serial] +async fn azure_schema_apply_post_commit_failure_leaves_readable_contract() { + let Some(uri) = azure_test_graph_uri("schema-apply-post-commit") else { + eprintln!( + "skipping Azure atomic schema publication: OMNIGRAPH_AZURE_TEST_CONTAINER is not set" + ); + return; + }; + let _scenario = FailScenario::setup(); + assert_schema_apply_post_commit_failure_leaves_readable_contract(&uri).await; + delete_azure_graph(&uri).await; +} + /// Person HEAD and published version on `branch`, with the linear HEAD read /// from the ref the entry names. async fn person_versions(db: &Omnigraph, branch: &str) -> (u64, u64) { @@ -7576,15 +7435,34 @@ async fn rfc_0067_unavailable_ack_readback_is_indeterminate_without_replay() { }; let error = outcome.unwrap_err().to_string(); assert!(error.contains("outcome is in doubt"), "{error}"); + mutate_main( + &db, + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", "after_indeterminate")], &[("$age", 43)]), + ) + .await + .expect("same handle continues before any read refresh"); let fresh = Omnigraph::open_read_only(dir.path().to_str().unwrap()) .await .unwrap(); assert_eq!( count_rows(&fresh, "node:Person").await, - before + 1, + before + 2, "no automatic replay" ); - assert_eq!(count_rows(&db, "node:Person").await, before + 1); + let people = read_table(&fresh, "node:Person").await; + let names = collect_column_strings(&people, "name"); + for expected in ["indeterminate", "after_indeterminate"] { + assert_eq!( + names + .iter() + .filter(|name| name.as_str() == expected) + .count(), + 1 + ); + } + assert_eq!(count_rows(&db, "node:Person").await, before + 2); } /// GENERALIZED LIVENESS, persistent pre-publish faults: the same live handle @@ -7772,7 +7650,7 @@ async fn live_handle_writes_after_every_write_family_seam_failure() { Driver::Merge, ), ("schema_apply.post_lock_pre_effect", Driver::SchemaApply), - ("schema_apply.before_staging_write", Driver::SchemaApply), + ("graph_publish.before_commit_append", Driver::SchemaApply), ("schema_apply.after_manifest_commit", Driver::SchemaApply), ( "ensure_indices.post_stage_pre_commit_btree", diff --git a/crates/omnigraph/tests/helpers/failpoint.rs b/crates/omnigraph/tests/helpers/failpoint.rs index 5624f2b8b..bb0444e9f 100644 --- a/crates/omnigraph/tests/helpers/failpoint.rs +++ b/crates/omnigraph/tests/helpers/failpoint.rs @@ -58,6 +58,10 @@ impl Rendezvous { self.hold.reached() } + pub fn timed_out(&self) -> bool { + self.hold.timed_out() + } + /// Release the parked thread so it resumes past the seam. pub fn release(&self) { self.hold.release(); diff --git a/crates/omnigraph/tests/helpers/mod.rs b/crates/omnigraph/tests/helpers/mod.rs index 60e11da2f..90d2d3b28 100644 --- a/crates/omnigraph/tests/helpers/mod.rs +++ b/crates/omnigraph/tests/helpers/mod.rs @@ -56,6 +56,47 @@ query insert_person_and_friend($name: String, $age: I32, $friend: String) { } "#; +/// Materialize main's accepted contract as legacy files before removing its row. +pub async fn make_legacy_flat_manifest_fixture(root: &std::path::Path, stamp: u32) { + assert!((5..=11).contains(&stamp)); + let uri = root.to_str().unwrap(); + let mut manifest = lance::Dataset::open(&format!("{uri}/__manifest")) + .await + .unwrap(); + let contract = omnigraph_catalog::ManifestCoordinator::read_schema_contract_at( + uri, + None, + manifest.version().version, + ) + .await + .unwrap(); + let ir: omnigraph_compiler::SchemaIR = serde_json::from_str(&contract.ir).unwrap(); + let state = serde_json::json!({ + "format_version": 2, + "schema_shape_hash": omnigraph_compiler::schema_shape_hash_from_ir(&ir).unwrap(), + "schema_ir_hash": contract.head.schema_ir_hash, + "schema_identity_version": contract.head.schema_identity_version, + "schema_identity_domain": contract.head.schema_identity_domain, + }); + std::fs::write(root.join("_schema.pg"), &contract.source).unwrap(); + std::fs::write(root.join("_schema.ir.json"), &contract.ir).unwrap(); + std::fs::write( + root.join("__schema_state.json"), + serde_json::to_vec_pretty(&state).unwrap(), + ) + .unwrap(); + omnigraph_catalog::migrations::restamp_flat_for_test(&mut manifest, stamp) + .await + .unwrap(); + assert_eq!( + omnigraph_catalog::migrations::read_stamp(&manifest), + Some(stamp) + ); + assert!(manifest.schema().field("record").is_none()); + assert!(manifest.schema().field("schema_source").is_none()); + assert!(manifest.schema().field("schema_ir").is_none()); +} + /// Build the graph-level selector used by the dedicated Blob read facade. pub fn node_blob_cell( type_name: impl Into, diff --git a/crates/omnigraph/tests/lance_surface_guards.rs b/crates/omnigraph/tests/lance_surface_guards.rs index 0d95158b6..5e568bb7b 100644 --- a/crates/omnigraph/tests/lance_surface_guards.rs +++ b/crates/omnigraph/tests/lance_surface_guards.rs @@ -5866,4 +5866,41 @@ async fn packed_struct_refuses_null_values_and_lone_child_projection_lance_11() .contains("invalid variable-width layout for UInt64"), "{error}" ); + let mut filtered = dataset.scan(); + filtered.project(&["id", "record"]).unwrap(); + filtered.filter("id = 'b'").unwrap(); + let error = filtered.try_into_batch().await.expect_err( + "Lance 11's default materialization splits the fixed child out of a packed struct", + ); + assert!( + error + .to_string() + .contains("invalid variable-width layout for UInt64"), + "{error}" + ); + let mut late = dataset.scan(); + late.project(&["id", "record"]).unwrap(); + late.filter("id = 'b'").unwrap(); + late.materialization_style(lance::dataset::scanner::MaterializationStyle::AllLate); + let late_batch = late.try_into_batch().await.unwrap(); + let mut early = dataset.scan(); + early.project(&["id", "record"]).unwrap(); + early.filter("id = 'b'").unwrap(); + early.materialization_style(lance::dataset::scanner::MaterializationStyle::AllEarly); + let batch = early.try_into_batch().await.unwrap(); + assert_eq!( + late_batch, batch, + "both explicit styles retain the whole record" + ); + assert_eq!(batch.num_rows(), 1); + assert_eq!(batch.column(0).as_string::().value(0), "b"); + let record = batch.column(1).as_struct(); + assert_eq!(record.column(0).as_string::().value(0), ""); + assert_eq!( + record + .column(1) + .as_primitive::() + .value(0), + 0 + ); } diff --git a/crates/omnigraph/tests/legacy_columns.rs b/crates/omnigraph/tests/legacy_columns.rs index 59845819d..8bdd40653 100644 --- a/crates/omnigraph/tests/legacy_columns.rs +++ b/crates/omnigraph/tests/legacy_columns.rs @@ -3,8 +3,6 @@ mod helpers; -use std::fs; - use arrow_array::{StringArray, StructArray}; use omnigraph::changes::{ChangeFeedScope, ChangeFilter, ChangeOp, ChangeOpKind}; @@ -45,9 +43,16 @@ async fn legacy_vintage_graph_works_end_to_end() { .unwrap(), ); - let ir: serde_json::Value = - serde_json::from_str(&fs::read_to_string(dir.path().join("_schema.ir.json")).unwrap()) - .unwrap(); + let ir: serde_json::Value = serde_json::from_str( + &omnigraph_catalog::ManifestCoordinator::open(uri) + .await + .unwrap() + .read_schema_contract() + .await + .unwrap() + .ir, + ) + .unwrap(); assert_eq!( ir["ir_version"].as_u64(), Some(u64::from(SCHEMA_IR_VERSION)), @@ -57,7 +62,7 @@ async fn legacy_vintage_graph_works_end_to_end() { db.internal_schema_version_of(ReadTarget::branch("main")) .await .unwrap(), - 12, + omnigraph::db::manifest::INTERNAL_MANIFEST_SCHEMA_VERSION, "a legacy-vintage graph is born at the current `__manifest` stamp; the vintage lives in the schema IR (RFC 0040 Compatibility, RFC 0067, detached-only tables)" ); let snap = snapshot_main(&db).await.unwrap(); @@ -428,9 +433,16 @@ edge WorksAt: Person -> Company { ) .await .expect("legacy evolution restates historically legal underscore names"); - let ir: serde_json::Value = - serde_json::from_str(&fs::read_to_string(dir.path().join("_schema.ir.json")).unwrap()) - .unwrap(); + let ir: serde_json::Value = serde_json::from_str( + &omnigraph_catalog::ManifestCoordinator::open(uri) + .await + .unwrap() + .read_schema_contract() + .await + .unwrap() + .ir, + ) + .unwrap(); assert_eq!( ir["ir_version"].as_u64(), Some(u64::from(SCHEMA_IR_VERSION)), @@ -538,23 +550,22 @@ async fn legacy_endpoint_constraints_keep_the_accepted_shape_hash() { .await .unwrap(), ); - let old_state: serde_json::Value = serde_json::from_str( - &fs::read_to_string(dir.path().join("__schema_state.json")).unwrap(), + let old_ir: omnigraph_compiler::SchemaIR = serde_json::from_str( + &omnigraph_catalog::ManifestCoordinator::open(uri) + .await + .unwrap() + .read_schema_contract() + .await + .unwrap() + .ir, ) .unwrap(); - let old_ir: omnigraph_compiler::SchemaIR = - serde_json::from_str(&fs::read_to_string(dir.path().join("_schema.ir.json")).unwrap()) - .unwrap(); let old_shape = omnigraph_compiler::compile_schema_shape( &omnigraph_compiler::schema::parser::parse_persisted_schema_contract(&source).unwrap(), ) .unwrap(); assert_eq!( - old_state["schema_shape_hash"], - omnigraph_compiler::schema_shape_hash(&old_shape).unwrap() - ); - assert_eq!( - old_state["schema_shape_hash"], + omnigraph_compiler::schema_shape_hash(&old_shape).unwrap(), omnigraph_compiler::schema_shape_hash_from_ir(&old_ir).unwrap() ); drop(db); diff --git a/crates/omnigraph/tests/lifecycle.rs b/crates/omnigraph/tests/lifecycle.rs index 4732a9e85..55773f905 100644 --- a/crates/omnigraph/tests/lifecycle.rs +++ b/crates/omnigraph/tests/lifecycle.rs @@ -6,7 +6,7 @@ use omnigraph::db::{InitOptions, Omnigraph, ReadTarget}; use omnigraph_compiler::schema::parser::{parse_persisted_schema_contract, parse_schema}; use omnigraph_compiler::{ SchemaIR, SchemaIdentityDomain, compile_schema_shape, resolve_schema_ir, schema_ir_hash, - schema_ir_pretty_json, schema_shape_hash, schema_shape_hash_from_ir, + schema_ir_pretty_json, }; use helpers::*; @@ -19,27 +19,198 @@ fn compile_persisted_shape(source: &str) -> omnigraph_compiler::SchemaShape { compile_schema_shape(&parse_persisted_schema_contract(source).unwrap()).unwrap() } -fn schema_state_json(ir: &SchemaIR) -> serde_json::Value { - serde_json::json!({ - "format_version": 2, - "schema_shape_hash": schema_shape_hash_from_ir(ir).unwrap(), - "schema_ir_hash": schema_ir_hash(ir).unwrap(), - "schema_identity_version": 2, - "schema_identity_domain": ir.schema_identity_domain.as_str(), - }) +/// Replace the `schema_contract` row of main's `__manifest` with `source` and +/// `ir` in one publish, as an apply would, without touching any table: the +/// way a test plants a contract the engine did not write. +async fn publish_schema_contract_row(uri: &str, source: &str, ir: &SchemaIR) -> u64 { + publish_schema_contract_text(uri, source, &schema_ir_pretty_json(ir).unwrap(), ir).await } -fn persist_schema_contract(root: &std::path::Path, ir: &SchemaIR) { - fs::write( - root.join("_schema.ir.json"), - schema_ir_pretty_json(ir).unwrap(), +async fn publish_schema_contract_text( + uri: &str, + source: &str, + ir_text: &str, + ir: &SchemaIR, +) -> u64 { + let mut manifest = omnigraph_catalog::ManifestCoordinator::open(uri) + .await + .unwrap(); + manifest + .commit_changes(&[omnigraph_catalog::ManifestChange::SchemaContract( + omnigraph_catalog::SchemaContractRow { + source: source.to_string(), + ir: ir_text.to_string(), + head: omnigraph_catalog::SchemaContractHead { + schema_ir_hash: schema_ir_hash(ir).unwrap(), + schema_identity_version: 2, + schema_identity_domain: ir.schema_identity_domain.as_str().to_string(), + }, + }, + )]) + .await + .unwrap() +} + +async fn read_contract(uri: &str) -> omnigraph_catalog::SchemaContractRow { + omnigraph_catalog::ManifestCoordinator::open(uri) + .await + .unwrap() + .read_schema_contract() + .await + .unwrap() +} + +#[tokio::test] +async fn warm_publisher_preserves_foreign_contract_text_at_the_same_identity() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let db = init_and_load(&dir).await; + mutate_main( + &db, + MUTATION_QUERIES, + "set_age", + &mixed_params(&[("$name", "Alice")], &[("$age", 31)]), ) + .await .unwrap(); - fs::write( - root.join("__schema_state.json"), - serde_json::to_string_pretty(&schema_state_json(ir)).unwrap(), + let before = read_contract(uri).await; + let ir: SchemaIR = serde_json::from_str(&before.ir).unwrap(); + let source = format!("\n{}\n", before.source); + let ir_text = format!("\n{}\n", serde_json::to_string(&ir).unwrap()); + let old_head = db + .snapshot_of(ReadTarget::branch("main")) + .await + .unwrap() + .graph_head(None) + .map(str::to_owned); + publish_schema_contract_text(uri, &source, &ir_text, &ir).await; + let replacement = read_contract(uri).await; + assert_eq!(replacement.head, before.head); + assert_ne!(replacement.source, before.source); + assert_ne!(replacement.ir, before.ir); + assert_eq!( + omnigraph_catalog::ManifestCoordinator::open(uri) + .await + .unwrap() + .exact_graph_head(), + old_head + ); + let (resolved, io) = helpers::cost::measure(db.resolve_snapshot("main")).await; + let resolved = resolved.unwrap(); + assert_eq!(Some(resolved.as_str()), old_head.as_deref()); + assert_eq!(io.version_probes, 1); + assert_eq!(io.internal_open_count, 1); + assert_eq!(io.manifest_scan_count, 1); + mutate_main( + &db, + MUTATION_QUERIES, + "set_age", + &mixed_params(&[("$name", "Bob")], &[("$age", 77)]), ) + .await .unwrap(); + assert_eq!(read_contract(uri).await, replacement); + db.refresh().await.unwrap(); + assert_eq!(db.schema_source().as_str(), source); + let fresh = helpers::session(Omnigraph::open_read_only(uri).await.unwrap()); + for (name, age) in [("Alice", 31), ("Bob", 77)] { + let result = fresh + .query( + ReadTarget::branch("main"), + TEST_QUERIES, + "get_person", + ¶ms(&[("$name", name)]), + ) + .await + .unwrap(); + let batch = result.concat_batches().unwrap(); + assert_eq!(batch.num_rows(), 1); + assert_eq!( + batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + age + ); + } +} + +#[tokio::test] +async fn schema_contract_integrity_refuses_mismatched_identity() { + for corruption in ["source", "ir", "hash", "domain", "version"] { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let held = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); + held.branch_create("merge_source").await.unwrap(); + held.snapshot_of(ReadTarget::branch("main")).await.unwrap(); + let refreshing = Omnigraph::open(uri).await.unwrap(); + let writing = Omnigraph::open(uri).await.unwrap(); + let merging = helpers::session(Omnigraph::open(uri).await.unwrap()); + let mut manifest = omnigraph_catalog::ManifestCoordinator::open(uri) + .await + .unwrap(); + let mut row = manifest.read_schema_contract().await.unwrap(); + if corruption == "source" { + row.source = "node Different { age: String }".to_string(); + } else if corruption == "ir" { + row.ir = "not valid JSON".to_string(); + } else if corruption == "domain" { + row.head.schema_identity_domain = SchemaIdentityDomain::from_ulid(ulid::Ulid::new()) + .as_str() + .to_string(); + } else if corruption == "version" { + row.head.schema_identity_version += 1; + } else { + row.head.schema_ir_hash = "invalid-hash".to_string(); + } + manifest + .commit_changes(&[omnigraph_catalog::ManifestChange::SchemaContract(row)]) + .await + .unwrap(); + let before = manifest.version(); + let expected = match corruption { + "source" => "source no longer matches", + "ir" => "schema contract in the schema_contract row is invalid", + "domain" => "identity domain", + "version" => "identity version", + _ => "schema_ir_hash", + }; + let refresh_error = refreshing.refresh().await.unwrap_err(); + assert!( + refresh_error.to_string().contains(expected), + "{refresh_error}" + ); + let warm_error = held + .snapshot_of(ReadTarget::branch("main")) + .await + .unwrap_err(); + assert!(warm_error.to_string().contains(expected), "{warm_error}"); + let write_error = writing.branch_create("must_refuse").await.unwrap_err(); + assert!(write_error.to_string().contains(expected), "{write_error}"); + let merge_error = merging + .branch_merge("merge_source", "main") + .await + .unwrap_err(); + assert!(merge_error.to_string().contains(expected), "{merge_error}"); + for read_only in [true, false] { + let result = if read_only { + Omnigraph::open_read_only(uri).await + } else { + Omnigraph::open(uri).await + }; + let error = result.err().expect("corrupt contract must refuse open"); + assert!(error.to_string().contains(expected), "{error}"); + } + assert_eq!( + omnigraph_catalog::ManifestCoordinator::open(uri) + .await + .unwrap() + .version(), + before + ); + } } #[tokio::test] @@ -52,41 +223,29 @@ async fn init_creates_graph() { .await .unwrap(); - assert!(dir.path().join("_schema.pg").exists()); - assert!(dir.path().join("_schema.ir.json").exists()); - assert!(dir.path().join("__schema_state.json").exists()); - - let ir: SchemaIR = - serde_json::from_str(&fs::read_to_string(dir.path().join("_schema.ir.json")).unwrap()) - .unwrap(); - let state: serde_json::Value = - serde_json::from_str(&fs::read_to_string(dir.path().join("__schema_state.json")).unwrap()) - .unwrap(); + for name in ["_schema.pg", "_schema.ir.json", "__schema_state.json"] { + assert!( + !dir.path().join(name).exists(), + "init must not write {name}" + ); + } + let contract_row = read_contract(uri).await; + let ir: SchemaIR = serde_json::from_str(&contract_row.ir).unwrap(); assert_eq!(ir.ir_version, 5); assert!(ir.features.contains("system-columns")); - let persisted: serde_json::Value = - serde_json::from_slice(&fs::read(dir.path().join("_schema.ir.json")).unwrap()).unwrap(); + let persisted: serde_json::Value = serde_json::from_str(&contract_row.ir).unwrap(); assert!(persisted.get("actor_provenance").is_none()); assert_eq!(db.schema_source().as_str(), TEST_SCHEMA); assert!(ir.next_identity_id > 1); assert!(SchemaIdentityDomain::parse(ir.schema_identity_domain.as_str()).is_ok()); - assert_eq!(state["format_version"].as_u64(), Some(2)); - assert_eq!(state["schema_identity_version"].as_u64(), Some(2)); - assert_eq!( - state["schema_ir_hash"].as_str(), - Some(schema_ir_hash(&ir).unwrap().as_str()) - ); + assert_eq!(contract_row.head.schema_identity_version, 2); assert_eq!( - state["schema_shape_hash"].as_str(), - Some( - schema_shape_hash(&compile_shape(TEST_SCHEMA)) - .unwrap() - .as_str() - ) + contract_row.head.schema_ir_hash, + schema_ir_hash(&ir).unwrap() ); assert_eq!( - state["schema_identity_domain"].as_str(), - Some(ir.schema_identity_domain.as_str()) + contract_row.head.schema_identity_domain, + ir.schema_identity_domain.as_str() ); assert_eq!( db.catalog() @@ -102,9 +261,10 @@ async fn init_creates_graph() { db.internal_schema_version_of(ReadTarget::branch("main")) .await .unwrap(), - 12, - "fresh graphs are stamped at the current manifest format (v12, the packed catalog record over detached-only tables, RFC 0067 detached table commits, RFC 0040 system columns and RFC 0042 retirement metadata)" + 13, + "fresh graphs are stamped at the current manifest format (v13, the schema contract as a `schema_contract` row of main's `__manifest` over the packed catalog record, detached-only tables, RFC 0067 detached table commits, RFC 0040 system columns and RFC 0042 retirement metadata)" ); + assert_eq!(contract_row.source, TEST_SCHEMA); assert!(snap.dataset("node:Person").is_some()); assert!(snap.dataset("node:Company").is_some()); assert!(snap.dataset("edge:Knows").is_some()); @@ -186,10 +346,6 @@ node Document { .contains("@unique is not supported on blob property Document.content") ); - // Build a normal v6 root, then replace its source/accepted identity - // artifacts with the exact shape the pre-v0.10 parser admitted. The table - // identity and physical schema are unchanged; only the logical constraint - // differs, so this models a historical root without weakening new init. let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); let db = Omnigraph::init(uri, BASE_SCHEMA).await.unwrap(); @@ -199,8 +355,7 @@ node Document { let historical_ir = resolve_schema_ir(&accepted, &compile_persisted_shape(HISTORICAL_SCHEMA)) .unwrap() .schema_ir; - fs::write(dir.path().join("_schema.pg"), HISTORICAL_SCHEMA).unwrap(); - persist_schema_contract(dir.path(), &historical_ir); + publish_schema_contract_row(uri, HISTORICAL_SCHEMA, &historical_ir).await; let reopened = Omnigraph::open(uri) .await @@ -255,67 +410,7 @@ async fn open_reads_existing_graph() { } #[tokio::test] -async fn open_refuses_missing_identity_contract_without_bootstrap() { - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap(); - Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - - fs::remove_file(dir.path().join("_schema.ir.json")).unwrap(); - fs::remove_file(dir.path().join("__schema_state.json")).unwrap(); - - let err = match Omnigraph::open(uri).await { - Ok(_) => panic!("open must not reconstruct stable IDs from mutable source names"), - Err(err) => err, - }; - assert!( - err.to_string() - .contains("automatic bootstrap is not supported") - ); - assert!(!dir.path().join("_schema.ir.json").exists()); - assert!(!dir.path().join("__schema_state.json").exists()); -} - -#[tokio::test] -async fn open_refuses_partial_identity_contract() { - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap(); - Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - - fs::remove_file(dir.path().join("__schema_state.json")).unwrap(); - - let err = match Omnigraph::open(uri).await { - Ok(_) => panic!("open must reject a partial identity contract"), - Err(err) => err, - }; - assert!(err.to_string().contains("schema contract is incomplete")); - assert!(dir.path().join("_schema.ir.json").exists()); - assert!(!dir.path().join("__schema_state.json").exists()); -} - -#[tokio::test] -async fn open_refuses_pre_identity_schema_state_format() { - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap(); - Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - - let state_path = dir.path().join("__schema_state.json"); - let mut state: serde_json::Value = - serde_json::from_str(&fs::read_to_string(&state_path).unwrap()).unwrap(); - state["format_version"] = serde_json::json!(1); - fs::write(&state_path, serde_json::to_string_pretty(&state).unwrap()).unwrap(); - - let err = match Omnigraph::open(uri).await { - Ok(_) => panic!("schema-state v1 must not be served as identity-capable state"), - Err(err) => err, - }; - assert!( - err.to_string() - .contains("schema state format 1 is unsupported") - ); -} - -#[tokio::test] -async fn open_refuses_v3_live_or_staged_schema_without_changing_files() { +async fn open_refuses_v3_schema_without_changing_files() { fn files(root: &std::path::Path) -> std::collections::BTreeMap> { let mut result = std::collections::BTreeMap::new(); let mut pending = vec![root.to_path_buf()]; @@ -337,33 +432,25 @@ async fn open_refuses_v3_live_or_staged_schema_without_changing_files() { // The unsupported version is the boundary, including disabled bindings. // Do not require this binary to understand the removed binding shape. - for (filename, enabled, duplicate_features) in [ - ("_schema.ir.json", true, false), - ("_schema.ir.json", false, false), - ("_schema.ir.json.staging", true, false), - ("_schema.ir.json", true, true), - ("_schema.ir.json.staging", true, true), - ] { + for (enabled, duplicate_features) in [(true, false), (false, false), (true, true)] { let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); - drop(Omnigraph::init(uri, TEST_SCHEMA).await.unwrap()); + let accepted = Omnigraph::init(uri, TEST_SCHEMA) + .await + .unwrap() + .catalog() + .bound_schema_ir() + .unwrap() + .clone(); let mut ir: serde_json::Value = - serde_json::from_slice(&fs::read(dir.path().join("_schema.ir.json")).unwrap()).unwrap(); + serde_json::from_str(&schema_ir_pretty_json(&accepted).unwrap()).unwrap(); ir["ir_version"] = serde_json::json!(3); ir["actor_provenance"] = serde_json::json!({ "enabled": enabled }); - // With matching source, historical recovery would clean these - // staged files before later discovering an unsupported live IR. - fs::write(dir.path().join("_schema.pg.staging"), TEST_SCHEMA).unwrap(); - fs::copy( - dir.path().join("__schema_state.json"), - dir.path().join("__schema_state.json.staging"), - ) - .unwrap(); let mut ir_text = serde_json::to_string_pretty(&ir).unwrap(); if duplicate_features { ir_text.insert_str(1, "\"features\": [],"); } - fs::write(dir.path().join(filename), ir_text).unwrap(); + publish_schema_contract_text(uri, TEST_SCHEMA, &ir_text, &accepted).await; let before = files(dir.path()); for read_only in [true, false] { @@ -393,11 +480,13 @@ async fn open_refuses_v3_live_or_staged_schema_without_changing_files() { async fn open_rejects_same_alias_with_foreign_table_identity() { let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); - Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - - let accepted: SchemaIR = - serde_json::from_str(&fs::read_to_string(dir.path().join("_schema.ir.json")).unwrap()) - .unwrap(); + let accepted = Omnigraph::init(uri, TEST_SCHEMA) + .await + .unwrap() + .catalog() + .bound_schema_ir() + .unwrap() + .clone(); let company_only = r#" node Company { name: String @key @@ -423,7 +512,7 @@ node Company { .unwrap() .type_id ); - persist_schema_contract(dir.path(), &replacement); + publish_schema_contract_row(uri, TEST_SCHEMA, &replacement).await; let err = match Omnigraph::open(uri).await { Ok(_) => panic!("open must reject a same-name table with a foreign stable identity"), @@ -441,11 +530,13 @@ node Company { async fn open_rejects_live_manifest_tables_absent_from_schema_ir() { let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); - Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - - let accepted: SchemaIR = - serde_json::from_str(&fs::read_to_string(dir.path().join("_schema.ir.json")).unwrap()) - .unwrap(); + let accepted = Omnigraph::init(uri, TEST_SCHEMA) + .await + .unwrap() + .catalog() + .bound_schema_ir() + .unwrap() + .clone(); let company_only = r#" node Company { name: String @key @@ -454,8 +545,7 @@ node Company { let replacement = resolve_schema_ir(&accepted, &compile_shape(company_only)) .unwrap() .schema_ir; - fs::write(dir.path().join("_schema.pg"), company_only).unwrap(); - persist_schema_contract(dir.path(), &replacement); + publish_schema_contract_row(uri, company_only, &replacement).await; let err = match Omnigraph::open(uri).await { Ok(_) => panic!("open must reject manifest tables omitted by accepted SchemaIR"), @@ -476,8 +566,7 @@ async fn refresh_rejects_schema_ir_tables_missing_from_manifest() { let replacement = resolve_schema_ir(&accepted, &compile_shape(&with_temporary_source)) .unwrap() .schema_ir; - fs::write(dir.path().join("_schema.pg"), with_temporary_source).unwrap(); - persist_schema_contract(dir.path(), &replacement); + publish_schema_contract_row(uri, &with_temporary_source, &replacement).await; let err = db .refresh() @@ -488,16 +577,16 @@ async fn refresh_rejects_schema_ir_tables_missing_from_manifest() { } #[tokio::test] -async fn write_capture_rejects_schema_ir_tables_missing_from_manifest() { +async fn write_preparation_manifest_read_failures_carry_before_effect_evidence() { let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); let db = helpers::session(Omnigraph::init(uri, TEST_SCHEMA).await.unwrap()); db.branch_create("refusal-probe").await.unwrap(); let before = db.list_commits(None).await.unwrap(); let branches_before = db.branch_list().await.unwrap(); - let schema_path = dir.path().join("_schema.pg"); - let unavailable_path = dir.path().join("_schema.pg.unavailable"); - fs::rename(&schema_path, &unavailable_path).unwrap(); + let manifest_path = dir.path().join("__manifest"); + let unavailable_path = dir.path().join("manifest-held"); + fs::rename(&manifest_path, &unavailable_path).unwrap(); let mutation_error = mutate_main( &db, MUTATION_QUERIES, @@ -533,7 +622,7 @@ async fn write_capture_rejects_schema_ir_tables_missing_from_manifest() { .await .unwrap_err(); let schema_error = db.apply_schema(TEST_SCHEMA).await.unwrap_err(); - fs::rename(&unavailable_path, &schema_path).unwrap(); + fs::rename(&unavailable_path, &manifest_path).unwrap(); for (door, error) in [ ("mutation", mutation_error), ("deletion", deletion_error), @@ -548,7 +637,7 @@ async fn write_capture_rejects_schema_ir_tables_missing_from_manifest() { assert_eq!( error.completion_evidence(), Some(omnigraph::error::CompletionEvidence::BeforeEffect), - "{door}: a failed contract GET before effects must carry owning evidence: {error}" + "{door}: a failed manifest authority read before effects must carry owning evidence: {error}" ); assert_eq!( error.storage_failure().map(|failure| failure.kind), @@ -558,14 +647,20 @@ async fn write_capture_rejects_schema_ir_tables_missing_from_manifest() { assert_eq!(db.list_commits(None).await.unwrap(), before); assert_eq!(db.branch_list().await.unwrap(), branches_before); db.branch_delete("refusal-probe").await.unwrap(); +} + +#[tokio::test] +async fn write_capture_rejects_schema_ir_tables_missing_from_manifest() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let db = helpers::session(Omnigraph::init(uri, TEST_SCHEMA).await.unwrap()); let accepted = db.catalog().bound_schema_ir().unwrap().clone(); let with_temporary_source = format!("{TEST_SCHEMA}\nnode Temporary {{\n key: String @key\n}}\n"); let replacement = resolve_schema_ir(&accepted, &compile_shape(&with_temporary_source)) .unwrap() .schema_ir; - fs::write(dir.path().join("_schema.pg"), with_temporary_source).unwrap(); - persist_schema_contract(dir.path(), &replacement); + publish_schema_contract_row(uri, &with_temporary_source, &replacement).await; let err = db .load_jsonl( @@ -578,93 +673,6 @@ async fn write_capture_rejects_schema_ir_tables_missing_from_manifest() { assert!(err.to_string().contains("is missing from manifest")); } -#[tokio::test] -async fn long_lived_handle_rejects_schema_source_drift() { - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap(); - let db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - - let drifted = TEST_SCHEMA.replace("age: I32?", "age: I64?"); - fs::write(dir.path().join("_schema.pg"), drifted).unwrap(); - - let err = match db.snapshot_of(ReadTarget::branch("main")).await { - Ok(_) => panic!("expected schema source drift to be rejected"), - Err(err) => err, - }; - assert!( - err.to_string() - .contains("current _schema.pg no longer matches the accepted compiled schema") - ); -} - -#[tokio::test] -async fn long_lived_handle_rejects_schema_ir_drift() { - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap(); - let db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - - fs::write(dir.path().join("_schema.ir.json"), "{not valid json").unwrap(); - - let err = match db.snapshot_of(ReadTarget::branch("main")).await { - Ok(_) => panic!("expected schema IR drift to be rejected"), - Err(err) => err, - }; - assert!( - err.to_string() - .contains("accepted compiled schema contract in _schema.ir.json is invalid") - ); -} - -#[tokio::test] -async fn long_lived_handle_rejects_ir_and_source_updates_without_state_update() { - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap(); - let db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - - let drifted = TEST_SCHEMA.replace("age: I32?", "age: I64?"); - let accepted_ir: SchemaIR = - serde_json::from_str(&fs::read_to_string(dir.path().join("_schema.ir.json")).unwrap()) - .unwrap(); - let drifted_ir = resolve_schema_ir(&accepted_ir, &compile_shape(&drifted)) - .unwrap() - .schema_ir; - let drifted_ir_json = schema_ir_pretty_json(&drifted_ir).unwrap(); - fs::write(dir.path().join("_schema.pg"), drifted).unwrap(); - fs::write(dir.path().join("_schema.ir.json"), drifted_ir_json).unwrap(); - - let err = match db.snapshot_of(ReadTarget::branch("main")).await { - Ok(_) => panic!("expected schema state mismatch to be rejected"), - Err(err) => err, - }; - assert!( - err.to_string() - .contains("accepted compiled schema does not match the recorded schema state") - ); -} - -#[tokio::test] -async fn long_lived_handle_rejects_schema_state_domain_drift() { - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap(); - let db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - - let state_path = dir.path().join("__schema_state.json"); - let mut state: serde_json::Value = - serde_json::from_str(&fs::read_to_string(&state_path).unwrap()).unwrap(); - state["schema_identity_domain"] = - serde_json::Value::String(SchemaIdentityDomain::new().as_str().to_string()); - fs::write(&state_path, serde_json::to_string_pretty(&state).unwrap()).unwrap(); - - let err = match db.snapshot_of(ReadTarget::branch("main")).await { - Ok(_) => panic!("expected schema identity-domain drift to be rejected"), - Err(err) => err, - }; - assert!( - err.to_string() - .contains("identity domain does not match the recorded schema state") - ); -} - #[tokio::test] async fn refresh_detects_identity_aba_when_source_bytes_are_unchanged() { let dir = tempfile::tempdir().unwrap(); @@ -693,19 +701,7 @@ async fn refresh_detects_identity_aba_when_source_bytes_are_unchanged() { schema_ir_hash(&accepted).unwrap() ); - fs::write( - dir.path().join("_schema.ir.json"), - schema_ir_pretty_json(&replacement).unwrap(), - ) - .unwrap(); - fs::write( - dir.path().join("__schema_state.json"), - serde_json::to_string_pretty(&schema_state_json(&replacement)).unwrap(), - ) - .unwrap(); - - // `_schema.pg` is deliberately byte-identical. A source-byte fast path - // would preserve the stale identity allocator and bound catalog here. + publish_schema_contract_row(uri, TEST_SCHEMA, &replacement).await; db.refresh().await.unwrap(); let refreshed = db.catalog(); let refreshed_ir = refreshed.bound_schema_ir().unwrap(); @@ -716,19 +712,6 @@ async fn refresh_detects_identity_aba_when_source_bytes_are_unchanged() { ); } -#[tokio::test] -async fn comment_only_schema_edit_keeps_schema_state_valid() { - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap(); - let db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - - let commented = format!("// comment-only drift\n{}", TEST_SCHEMA); - fs::write(dir.path().join("_schema.pg"), commented).unwrap(); - - let snapshot = db.snapshot_of(ReadTarget::branch("main")).await.unwrap(); - assert!(snapshot.dataset("node:Person").is_some()); -} - #[tokio::test] async fn open_nonexistent_fails() { let result = Omnigraph::open("/tmp/nonexistent_omnigraph_test_xyz").await; @@ -758,70 +741,18 @@ async fn snapshot_version_is_pinned() { assert_eq!(snap1.graph_manifest_version(), v1); } -/// Regression for the `Omnigraph::init` re-init footgun (MR-668 -/// follow-up): a second `init` against a URI that already holds a -/// graph must NOT modify or destroy the existing graph's schema -/// artifacts. Today's behavior is destructive either way — the -/// `write_text(_schema.pg, ...)` call at the top of -/// `init_commit_phase` overwrites the existing file before any -/// preflight, and `best_effort_cleanup_init_artifacts` will later -/// delete all three files if the inner `GraphCoordinator::init` -/// fails. Both outcomes corrupt an existing graph. -/// -/// After the fix: strict-mode `init` (no `force` flag) errors out -/// before touching any file, and the original schema artifacts -/// match their pre-attempt contents byte-for-byte. #[tokio::test] async fn init_on_existing_graph_uri_does_not_destroy_existing_schema() { let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); - - // Establish the first graph and snapshot its three schema files. Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - let original_schema_pg = fs::read_to_string(dir.path().join("_schema.pg")).unwrap(); - let original_schema_ir = fs::read_to_string(dir.path().join("_schema.ir.json")).unwrap(); - let original_schema_state = fs::read_to_string(dir.path().join("__schema_state.json")).unwrap(); - - // Attempt a re-init with a deliberately different schema so any - // overwrite would be observable in the file contents. - let different_schema = "node Other { id: String @key }\n"; - let result = Omnigraph::init(uri, different_schema).await; - - // The new init must report the conflict, not silently mutate. - assert!( - result.is_err(), - "init against an existing graph URI must error, not silently overwrite" - ); - - // The three schema files must remain present and byte-identical to - // their pre-attempt contents. - assert!( - dir.path().join("_schema.pg").exists(), - "_schema.pg must not be deleted by a failed re-init" - ); - assert!( - dir.path().join("_schema.ir.json").exists(), - "_schema.ir.json must not be deleted by a failed re-init" - ); + let before = read_contract(uri).await; assert!( - dir.path().join("__schema_state.json").exists(), - "__schema_state.json must not be deleted by a failed re-init" - ); - assert_eq!( - fs::read_to_string(dir.path().join("_schema.pg")).unwrap(), - original_schema_pg, - "_schema.pg contents must be preserved when re-init is rejected" - ); - assert_eq!( - fs::read_to_string(dir.path().join("_schema.ir.json")).unwrap(), - original_schema_ir, - "_schema.ir.json contents must be preserved when re-init is rejected" - ); - assert_eq!( - fs::read_to_string(dir.path().join("__schema_state.json")).unwrap(), - original_schema_state, - "__schema_state.json contents must be preserved when re-init is rejected" + Omnigraph::init(uri, "node Other { id: String @key }\n") + .await + .is_err() ); + assert_eq!(read_contract(uri).await, before); } #[tokio::test] @@ -830,14 +761,7 @@ async fn force_init_refuses_existing_manifest_and_preserves_identity_contract() let uri = dir.path().to_str().unwrap(); Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - let schema_path = dir.path().join("_schema.pg"); - let ir_path = dir.path().join("_schema.ir.json"); - let state_path = dir.path().join("__schema_state.json"); - let before = [ - fs::read_to_string(&schema_path).unwrap(), - fs::read_to_string(&ir_path).unwrap(), - fs::read_to_string(&state_path).unwrap(), - ]; + let before = read_contract(uri).await; let err = match Omnigraph::init_with_options( uri, @@ -850,64 +774,28 @@ async fn force_init_refuses_existing_manifest_and_preserves_identity_contract() Err(err) => err, }; assert!(err.to_string().contains("force init refuses graph root")); - assert_eq!(fs::read_to_string(schema_path).unwrap(), before[0]); - assert_eq!(fs::read_to_string(ir_path).unwrap(), before[1]); - assert_eq!(fs::read_to_string(state_path).unwrap(), before[2]); + assert_eq!(read_contract(uri).await, before); } -/// Happy-path sibling to the strict re-init regression above: -/// `InitOptions { force: true }` may replace orphan schema artifacts when the -/// operator deliberately recovers from a failed prior init. -/// -/// Force does not purge Lance state and refuses any existing `__manifest`. -/// The supported recovery scenario is therefore "schema files exist but Lance -/// state doesn't," which this test reproduces. -/// -/// Without this test, a future refactor could invert the `if !force` -/// branch and silently break the operator-facing escape hatch. #[tokio::test] -async fn init_with_force_recovers_from_orphan_schema_files() { - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().to_str().unwrap(); - - // Simulate orphan schema files: write `_schema.pg` to disk - // without running a full init. The preflight will see it and - // bail in strict mode. - fs::write(dir.path().join("_schema.pg"), TEST_SCHEMA).unwrap(); - - // Strict mode refuses because `_schema.pg` exists. - let strict_err = match Omnigraph::init(uri, TEST_SCHEMA).await { - Ok(_) => panic!("strict init must refuse when orphan _schema.pg exists"), - Err(e) => e, - }; - assert!( - strict_err.to_string().contains("already initialized"), - "strict init must surface AlreadyInitialized (sanity check); got: {strict_err}" - ); - - // Force init succeeds after proving no manifest exists, overwrites the - // orphan file, and proceeds to initialize Lance state. - let db = Omnigraph::init_with_options(uri, TEST_SCHEMA, InitOptions { force: true }) - .await - .expect("force init must succeed when only orphan schema files block strict init"); - - // Confirm the catalog is populated as expected — proves the - // graph is functional after force-recovery, not just that the - // call returned Ok. - assert!( - db.catalog().node_types.contains_key("Person"), - "force-recovered graph must have the new catalog installed" - ); - assert!( - dir.path().join("__schema_state.json").exists(), - "force-recovered graph must have full schema state written" - ); +async fn init_ignores_orphan_schema_files() { + for force in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + fs::write(dir.path().join("_schema.pg"), "orphan source").unwrap(); + let db = Omnigraph::init_with_options(uri, TEST_SCHEMA, InitOptions { force }) + .await + .unwrap(); + assert!(db.catalog().node_types.contains_key("Person")); + assert_eq!(read_contract(uri).await.source, TEST_SCHEMA); + assert_eq!( + fs::read_to_string(dir.path().join("_schema.pg")).unwrap(), + "orphan source" + ); + } } -/// E2e for the schema-level `.pg` surface: `@description` (node / edge / -/// property) and `@instruction` (node / edge only) parse, validate, and -/// persist verbatim into the on-disk `_schema.ir.json` through `Omnigraph::init` -/// — the contract that surfaces them in catalog metadata for tooling. +/// Schema annotations persist in the contract row and catalog metadata. #[tokio::test] async fn schema_annotations_persist_into_ir_json_on_init() { let dir = tempfile::tempdir().unwrap(); @@ -923,7 +811,7 @@ edge DependsOn: Task -> Task @description("Hard dependency") @instruction("Use o Omnigraph::init(uri, schema).await.unwrap(); - let ir_json = fs::read_to_string(dir.path().join("_schema.ir.json")).unwrap(); + let ir_json = read_contract(uri).await.ir; let ir: serde_json::Value = serde_json::from_str(&ir_json).unwrap(); // Helper: collect the {name -> value} map of annotations that carry a diff --git a/crates/omnigraph/tests/merge_projection_cache.rs b/crates/omnigraph/tests/merge_projection_cache.rs index 8210c3f66..b280a4bdd 100644 --- a/crates/omnigraph/tests/merge_projection_cache.rs +++ b/crates/omnigraph/tests/merge_projection_cache.rs @@ -1,6 +1,6 @@ //! Structural gates for the incremental merge-authority projection cache: a //! repeated merge reuses acknowledged local publication views and refreshes -//! foreign changes through an incremental projection fold, retains at most +//! foreign changes through a checked projection refresh, retains at most //! one non-bound branch's complete authority, and a //! delete/recreate of a cached branch must be fenced to a full re-read, never //! a stale reuse. The explicit fold-vs-full correctness oracle lives with the @@ -106,10 +106,10 @@ async fn diverge(db: &Session, round: i64) { } /// Acknowledged local publishes retain exact projections; an external publish -/// still requires the physical-address incremental fold. Both paths avoid a -/// full history rebuild and must preserve the merged payload. +/// still requires a refresh of the replaced manifest fragment. The local path +/// performs no projection rebuild; both paths must preserve the merged payload. #[test] -fn repeated_merge_refreshes_projection_incrementally() { +fn repeated_merge_reuses_local_projection_and_refreshes_foreign() { on_big_stack(|| async { cost_harness(async { let dir = tempfile::tempdir().unwrap(); @@ -126,12 +126,9 @@ fn repeated_merge_refreshes_projection_incrementally() { let (outcome, io) = measure(db.branch_merge("feature", "main")).await; assert_eq!(outcome.unwrap().outcome, MergeOutcome::Merged); - assert!( - io.projection_incremental_refreshes >= 1, - "the repeated merge must refresh at least one cached branch authority \ - through the incremental projection fold (incremental {}, full {})", - io.projection_incremental_refreshes, - io.projection_full_refreshes, + assert_eq!( + io.projection_incremental_refreshes, 0, + "acknowledged local publication needs only an incarnation probe, not a projection fold", ); eprintln!("local publication reuse: {io:?}"); assert_eq!( @@ -151,7 +148,7 @@ fn repeated_merge_refreshes_projection_incrementally() { "the ground-truth object-store tracker must measure returned bytes" ); eprintln!( - "incremental repeated merge: manifest_reads={} manifest_read_bytes={}", + "local projection reuse: manifest_reads={} manifest_read_bytes={}", io.manifest_reads, io.manifest_read_bytes, ); // Keep a fixed ground-truth request ceiling alongside the @@ -160,7 +157,7 @@ fn repeated_merge_refreshes_projection_incrementally() { // full coordinator reopen to multiply the measured reads. assert!( io.manifest_reads <= 32, - "incremental repeated merge used {} manifest object reads; hidden full scans must not ride the measured path", + "local projection reuse used {} manifest object reads; hidden full scans must not ride the measured path", io.manifest_reads, ); @@ -185,7 +182,7 @@ fn repeated_merge_refreshes_projection_incrementally() { assert!(foreign_io.manifest_reads > 0 && foreign_io.manifest_read_bytes > 0); assert!( foreign_io.manifest_reads <= 40, - "foreign-source merge input protection and incremental refresh used {} manifest reads", + "foreign-source merge input protection and projection refresh used {} manifest reads", foreign_io.manifest_reads, ); diff --git a/crates/omnigraph/tests/point_in_time.rs b/crates/omnigraph/tests/point_in_time.rs index 574af606a..d6ca67480 100644 --- a/crates/omnigraph/tests/point_in_time.rs +++ b/crates/omnigraph/tests/point_in_time.rs @@ -309,6 +309,9 @@ async fn historical_query_resolves_rename_by_identity_but_rejects_reincarnation( .await .unwrap(); let before_rename = version_main(&db).await.unwrap(); + let stale_handle = session(Omnigraph::open(uri).await.unwrap()); + let stale_snapshot_handle = session(Omnigraph::open(uri).await.unwrap()); + let snapshot_before_rename = db.resolve_snapshot("main").await.unwrap(); db.apply_schema( "node Human @rename_from(\"Person\") { name: String @key }\n\ @@ -329,6 +332,32 @@ async fn historical_query_resolves_rename_by_identity_but_rejects_reincarnation( collect_column_strings(renamed_view.batches(), "p.name"), vec!["Alice".to_string()] ); + let stale_view = stale_handle + .run_query_at( + before_rename, + ALL_HUMANS_QUERY, + "all_humans", + &ParamMap::new(), + ) + .await + .unwrap(); + assert_eq!( + collect_column_strings(stale_view.batches(), "p.name"), + vec!["Alice"] + ); + let stale_snapshot_view = stale_snapshot_handle + .query( + omnigraph::db::ReadTarget::Snapshot(snapshot_before_rename), + ALL_HUMANS_QUERY, + "all_humans", + &ParamMap::new(), + ) + .await + .unwrap(); + assert_eq!( + collect_column_strings(stale_snapshot_view.batches(), "p.name"), + vec!["Alice"] + ); db.apply_schema( "node Human { name: String @key }\n\ diff --git a/crates/omnigraph/tests/recovery.rs b/crates/omnigraph/tests/recovery.rs index 620bd3a84..984c81c89 100644 --- a/crates/omnigraph/tests/recovery.rs +++ b/crates/omnigraph/tests/recovery.rs @@ -191,39 +191,23 @@ async fn recovery_ensure_indices_handles_empty_tables() { ); } -/// `OpenMode::ReadOnly` must NOT run `recover_schema_state_files`, -/// which can delete or rename schema-staging files. Read-only consumers -/// may run with read-only object-store credentials, and silent open-time -/// mutations violate the contract. -/// -/// This test drops a schema-staging file (which the recovery sweep -/// would normally delete) then opens with ReadOnly mode. The staging -/// file must remain untouched. +/// Served roots ignore legacy schema artifacts in both open modes. #[tokio::test] -async fn read_only_open_skips_schema_state_recovery() { +async fn open_ignores_legacy_schema_files() { let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); - - let _ = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - - // Drop a leftover schema-staging file. The schema-state recovery - // sweep would normally tidy this on open (either delete or rename - // depending on whether it matches the live schema). ReadOnly must - // skip that work. + Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); let staging_path = dir.path().join("_schema.pg.staging"); - std::fs::write(&staging_path, "node Person { name: String @key }\n").unwrap(); - assert!(staging_path.exists()); - - let _db = Omnigraph::open_read_only(uri).await.unwrap(); - - // Staging file must be untouched. - assert!( - staging_path.exists(), - "ReadOnly open must not delete schema-staging files (no object-store mutations)" - ); - let content = std::fs::read_to_string(&staging_path).unwrap(); - assert_eq!( - content, "node Person { name: String @key }\n", - "staging file content must be unchanged" - ); + let contents = "invalid legacy schema residue"; + std::fs::write(&staging_path, contents).unwrap(); + for read_only in [false, true] { + let db = if read_only { + Omnigraph::open_read_only(uri).await + } else { + Omnigraph::open(uri).await + } + .unwrap(); + assert_eq!(db.schema_source().as_str(), TEST_SCHEMA); + assert_eq!(std::fs::read_to_string(&staging_path).unwrap(), contents); + } } diff --git a/crates/omnigraph/tests/recovery_probe.rs b/crates/omnigraph/tests/recovery_probe.rs index 4b59ec23a..aa7184b0c 100644 --- a/crates/omnigraph/tests/recovery_probe.rs +++ b/crates/omnigraph/tests/recovery_probe.rs @@ -26,7 +26,7 @@ fn files(root: &Path) -> BTreeMap> { } #[tokio::test] -async fn recovery_probe_refuses_unknown_sidecars_and_staging_without_writes() { +async fn recovery_probe_refuses_unknown_sidecars_without_writes() { let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); Omnigraph::init(uri, "node Person { name: String @key }") @@ -48,24 +48,6 @@ async fn recovery_probe_refuses_unknown_sidecars_and_staging_without_writes() { assert_eq!(files(dir.path()), before); fs::remove_file(pending).unwrap(); - for name in [ - "_schema.pg.staging", - "_schema.ir.json.staging", - "__schema_state.json.staging", - ] { - let pending = dir.path().join(name); - fs::write(&pending, "uncertain staging").unwrap(); - let before = files(dir.path()); - let err = Omnigraph::ensure_no_pending_recovery(uri) - .await - .unwrap_err(); - assert!( - err.to_string().contains("staged schema recovery"), - "{name}: {err}" - ); - assert_eq!(files(dir.path()), before); - fs::remove_file(pending).unwrap(); - } // Existing inventory semantics ignore non-JSON residue within the bound. fs::write(recovery.join("note.txt"), "not a recovery sidecar").unwrap(); let before = files(dir.path()); diff --git a/crates/omnigraph/tests/schema_apply.rs b/crates/omnigraph/tests/schema_apply.rs index 4de6a5f12..5772c755d 100644 --- a/crates/omnigraph/tests/schema_apply.rs +++ b/crates/omnigraph/tests/schema_apply.rs @@ -1,6 +1,5 @@ mod helpers; -use std::fs; #[cfg(feature = "failpoints")] use std::sync::Arc; @@ -162,6 +161,23 @@ async fn apply_schema_interface_evolution_is_identity_only_and_durable() { table_versions_after.sort(); assert_eq!(table_versions_after, table_versions_before); + let contract_row = omnigraph_catalog::ManifestCoordinator::open(uri) + .await + .unwrap() + .read_schema_contract() + .await + .expect("the apply's publish replaced the schema_contract row"); + let accepted_ir = db.catalog().bound_schema_ir().unwrap().clone(); + assert_eq!( + contract_row.head.schema_ir_hash, + omnigraph_compiler::schema_ir_hash(&accepted_ir).unwrap() + ); + assert_eq!(contract_row.source, extended_interface); + assert_eq!( + contract_row.ir, + omnigraph_compiler::schema_ir_pretty_json(&accepted_ir).unwrap() + ); + drop(db); let reopened = Omnigraph::open(uri).await.unwrap(); assert_eq!(reopened.catalog().type_id("Named"), Some(interface_id)); @@ -393,11 +409,8 @@ async fn mutation_waits_for_mid_apply_schema_gate_then_reprepares() { }); mutation_rv.wait_until_reached().await; - // Start schema apply and park it after its staging files (and any table - // rewrite) exist but before manifest/schema promotion. The outer apply owns - // the schema-control gate throughout this window. let schema_rv = - helpers::failpoint::Rendezvous::park_first(&catalog::SCHEMA_APPLY_AFTER_STAGING_WRITE); + helpers::failpoint::Rendezvous::park_first(&catalog::SCHEMA_APPLY_POST_LOCK_PRE_EFFECT); let schema_db = Arc::clone(&db); let schema_task = tokio::spawn(async move { schema_db.apply_schema(&desired).await }); schema_rv.wait_until_reached().await; @@ -429,9 +442,7 @@ async fn mutation_waits_for_mid_apply_schema_gate_then_reprepares() { assert_eq!(count_rows(&db, "node:Person").await, 5); } -/// The mis-classified-writer tripwire: a writer parked inside its envelope -/// holds its SHARED permit, so the apply must not reach the seam after its -/// sentinel. Falsified by dropping the permit in `HeldWriteGates::new`: red. +/// A writer holds its shared schema permit through publication. #[cfg(feature = "failpoints")] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial_test::serial] @@ -457,13 +468,13 @@ async fn parked_writer_blocks_schema_apply() { }); in_envelope.wait_until_reached().await; - let post_sentinel = - helpers::failpoint::Rendezvous::park_first(&catalog::SCHEMA_APPLY_POST_SENTINEL); + let post_gate = + helpers::failpoint::Rendezvous::park_first(&catalog::SCHEMA_APPLY_POST_LOCK_PRE_EFFECT); let schema_db = Arc::clone(&db); let schema_task = tokio::spawn(async move { schema_db.apply_schema(&desired).await }); // Wall time: an apply past the gate makes store requests before the seam. let crossed = tokio::time::timeout(std::time::Duration::from_secs(1), async { - while !post_sentinel.reached() { + while !post_gate.reached() { tokio::time::sleep(std::time::Duration::from_millis(5)).await; } }) @@ -471,7 +482,7 @@ async fn parked_writer_blocks_schema_apply() { assert!( crossed.is_err(), "schema apply must wait behind a writer's held shared schema permit; it \ - created its sentinel while the writer was parked", + crossed its effect gate while the writer was parked", ); assert!(!schema_task.is_finished()); @@ -480,8 +491,8 @@ async fn parked_writer_blocks_schema_apply() { .await .unwrap() .expect("the parked writer must publish after release"); - post_sentinel.wait_until_reached().await; - post_sentinel.release(); + post_gate.wait_until_reached().await; + post_gate.release(); schema_task .await .unwrap() @@ -489,10 +500,7 @@ async fn parked_writer_blocks_schema_apply() { assert_eq!(count_rows(&db, "node:Person").await, 5); } -/// ReadOnly opens participate in the process-local schema publication gate even -/// though they perform no recovery writes. Park an open immediately before its -/// source/IR/state read: schema apply must not reach staging until that coherent -/// catalog capture finishes. +/// Read-only opens hold the schema gate through coherent catalog capture. #[cfg(feature = "failpoints")] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial_test::serial] @@ -514,7 +522,7 @@ async fn read_only_open_holds_schema_gate_through_catalog_capture() { open_rv.wait_until_reached().await; let apply_rv = - helpers::failpoint::Rendezvous::park_first(&catalog::SCHEMA_APPLY_AFTER_STAGING_WRITE); + helpers::failpoint::Rendezvous::park_first(&catalog::SCHEMA_APPLY_POST_LOCK_PRE_EFFECT); let apply_owner = Arc::clone(&owner); let apply_task = tokio::spawn(async move { apply_owner.apply_schema(&desired).await }); assert!( @@ -571,7 +579,7 @@ async fn refresh_holds_schema_gate_through_catalog_publication() { reload_rv.wait_until_reached().await; let apply_rv = - helpers::failpoint::Rendezvous::park_first(&catalog::SCHEMA_APPLY_AFTER_STAGING_WRITE); + helpers::failpoint::Rendezvous::park_first(&catalog::SCHEMA_APPLY_POST_LOCK_PRE_EFFECT); let apply_owner = Arc::clone(&owner); let apply_task = tokio::spawn(async move { apply_owner.apply_schema(&desired).await }); assert!( @@ -601,18 +609,32 @@ async fn refresh_holds_schema_gate_through_catalog_publication() { #[tokio::test] #[cfg_attr(feature = "failpoints", serial_test::parallel)] -async fn plan_schema_rejects_when_schema_contract_has_drifted() { +async fn plan_schema_on_unrefreshed_handle_plans_against_the_applied_contract() { let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); - let db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); + let schema_owner = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); + let stale_handle = Omnigraph::open(uri).await.unwrap(); - let drifted = TEST_SCHEMA.replace("age: I32?", "age: I64?"); - fs::write(dir.path().join("_schema.pg"), drifted).unwrap(); + let with_nickname = TEST_SCHEMA.replace("age: I32?", "age: I32?\n nickname: String?"); + schema_owner.apply_schema(&with_nickname).await.unwrap(); - let err = db.plan_schema(TEST_SCHEMA).await.unwrap_err(); + let plan = stale_handle.plan_schema(TEST_SCHEMA).await.unwrap(); assert!( - err.to_string() - .contains("current _schema.pg no longer matches the accepted compiled schema") + plan.steps.iter().any(|step| matches!( + step, + SchemaMigrationStep::DropProperty { type_name, property_name, .. } + if type_name == "Person" && property_name == "nickname" + )), + "an unrefreshed handle plans against the applied contract, so the original schema drops the added property: {:?}", + plan.steps + ); + assert!( + stale_handle + .plan_schema(&with_nickname) + .await + .unwrap() + .steps + .is_empty() ); } diff --git a/crates/omnigraph/tests/system_column_upgrade.rs b/crates/omnigraph/tests/system_column_upgrade.rs index 76af1ac0f..38dccd7de 100644 --- a/crates/omnigraph/tests/system_column_upgrade.rs +++ b/crates/omnigraph/tests/system_column_upgrade.rs @@ -7,8 +7,6 @@ mod helpers; -use std::fs; - use helpers::recovery::sidecar_operation_ids; use helpers::*; use omnigraph::Session; @@ -77,28 +75,14 @@ async fn legacy_graph_with_data(dir: &tempfile::TempDir) -> Session { db } -fn schema_ir(dir: &tempfile::TempDir) -> serde_json::Value { - serde_json::from_str(&fs::read_to_string(dir.path().join("_schema.ir.json")).unwrap()).unwrap() -} - -/// The lock is a native `__manifest` branch: its ref file is -/// `__schema_apply_lock__..json` under the dataset's branch refs, and -/// a deleted branch keeps that file with a retirement marker inside. -fn schema_apply_lock_present(dir: &tempfile::TempDir) -> bool { - let refs = dir.path().join("__manifest").join("_refs").join("branches"); - fs::read_dir(&refs) - .map(|entries| { - entries.filter_map(Result::ok).any(|entry| { - entry - .file_name() - .to_string_lossy() - .starts_with("__schema_apply_lock__") - && !fs::read_to_string(entry.path()) - .unwrap() - .contains("omnigraph.retired_manifest_branch") - }) - }) - .unwrap_or_else(|error| panic!("{}: {error}", refs.display())) +async fn schema_ir(dir: &tempfile::TempDir) -> serde_json::Value { + let row = omnigraph_catalog::ManifestCoordinator::open(dir.path().to_str().unwrap()) + .await + .unwrap() + .read_schema_contract() + .await + .unwrap(); + serde_json::from_str(&row.ir).unwrap() } async fn assert_history_readable(db: &Session, version_before: u64, snapshot_before: &SnapshotId) { @@ -149,7 +133,7 @@ async fn assert_upgraded(db: &Session, dir: &tempfile::TempDir, expected_export: .unwrap(), omnigraph::db::manifest::INTERNAL_MANIFEST_SCHEMA_VERSION ); - let ir = schema_ir(dir); + let ir = schema_ir(dir).await; assert_eq!(ir["ir_version"].as_u64(), Some(5)); assert!( ir["features"] @@ -159,7 +143,7 @@ async fn assert_upgraded(db: &Session, dir: &tempfile::TempDir, expected_export: .any(|feature| feature == "system-columns"), "the promoted IR must record the current vintage: {ir}" ); - let source = fs::read_to_string(dir.path().join("_schema.pg")).unwrap(); + let source = db.schema_source(); assert!( source.contains("@unique(@src, @dst)"), "the promoted source must spell its endpoints as meta-fields: {source}" @@ -283,7 +267,7 @@ async fn system_column_upgrade_respells_a_legacy_graph_in_place() { omnigraph::db::manifest::INTERNAL_MANIFEST_SCHEMA_VERSION, "check mode writes nothing" ); - assert_eq!(schema_ir(&dir)["ir_version"].as_u64(), Some(2)); + assert_eq!(schema_ir(&dir).await["ir_version"].as_u64(), Some(2)); let report = db .upgrade_system_columns(SystemColumnUpgradeOptions::default()) @@ -345,24 +329,26 @@ async fn system_column_upgrade_respells_a_legacy_graph_in_place() { ); } -/// A legacy-column graph reaches this binary at v11 (every upgrade route ends -/// there) and is respelled as it is; the respelling's own publish converts -/// main to v12 like any publish, and the report says so. +/// A flat-v11 source needs storage conversion before system-column respelling. #[tokio::test] -async fn system_column_upgrade_respells_a_v11_graph_and_its_publish_converts_it() { +async fn system_column_upgrade_respells_a_legacy_graph_after_storage_conversion() { let _scenario = FailScenario::setup(); let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); let db = legacy_graph_with_data(&dir).await; let export_before = db.export_jsonl("main", &[]).await.unwrap(); drop(db); - let mut manifest = lance::Dataset::open(&format!("{uri}/__manifest")) + helpers::make_legacy_flat_manifest_fixture(dir.path(), 11).await; + assert!(Omnigraph::open(uri).await.is_err()); + assert!(Omnigraph::open_read_only(uri).await.is_err()); + let upgraded = omnigraph::db::upgrade_storage(uri, omnigraph::db::UpgradeOptions::default()) .await .unwrap(); - omnigraph_catalog::migrations::restamp_flat_for_test(&mut manifest, 11) - .await - .unwrap(); - drop(manifest); + assert_eq!( + upgraded.outcome, + omnigraph::db::UpgradeOutcome::Completed, + "{upgraded:?}" + ); let db = helpers::session(Omnigraph::open(uri).await.unwrap()); let check = db @@ -374,7 +360,7 @@ async fn system_column_upgrade_respells_a_v11_graph_and_its_publish_converts_it( SystemColumnUpgradeOutcome::CheckPassed, "{check:?}" ); - assert_eq!((check.stamp_before, check.stamp_after), (11, 11)); + assert_eq!((check.stamp_before, check.stamp_after), (13, 13)); let report = db .upgrade_system_columns(SystemColumnUpgradeOptions::default()) @@ -385,13 +371,13 @@ async fn system_column_upgrade_respells_a_v11_graph_and_its_publish_converts_it( SystemColumnUpgradeOutcome::Completed, "{report:?}" ); - assert_eq!((report.stamp_before, report.stamp_after), (11, 12)); + assert_eq!((report.stamp_before, report.stamp_after), (13, 13)); assert_eq!( db.internal_schema_version_of(omnigraph::db::ReadTarget::branch("main")) .await .unwrap(), - 12, - "the respelling's publish converts main like any publish" + 13, + "system-column respelling preserves the served storage format" ); assert_eq!(db.export_jsonl("main", &[]).await.unwrap(), export_before); assert_eq!(count_rows(&db, "node:Person").await, 2); @@ -554,7 +540,7 @@ async fn system_column_upgrade_refuses_before_any_effect() { omnigraph::db::manifest::INTERNAL_MANIFEST_SCHEMA_VERSION ); assert!(sidecar_operation_ids(dir.path()).is_empty()); - assert_eq!(schema_ir(&dir)["ir_version"].as_u64(), Some(2)); + assert_eq!(schema_ir(&dir).await["ir_version"].as_u64(), Some(2)); let reserved_dir = tempfile::tempdir().unwrap(); let reserved_uri = reserved_dir.path().to_str().unwrap(); @@ -603,11 +589,7 @@ fn assert_no_staging(dir: &tempfile::TempDir) { } } -/// A failure before the upgrade's one manifest commit leaves the graph -/// exactly as it was: the plain error, no sidecar, the sentinel released, the -/// legacy contract still served (a read-only open included), every linear -/// HEAD at its pin, any staged contract discarded by the next read-write -/// open, and the retry completes. +/// Pre-publication failure preserves the old contract and admits a retry. async fn crash_before_publication_leaves_no_residue(seam: &'static DecideSeam) { let failpoint = seam.name(); let _scenario = FailScenario::setup(); @@ -632,10 +614,6 @@ async fn crash_before_publication_leaves_no_residue(seam: &'static DecideSeam) { "nothing was published at {failpoint}, so nothing needs recovery: {error}" ); assert!(sidecar_operation_ids(dir.path()).is_empty()); - assert!( - !schema_apply_lock_present(&dir), - "a failed upgrade releases its sentinel" - ); assert_eq!( person_head_and_pin(&db, &dir).await, (head_before, pin_before), @@ -676,21 +654,9 @@ async fn system_column_upgrade_failure_after_the_first_rename_leaves_no_residue( crash_before_publication_leaves_no_residue(&catalog::SCHEMA_APPLY_POST_TABLE_COMMIT).await; } +/// A failure after publication leaves the complete contract readable in both modes. #[tokio::test] -async fn system_column_upgrade_failure_before_staging_leaves_no_residue() { - crash_before_publication_leaves_no_residue(&catalog::SCHEMA_APPLY_BEFORE_STAGING_WRITE).await; -} - -#[tokio::test] -async fn system_column_upgrade_failure_after_staging_leaves_no_residue() { - crash_before_publication_leaves_no_residue(&catalog::SCHEMA_APPLY_AFTER_STAGING_WRITE).await; -} - -/// A failure after the manifest commit reports the published commit. The -/// manifest already names the renamed tables, so a read-only open refuses -/// the uninstalled contract and the next read-write open installs it. -#[tokio::test] -async fn system_column_upgrade_post_commit_failure_is_finished_by_the_next_open() { +async fn system_column_upgrade_post_commit_failure_is_readable_in_both_open_modes() { let _scenario = FailScenario::setup(); let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); @@ -704,21 +670,22 @@ async fn system_column_upgrade_post_commit_failure_is_finished_by_the_next_open( }; assert!( matches!(error, omnigraph::error::OmniError::RecoveryRequired { .. }), - "a published upgrade whose contract is not installed names its commit: {error}" + "a post-publication failure names its committed upgrade: {error}" ); assert!(sidecar_operation_ids(dir.path()).is_empty()); drop(db); - let read_only = Omnigraph::open_read_only(uri) - .await - .err() - .expect("a read-only open must not serve a published upgrade under the old contract"); - assert!(read_only.to_string().contains("read-write"), "{read_only}"); + let read_only = helpers::session(Omnigraph::open_read_only(uri).await.unwrap()); + assert!(read_only.schema_source().contains("@unique(@src, @dst)")); + assert_eq!( + read_only.export_jsonl("main", &[]).await.unwrap(), + export_before + ); let recovered = helpers::session( Omnigraph::open(uri) .await - .expect("the read-write open installs the published contract"), + .expect("the read-write open reads the published contract"), ); assert_no_staging(&dir); assert_upgraded(&recovered, &dir, &export_before).await; @@ -781,13 +748,13 @@ async fn system_column_upgrade_writes_no_control_object() { assert_eq!(report.outcome, SystemColumnUpgradeOutcome::Completed); assert_eq!( counts.write_text() - before_write_text, - 6, - "the upgrade writes the staged and live contract files and no sidecar" + 0, + "the upgrade publishes its contract without control-object writes" ); assert_eq!( counts.delete() - before_delete, - 3, - "the upgrade deletes only its three staging files" + 0, + "the upgrade has no contract files to delete" ); assert!(sidecar_operation_ids(dir.path()).is_empty()); } @@ -815,10 +782,9 @@ async fn system_column_upgrade_retries_on_the_same_handle_after_a_crash() { assert_upgraded(&db, &dir, &export_before).await; } -/// A writer that dies holding the sentinel leaves it behind with nothing -/// published; the next read-write open reclaims it and the upgrade runs. +/// A panicking writer releases its process gate without publishing. #[tokio::test] -async fn system_column_upgrade_open_reclaims_a_dead_writers_lock() { +async fn system_column_upgrade_crashed_writer_releases_process_gate() { let _scenario = FailScenario::setup(); let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); @@ -835,23 +801,15 @@ async fn system_column_upgrade_open_reclaims_a_dead_writers_lock() { }; assert!( crashed - .expect_err("the writer dies under its sentinel, releasing nothing") + .expect_err("the writer panics while holding its process gate") .is_panic() ); - assert!( - schema_apply_lock_present(&dir), - "a dead writer leaves its lock behind" - ); assert!(sidecar_operation_ids(dir.path()).is_empty()); let recovered = helpers::session( Omnigraph::open(uri) .await - .expect("the read-write open reclaims the stale sentinel"), - ); - assert!( - !schema_apply_lock_present(&dir), - "the open reclaims the dead writer's lock" + .expect("the graph remains readable after the panic"), ); assert_eq!( recovered.export_jsonl("main", &[]).await.unwrap(), diff --git a/crates/omnigraph/tests/warm_read_cost.rs b/crates/omnigraph/tests/warm_read_cost.rs index c5bea9e1d..3a93c96e3 100644 --- a/crates/omnigraph/tests/warm_read_cost.rs +++ b/crates/omnigraph/tests/warm_read_cost.rs @@ -37,14 +37,14 @@ async fn warm_same_branch_read_does_no_resolution_opens() { // Deep history: warm-read resolution cost must be flat in commit count. commit_many(&db, 20).await; - let (out, io) = measure(db.query( + let (out, io) = measure(db.query_with_head( ReadTarget::branch("main"), TEST_QUERIES, "total_people", ¶ms(&[]), )) .await; - out.unwrap(); + let (_, head) = out.unwrap(); // A warm same-branch read opens nothing from the internal tables, even at // commit-history depth. Fix 1 reuses the coordinator (no re-open: 0 @@ -60,6 +60,15 @@ async fn warm_same_branch_read_does_no_resolution_opens() { io.version_probes, 1, "warm same-branch read performs exactly one version probe" ); + for branch in ["main", " main "] { + let (resolved, io) = measure(db.resolve_snapshot(branch)).await; + let resolved = resolved.unwrap(); + assert_eq!(Some(resolved.as_str()), head.as_deref()); + assert_eq!(io.internal_open_count, 0); + assert_eq!(io.manifest_scan_count, 0); + assert_eq!(io.version_probes, 1); + } + assert!(db.resolve_snapshot(" ").await.is_err()); }) .await; } @@ -104,6 +113,7 @@ async fn external_commit_observed_by_warm_reader() { let reader = Omnigraph::open(uri).await.unwrap(); let before = count_rows(&reader, "node:Person").await; + let old_head = reader.resolve_snapshot("main").await.unwrap(); // External commit through a separate handle. mutate_main( @@ -115,6 +125,10 @@ async fn external_commit_observed_by_warm_reader() { .await .unwrap(); + let resolved = reader.resolve_snapshot("main").await.unwrap(); + assert_ne!(resolved, old_head); + assert_eq!(resolved, writer.resolve_snapshot("main").await.unwrap()); + let after = count_rows(&reader, "node:Person").await; assert_eq!( after, @@ -123,20 +137,179 @@ async fn external_commit_observed_by_warm_reader() { ); } -// ── Finding A: drop the redundant per-query schema validation ───────────────── -// -// Every query runs `ensure_schema_state_valid`. It ran TWICE per query (once in -// query()/run_query_at, once again in resolved_target/snapshot_at_graph_manifest_version), each -// reading 3 contract files + 2 existence probes (~10 storage ops). Finding A -// removes the redundant caller, so validation runs once. (A cheaper source-only -// probe was rejected: the codebase requires per-call detection of IR/state drift -// on long-lived handles -- lifecycle::long_lived_handle_rejects_schema_ir_drift -// -- which a source-only compare would miss.) Measured at the StorageAdapter -// boundary with the counting decorator. - -/// A warm query validates the schema contract exactly once (3 reads + 2 exists), -/// not twice. Fails before finding A, where query() and resolved_target each -/// validate (6 read_text + 4 exists). +#[tokio::test] +async fn cold_open_admits_contract_and_state_in_one_scan() { + cost_harness(async { + let dir = tempfile::tempdir().unwrap(); + drop(init_and_load(&dir).await); + let uri = dir.path().to_str().unwrap(); + for read_only in [false, true] { + let (opened, io) = measure(async { + if read_only { + Omnigraph::open_read_only(uri).await + } else { + Omnigraph::open(uri).await + } + }) + .await; + let db = opened.unwrap(); + assert_eq!(db.schema_source().as_str(), TEST_SCHEMA); + assert_eq!(io.manifest_scan_count, 1, "read_only={read_only}: {io:?}"); + assert_eq!(io.internal_open_count, 2, "read_only={read_only}: {io:?}"); + assert!(io.manifest_reads > 0); + let db = session(db); + let (rows, query_io) = measure(db.query( + ReadTarget::branch("main"), + TEST_QUERIES, + "get_person", + ¶ms(&[("$name", "Alice")]), + )) + .await; + assert_eq!(rows.unwrap().num_rows(), 1); + assert_eq!(query_io.manifest_scan_count, 0); + } + }) + .await; +} + +#[tokio::test] +async fn named_read_borrows_write_capture_and_rejects_recreated_branch() { + cost_harness(async { + let dir = tempfile::tempdir().unwrap(); + let db = init_and_load(&dir).await; + let writer = session(Omnigraph::open(dir.path().to_str().unwrap()).await.unwrap()); + db.branch_create("feature").await.unwrap(); + mutate_branch( + &db, + "feature", + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", "OldFeature")], &[("$age", 22)]), + ) + .await + .unwrap(); + let old_head = db.resolve_snapshot("feature").await.unwrap(); + let old_version = db + .graph_manifest_version_of(ReadTarget::branch("feature")) + .await + .unwrap(); + let (rows, io) = measure(db.query_with_head( + ReadTarget::branch("feature"), + TEST_QUERIES, + "get_person", + ¶ms(&[("$name", "OldFeature")]), + )) + .await; + let (rows, head) = rows.unwrap(); + assert_eq!(rows.num_rows(), 1); + assert_eq!(head.as_deref(), Some(old_head.as_str())); + assert_eq!(io.manifest_scan_count, 0); + assert_eq!(io.version_probes, 1); + + writer.branch_delete("feature").await.unwrap(); + mutate_main( + &writer, + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", "Replacement")], &[("$age", 44)]), + ) + .await + .unwrap(); + let head = writer.resolve_snapshot("main").await.unwrap(); + writer.branch_create("feature").await.unwrap(); + assert_eq!( + writer + .graph_manifest_version_of(ReadTarget::branch("feature")) + .await + .unwrap(), + old_version, + ); + for _ in 0..2 { + let (rows, io) = measure(db.query_with_head( + ReadTarget::branch("feature"), + TEST_QUERIES, + "get_person", + ¶ms(&[("$name", "Replacement")]), + )) + .await; + let (rows, served_head) = rows.unwrap(); + assert_eq!(rows.num_rows(), 1); + assert_eq!(served_head.as_deref(), Some(head.as_str())); + assert_eq!( + io.manifest_scan_count, 1, + "reads must not refill the write cache" + ); + } + }) + .await; +} + +#[tokio::test] +async fn branch_delete_reuses_only_a_current_write_capture() { + cost_harness(async { + let dir = tempfile::tempdir().unwrap(); + let db = init_and_load(&dir).await; + let writer = session(Omnigraph::open(dir.path().to_str().unwrap()).await.unwrap()); + for foreign in ["none", "write", "recreate"] { + db.branch_create("feature").await.unwrap(); + mutate_branch( + &db, + "feature", + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", "Local")], &[("$age", 22)]), + ) + .await + .unwrap(); + if foreign == "write" { + mutate_branch( + &writer, + "feature", + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", "Foreign")], &[("$age", 33)]), + ) + .await + .unwrap(); + } + if foreign == "recreate" { + let old_version = writer + .graph_manifest_version_of(ReadTarget::branch("feature")) + .await + .unwrap(); + writer.branch_delete("feature").await.unwrap(); + mutate_main( + &writer, + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", "NewMain")], &[("$age", 44)]), + ) + .await + .unwrap(); + writer.branch_create("feature").await.unwrap(); + assert_eq!( + writer + .graph_manifest_version_of(ReadTarget::branch("feature")) + .await + .unwrap(), + old_version, + ); + } + let main_rows = count_rows(&writer, "node:Person").await; + let (deleted, io) = measure(db.branch_delete("feature")).await; + deleted.unwrap(); + assert_eq!(io.manifest_scan_count, u64::from(foreign != "none")); + assert_eq!(io.version_probes, 1); + assert!(io.manifest_reads > 0); + assert!(db.resolve_snapshot("feature").await.is_err()); + assert_eq!(count_rows(&db, "node:Person").await, main_rows); + } + }) + .await; +} + +/// A warm query performs no contract read: 0 `read_text` and 0 `exists` +/// (before the row: 3 reads + 2 exists per query, the three contract files). #[tokio::test] async fn warm_query_validates_schema_contract_once() { use omnigraph::instrumentation::CountingStorageAdapter; @@ -164,45 +337,59 @@ async fn warm_query_validates_schema_contract_once() { assert_eq!( counts.read_text() - before_read_text, - 3, - "warm query should validate the schema contract once (3 reads), not twice" + 0, + "a warm query reads no schema contract file" ); assert_eq!( counts.exists() - before_exists, - 2, - "warm query should probe contract-file existence once (2 probes), not twice" + 0, + "a warm query probes no schema contract file" ); } -/// The cheap source-compare must still detect that the on-disk schema source has -/// drifted from the validated contract and fail the read, rather than serving the -/// stale-but-cached schema. Passes before and after finding A (regression guard -/// for the documented weaker per-query guard). +/// A handle serving 1,000 queries after open performs zero `StorageAdapter` +/// reads of any kind: the unchanged captured manifest image reuses its +/// validated contract row and accepted catalog. #[tokio::test] -async fn schema_source_drift_is_caught_on_read() { +async fn thousand_warm_queries_read_no_schema_contract() { + use omnigraph::instrumentation::CountingStorageAdapter; + use omnigraph::storage::storage_for_uri; + let dir = tempfile::tempdir().unwrap(); - let _writer = init_and_load(&dir).await; + let _ = init_and_load(&dir).await; let uri = dir.path().to_str().unwrap(); - let reader = session(Omnigraph::open(uri).await.unwrap()); - - // Drift the on-disk schema source behind the reader's back. - std::fs::write( - dir.path().join("_schema.pg"), - "this is not a valid schema {{{", - ) - .unwrap(); + let (adapter, counts) = CountingStorageAdapter::new(storage_for_uri(uri).unwrap()); + let db = session(Omnigraph::open_with_storage(uri, adapter).await.unwrap()); - let result = reader - .query( + let before = [ + counts.read_text(), + counts.read_text_if_exists(), + counts.read_bytes_if_exists(), + counts.exists(), + counts.read_text_versioned(), + counts.list_dir(), + ]; + for _ in 0..1_000 { + db.query( ReadTarget::branch("main"), TEST_QUERIES, "total_people", ¶ms(&[]), ) - .await; - assert!( - result.is_err(), - "a query must fail when the on-disk schema source has drifted from the validated contract" + .await + .unwrap(); + } + let after = [ + counts.read_text(), + counts.read_text_if_exists(), + counts.read_bytes_if_exists(), + counts.exists(), + counts.read_text_versioned(), + counts.list_dir(), + ]; + assert_eq!( + after, before, + "1,000 warm queries must perform no StorageAdapter read (read_text, read_text_if_exists, read_bytes_if_exists, exists, read_text_versioned, list_dir)" ); } @@ -223,7 +410,18 @@ async fn warm_branch_read_uses_one_ref_witness_without_manifest_scan() { // The branch snapshot must stay warm and bounded at realistic history // depth; a shallow fixture would hide a cold manifest scan. commit_many(&db, 20).await; + let inherited_head = db.resolve_snapshot("main").await.unwrap(); db.branch_create("feature").await.unwrap(); + db.sync_branch("feature").await.unwrap(); + let (resolved, io) = measure(db.resolve_snapshot("feature")).await; + assert_eq!(resolved.unwrap(), inherited_head); + assert_eq!(io.version_probes, 1); + assert_eq!(io.internal_open_count, 0); + assert_eq!(io.manifest_scan_count, 0); + assert_eq!(io.manifest_reads, 1); + let reads = last_manifest_reads(); + assert_eq!(reads.len(), 1); + assert!(reads[0].contains("_refs/branches/feature") && reads[0].ends_with(".json")); // Write to the branch so its tables are branch-owned (under tree/feature). db.mutate( "feature", @@ -607,8 +805,8 @@ async fn assert_cached_borrower_survives_branch_delete( io.manifest_scan_count, io.version_probes ), - (2, 1, 0), - "cached table-borrower deletion must stay within its measured control cost" + (1, 0, 1), + "cached table-borrower deletion must validate and reuse its current capture" ); assert!( @@ -809,6 +1007,11 @@ async fn warm_read_on_recreated_branch_observes_new_incarnation() { "test setup must exercise branch incarnation reuse at one Lance version" ); + assert_eq!( + reader.resolve_snapshot("feature").await.unwrap(), + replacement_inherited_head, + ); + let (new_feature, io) = measure(reader.query_with_head( ReadTarget::branch("feature"), TEST_QUERIES, diff --git a/crates/omnigraph/tests/write_cost.rs b/crates/omnigraph/tests/write_cost.rs index 857e9414d..3ab55319b 100644 --- a/crates/omnigraph/tests/write_cost.rs +++ b/crates/omnigraph/tests/write_cost.rs @@ -151,10 +151,8 @@ async fn ensure_indices_writes_no_control_object() { ); } -/// RFC 0067: schema apply arms no recovery sidecar. A property addition -/// rewrites one table detached; the only control objects written are the -/// three staged contract files and the three live ones, and the only deletes -/// retire the staging. +/// A property addition rewrites a detached table and publishes its contract +/// in main's manifest. Schema apply writes or deletes no adapter control object. #[tokio::test] async fn schema_apply_writes_no_control_object() { use omnigraph::instrumentation::CountingStorageAdapter; @@ -177,13 +175,13 @@ async fn schema_apply_writes_no_control_object() { .unwrap(); assert_eq!( counts.write_text() - before_write_text, - 6, - "a detached schema apply writes the staged and live contract files and no sidecar" + 0, + "schema apply publishes its contract without control-object writes" ); assert_eq!( counts.delete() - before_delete, - 3, - "a detached schema apply deletes only its three staging files" + 0, + "schema apply has no contract files to delete" ); assert!( !dir.path().join("__recovery").exists() @@ -603,14 +601,9 @@ async fn keyed_insert_routes_through_fenced_adapter_only() { // ── (D) Step-3b capture-once fitness asserts (RED today → GREEN after WriteTxn) ── -/// A write performs one full schema validation while capturing the catalog-bound -/// `WriteTxn`, one trailing state-marker read that fences torn head/schema capture, -/// and one full validation under the pre-effect gates (7 `read_text` + 4 `exists` -/// total). Per-table resolves must not add more validation. The gate read is -/// correctness work: it arbitrates schema identity after preparation and before -/// any detached table effect; no recovery sidecar is written (RFC 0067). The shape is -/// the write twin of `warm_read_cost.rs::warm_query_validates_schema_contract_once`, -/// built with ZERO production change via the counting storage adapter. +/// A write reads no schema contract file (0 `read_text` + 0 `exists`; 7 + 4 +/// before the row): the identity rides the pinned manifest version and the +/// pre-effect gate compares the live version's identity against the token. #[tokio::test] async fn write_schema_io_is_bounded_to_capture_fence_and_effect_gate() { use omnigraph::instrumentation::CountingStorageAdapter; @@ -647,13 +640,10 @@ async fn write_schema_io_is_bounded_to_capture_fence_and_effect_gate() { "schema-contract reads on one write: read_text={read_text_delta} exists={exists_delta}" ); assert_eq!( - read_text_delta, 7, - "a write must do capture validation + trailing identity fence + pre-effect validation (7 reads), not per table", - ); - assert_eq!( - exists_delta, 4, - "a write must probe contract-file existence at capture + pre-effect revalidation (4 probes)", + read_text_delta, 0, + "a write reads no schema contract file: the identity rides the pinned manifest version", ); + assert_eq!(exists_delta, 0, "a write probes no schema contract file",); assert_eq!( write_text_delta, 0, "a detached write arms no recovery sidecar (RFC 0067): no control-object write", @@ -704,52 +694,33 @@ async fn keyed_insert_opens_table_at_most_once() { // ── (E) Ground-truth __manifest counting (PR2.1) — the blind-spot guard ── -/// The warm-coordinator freshness probe rides a long-lived handle, so a per-op -/// (fresh) tracker installed at measure time CANNOT see its reads — that was the -/// blind spot. `cost_harness` attaches the tracker BEFORE the coordinator opens, so -/// the probe's reads ARE counted (`manifest_reads` is ground truth, not just fresh -/// opens). Proven by measuring the same warm write both ways: ground truth strictly -/// exceeds fresh-only, by the probe's object-store RPCs. Reverting the ground-truth -/// wiring (so `manifest_reads` reverts to fresh-per-op) makes the two equal → RED. +/// Named-branch probes use the warm coordinator's store; the whole-operation +/// meter must include reads that a fresh-opener-only meter misses. #[tokio::test] async fn manifest_reads_capture_warm_probe() { - // Fresh-only (no `cost_harness`): the warm coordinator handle was opened outside - // any meter, so the freshness probe's reads escape `manifest_reads`. - // - // Heap-allocate this arm. It is the one measurement in this file that does - // NOT run inside `cost_harness` (which boxes its body for exactly this - // reason), so without the box this test frame carries a full init + - // four-write future *plus* the boxed ground-truth arm below. In a debug - // build those nested async frames accumulate far enough to overflow a - // default test-thread stack — observed on Linux CI while macOS stayed - // under the limit. - let fresh = Box::pin(async { + async fn warm_write() -> u64 { let dir = tempfile::tempdir().unwrap(); let db = local_graph(&dir).await; - commit_many(&db, 3).await; // warm the coordinator - let io = measure_insert(&db, "fresh").await; - eprintln!("fresh-only warm write: __manifest={}", io.manifest_reads); + commit_many(&db, 3).await; + db.branch_create("meter").await.unwrap(); + db.sync_branch("meter").await.unwrap(); + let (result, io) = measure(db.mutate( + "meter", + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", "metered")], &[("$age", 30)]), + )) + .await; + result.unwrap(); io.manifest_reads - }) - .await; - - // Ground truth (`cost_harness`): the same warm probe is now counted. + } + let fresh = Box::pin(warm_write()).await; cost_harness(async move { - let dir = tempfile::tempdir().unwrap(); - let db = local_graph(&dir).await; - commit_many(&db, 3).await; - let io = measure_insert(&db, "ground_truth").await; - eprintln!("ground-truth warm write: __manifest={}", io.manifest_reads); - assert!( - io.manifest_reads > fresh, - "ground-truth __manifest reads {} must exceed fresh-only {fresh} by the \ - warm-coordinator probe's RPCs — else the warm-handle probe is escaping the \ - tracker (the blind spot this guards). Reads: {:#?}", - io.manifest_reads, - last_manifest_reads(), - ); - }) - .await; + let full = warm_write().await; + assert!(full > fresh, + "warm-coordinator probe reads must enter the full meter: full={full}, fresh={fresh}, reads={:#?}", + last_manifest_reads()); + }).await; } // ── (F) Batched committed `@unique` probes — flat in DELTA rows ── @@ -821,3 +792,77 @@ node User { "data-table scan reads per load (batched unique probe)", ); } + +#[tokio::test] +async fn named_write_capture_reuses_the_exact_local_publication() { + cost_harness(async { + let dir = tempfile::tempdir().unwrap(); + let db = init_and_load(&dir).await; + db.branch_create("feature").await.unwrap(); + for (age, expected_scans) in [(31, 2), (32, 0)] { + let (result, io) = measure(helpers::mutate_branch( + &db, + "feature", + MUTATION_QUERIES, + "set_age", + &mixed_params(&[("$name", "Alice")], &[("$age", age)]), + )) + .await; + result.unwrap(); + assert_eq!(io.manifest_scan_count, expected_scans, "age {age}: {io:?}"); + assert!(io.version_probes > 0 && io.manifest_reads > 0); + } + let foreign = helpers::session( + omnigraph::db::Omnigraph::open(dir.path().to_str().unwrap()) + .await + .unwrap(), + ); + helpers::mutate_branch( + &foreign, + "feature", + MUTATION_QUERIES, + "set_age", + &mixed_params(&[("$name", "Bob")], &[("$age", 77)]), + ) + .await + .unwrap(); + let (result, io) = measure(helpers::mutate_branch( + &db, + "feature", + MUTATION_QUERIES, + "set_age", + &mixed_params(&[("$name", "Alice")], &[("$age", 33)]), + )) + .await; + result.unwrap(); + assert!(io.manifest_scan_count > 0 && io.manifest_read_bytes > 0); + for (branch, name, age) in [ + ("feature", "Alice", 33), + ("feature", "Bob", 77), + ("main", "Alice", 30), + ("main", "Bob", 25), + ] { + let result = db + .query( + omnigraph::db::ReadTarget::branch(branch), + helpers::TEST_QUERIES, + "get_person", + &helpers::params(&[("$name", name)]), + ) + .await + .unwrap(); + let batch = result.concat_batches().unwrap(); + assert_eq!(batch.num_rows(), 1); + assert_eq!( + batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + age + ); + } + }) + .await; +} diff --git a/crates/omnigraph/tests/writes.rs b/crates/omnigraph/tests/writes.rs index 2e0e90f71..ba5378506 100644 --- a/crates/omnigraph/tests/writes.rs +++ b/crates/omnigraph/tests/writes.rs @@ -498,43 +498,15 @@ async fn repeated_loads_do_not_accumulate_branches() { assert_eq!(db.branch_list().await.unwrap(), vec!["main".to_string()]); } -/// After MR-770, `__run__*` is an ordinary branch name — the Run state machine -/// and its `is_internal_run_branch` guard are gone. The surviving internal-ref -/// guard still rejects the active `__schema_apply_lock__` branch on the public -/// create/merge APIs. #[tokio::test] -async fn public_branch_apis_reject_internal_system_refs() { +async fn public_branch_apis_accept_former_system_names() { let dir = tempfile::tempdir().unwrap(); let db = init_and_load(&dir).await; - - // `__run__*` is no longer reserved — creating it now succeeds. - db.branch_create("__run__formerly_reserved") - .await - .expect("__run__ prefix is a normal branch name post-MR-770"); - - // The schema-apply lock branch is still rejected on public branch APIs. - let create_err = db.branch_create("__schema_apply_lock__").await.unwrap_err(); - let OmniError::Manifest(err) = create_err else { - panic!("expected Manifest error"); - }; - assert!( - err.message.contains("internal system ref"), - "unexpected error: {}", - err.message - ); - - let merge_err = db - .branch_merge("__schema_apply_lock__", "main") - .await - .unwrap_err(); - let OmniError::Manifest(err) = merge_err else { - panic!("expected Manifest error"); - }; - assert!( - err.message.contains("internal system refs"), - "unexpected error: {}", - err.message - ); + for name in ["__run__formerly_reserved", "__schema_apply_lock__"] { + db.branch_create(name).await.unwrap(); + db.branch_merge(name, "main").await.unwrap(); + db.branch_delete(name).await.unwrap(); + } } // ─── Staged-write rewire — additional contract tests ─────────────────────── diff --git a/docs/dev/invariants.md b/docs/dev/invariants.md index e13398e5a..a44864fd1 100644 --- a/docs/dev/invariants.md +++ b/docs/dev/invariants.md @@ -24,14 +24,16 @@ five more changes like this one?** 2. **There is one graph-content publication door.** A graph change becomes authoritative through one `__manifest` publication containing every - visible table pointer and graph-lineage update. A writer never moves a - table's linear HEAD: its effects are detached commits the `__manifest` + visible table pointer, accepted schema contract and graph-lineage update. + A write to an existing registered table never moves its linear HEAD: its effects are detached commits the `__manifest` pin `(published_dataset_version, staged_version, transaction_uuid)` names, every pin is a detached version that opens, and the pin is the table's version for its whole life. A graph table's linear HEAD stays at its creation version, recorded per registration as - `omnigraph.last_linear_version`, and only `repair` reads it. Per-table - publication is never graph publication. + `omnigraph.last_linear_version`; `repair` reads it for drift. Added-type + admission may also inspect an unregistered deterministic path, but accepts + only its qualified original empty version 1, never an advanced HEAD. + Per-table publication is never graph publication. 3. **Every operation uses one coherent accepted view.** A read holds one immutable snapshot for its lifetime. A writer captures schema, catalog, @@ -49,10 +51,9 @@ five more changes like this one?** staged file nothing references, which a retry ignores. `cleanup`'s collector reclaims such a version only when the publication authority its transaction properties record is provably gone; age does not prove - abandonment. A published table effect is complete at publication and - nothing finishes it. The one published effect with work left, a staged - schema contract, carries its publishing commit in the manifest itself. - There is no side record to classify. The collector holds two properties + abandonment. Published table effects and the accepted schema contract + are complete in one manifest publication; no contract-file installation + remains. There is no side record to classify. The collector holds two properties on every run: safety, that every pin of every retained `__manifest` version on every live branch still opens and every path its manifest lists still exists after the run; and progress, that a table version no @@ -141,20 +142,12 @@ different: - Azure writes require the admission wrapper and remain a qualification preview pending the adversarial live-Azure matrix. The narrower managed-identity smoke proof is complete. -- The open-time decisions about a schema apply another process may still be - running retain a one-mutation-process boundary: a read-write open discards - an unpublished staged schema contract and reclaims a schema-apply sentinel, - fenced only by process-local gates. A live apply in another process loses - its staging and then installs its contract from memory, so within that - boundary the graph does not tear, and its sentinel no longer excludes other - writers. The residual the boundary carries: if a concurrent read-write open - discards a live apply's staging and that apply then lands its manifest commit - but dies before installing from memory, the published outcome is paired with - the old contract with no staging left to install, so a same-identity - contract change can serve stale until the next apply. Add/drop and - identity-changing shapes still fail loudly; only a same-identity rewrite is - silent, and only outside the single-process boundary. Table effects carry no - such boundary: a detached commit needs no takeover. +- Schema apply retains its main-only, single-live-branch restriction and + process-local gate ordering. Its contract and table references publish + together; the removed file-installation protocol supplies no distributed + lock. Branch creation/deletion and cleanup retain their existing + single-writer-process control boundary. Explicit storage conversion + requires operator-enforced quiescence. - Physical index reconciliation is explicit; there is no background scheduler whose queue is a second authority. @@ -166,10 +159,9 @@ code, tests, guide, and—when irreversible—RFC. - Does one snapshot or authority token cover the whole operation? - Is graph visibility still one manifest publication? - Is every pre-publication durable effect unreachable until the manifest names - it, is a published table effect complete at its pin, and does a staged - contract carry what finishing it needs? -- Does the change read a table's linear HEAD anywhere but `repair`, or - rebuild a registration row without carrying + it, and are the table pins and accepted contract complete in that publication? +- Does it read linear HEAD outside `repair` or qualified original-version + admission for an unregistered added-type path, or rebuild a row without carrying `omnigraph.last_linear_version` forward? - Does the collector's mark set still cover every retained pin, and does its sweep still reach every unretained published version? diff --git a/docs/dev/lance.md b/docs/dev/lance.md index 5a7e9f050..057f1a336 100644 --- a/docs/dev/lance.md +++ b/docs/dev/lance.md @@ -204,8 +204,10 @@ Schema v8 defines native-ref retirement metadata; schema v9 adds the system-column namespace (RFC 0040); schema v10 lets a registration name a detached table commit (RFC 0067); schema v11 makes that commit the table's version for life and records `omnigraph.last_linear_version` on every -registration (RFC: Detached-only tables). Normal open serves v11 only; -qualified v6/v7/v8/v9/v10 graphs have explicit offline routes to v11, and the +registration (RFC: Detached-only tables). Schema v12 packs the catalog record; +schema v13 stores the schema contract row in each branch's `__manifest`. +Normal open serves v13 only; qualified v6/v7/v8/v9/v10/v11/v12 graphs have +explicit offline routes to v13, and the system-column respelling is a separate step on a served graph. The v7 → v8 handler changes only manifest configuration metadata and does not infer fork ownership or retire branches. Source v6/v7 graphs with reserved retirement metadata refuse; diff --git a/docs/dev/recovery.md b/docs/dev/recovery.md index 817095272..65dab3e1b 100644 --- a/docs/dev/recovery.md +++ b/docs/dev/recovery.md @@ -1,26 +1,24 @@ # Graph recovery **Audience:** engine and storage contributors -**Authority:** current crash-recovery model; the staged-contract pass in -`crates/omnigraph/src/db/schema_state.rs` is the exact authority for the one -published effect that still has work to finish - -A graph write has one durable step that matters: the `__manifest` publication. -Everything a writer does before it is invisible and needs no recovery; a -published table effect is complete at that step, and only a schema contract -has work left after it. There is no recovery -sidecar, classifier, roll-forward, rollback, `Restore` compensation or -recovery audit ([RFC 0067](../rfcs/0067-detached-table-commits.md)), and no -promotion of a table pin onto its linear history -([RFC: Detached-only tables](../rfcs/2026-09-21-detached-only-tables.md)). +**Authority:** current graph publication and explicit storage-upgrade recovery + +A graph write becomes visible at its branch's `__manifest` publication. The +published table references and accepted schema contract are complete together; +normal open never installs a contract from root files. No writer arms a +recovery sidecar, classifier, roll-forward, rollback, `Restore` compensation or +recovery audit ([RFC 0067](../rfcs/0067-detached-table-commits.md)). Table pins +are not promoted onto linear history +([Detached-only tables](../rfcs/2026-09-21-detached-only-tables.md)). Explicit +offline storage conversion retains its separate intent and activation protocol. ## What a crash can leave | Interrupted | What is on storage | Who finishes it | |---|---|---| -| Before the manifest commit | Detached Lance versions nothing references, possibly an unregistered added-type dataset, possibly a staged schema contract whose publishing commit is not in lineage | Nobody has to. The caller retries from scratch. `cleanup`'s collector reclaims the detached versions once the branch incarnation and graph head their transaction properties record can no longer be published against, the next schema apply reclaims the leftover dataset under its sentinel, and the next read-write open discards the contract staging | -| After the manifest commit | The pin `(published_dataset_version, staged_version, transaction_uuid)` in the branch's `__manifest`: the published rows live in the detached version, which is the table's version for its whole life | Nothing. Reads open `staged_version` | -| After a schema apply's or system-column upgrade's manifest commit, before the contract files are installed | A staged contract whose publishing commit is in lineage, and possibly the schema-apply sentinel | The same handle's next write entry (`settle_pending_schema_install`), any handle's `refresh`, or the next read-write open installs it; the open also reclaims the sentinel. A read-only open refuses until then | +| Before the manifest commit | Unreferenced detached Lance versions and possibly an unregistered original empty added-type dataset | The caller retries from a fresh capture. The collector reclaims detached versions only when their recorded publication authority is gone. Added-type retry preserves the existing path and qualifies its original Create as described in [writes.md](writes.md) | +| After the manifest commit | Complete table pins and the accepted contract row in the branch's `__manifest` | No durable installation remains. Reads open the published pins and contract; refresh rebuilds disposable memory | +| During explicit storage conversion | Main's pending intent, branch receipts and possibly partially deleted legacy contract files | Rerun the same upgrade-capable executable and target while the root remains offline; activation is last | ## Pins @@ -43,27 +41,41 @@ adopts it (`repair.rs`, `judge_against_last_linear_version`), and the collector deletes neither its manifest nor its files and lists it under `foreign_versions`. -## Staged schema contracts - -Schema apply and the system-column upgrade write `_schema.pg.staging`, -`_schema.ir.json.staging` and `__schema_state.json.staging` before their -manifest commit. The state file is written last and carries -`publication: { graph_commit_id, parent_commit_id }`. That commit in main's -lineage means the manifest already carries the new table set, so the contract -must follow; its absence means the manifest never moved, so the staging is -garbage. The writer installs the live files from memory right after its -commit, so another process discarding the staging cannot tear the graph. - -- A read-write open installs a published staging and discards an unpublished - or incomplete one (the same one-mutation-process boundary as every other - open-time decision: a live apply in another process loses its staging and - then installs from memory). -- `refresh` and the write-entry pass only install; anything else may belong - to a live apply. -- A read-only open writes nothing: it refuses a published-but-uninstalled - contract and serves an unpublished staging as absent. -- Complete staging files without a publication marker come from a build that - predates this protocol or from manual edits, and are refused for inspection. +## Schema contract publication + +Schema apply and the system-column upgrade publish the replacement +`schema_contract` row with their table references and graph-lineage change in +one main-branch manifest commit. Open and refresh validate that row; neither +uses root schema files or a schema-apply sentinel. A proven publication is +complete even if a later error interrupts adoption of the in-memory view. +Lost acknowledgement remains a distinct outcome checked against the exact +attempted publication, not evidence that the write lost. + +Historical queries keep the accepted live contract and rebind its aliases by +stable identity to the selected historical table image. Retained contract rows +do not introduce historical schema-language semantics. See +[Schema contract in the manifest](../rfcs/2026-09-30-schema-contract-in-manifest.md). + +## Explicit storage conversion + +`schema-contract-v11-v12-to-v13` validates the actual source layout of every +live branch and reads the legacy contract only through the upgrade reader. +Before effects it refuses legacy staging, a residual schema-apply sentinel +and unresolved recovery sidecars. Protocol 5's intent pins the semantic +contract identity and SHA-256 digests of the exact source and IR text. +Each strict branch overwrite includes the contract row, target stamp and +receipt without creating a graph commit. Foreign movement or contract drift +fails closed. + +All branches and receipts must validate before deleting `_schema.pg`, +`_schema.ir.json` and `__schema_state.json`. Main retains its pending marker +through cleanup. Once every branch has converted, retry takes the contract +from main's validated row and tolerates any subset of those files being absent. +Only after cleanup does main activate. Ordinary opens remain refused while the +intent is pending; never remove it by hand. A current-v13 no-op uses the row +and ignores orphan legacy schema files and staging. The former sentinel name +is available to user branches in v13. The separate recovery-sidecar admission +rule below still applies. ## Sidecars from older builds @@ -78,51 +90,48 @@ its own publication. ## Ordering and visibility -The contract pass uses the same gate order as writers (schema, then branch, -then sorted tables). It never treats a warm coordinator or cache as current -authority, and an installed contract invalidates derived handles -before later operations continue. A retried write is a new attempt with a new -lineage commit; nothing is replayed on the caller's behalf. +Accepted-view captures never treat a warm coordinator or cache as current +authority. Writers retain the gate order of schema, branch, then sorted +tables, and revalidate the complete captured authority before publication. +Adopting a changed contract invalidates derived handles. A retried write is a +new attempt with a new lineage commit; nothing replays on the caller's behalf. ## Liveness -A failed operation never wedges its own live handle: once the fault source -stops, the same `Omnigraph` instance's next ordinary write succeeds without -reopening. Failed attempts leave no handle-local poison: a -published-but-uninstalled contract is installed -by `settle_pending_schema_install` at the next write entry, and a -schema-apply sentinel this handle failed to release is retried there too -(`note_failed_sentinel_release`). This generalizes the retired -`RecoveryRequired`-specific check: the wedge class it watched is gone, but -the contract it enforced holds for every failure kind. Owners: the -failure-window matrix's default same-handle actor, the `live_handle_*` -liveness tests in `failpoints.rs` (persistent faults, persistent lost -acknowledgements, and the write-family seam sweep), and DST's -`Scenario::keep_handle` mode, which runs an entire fault storm on one -never-reopened handle (`dst_fault_storm_on_one_live_handle_keeps_writing`). +A failed operation must not wedge its own live handle: once the fault source +stops, the same `Omnigraph` instance can write again without reopening. +Failure-window tests cover both unchanged graphs before publication and +complete published graphs afterward. Their owners are the matrix's default +same-handle actor, the `live_handle_*` tests in `failpoints.rs` and DST's +`Scenario::keep_handle` mode +(`dst_fault_storm_on_one_live_handle_keeps_writing`). No contract-file install +or sentinel-release retry is part of write entry. ## Initialization ownership -Fresh-graph initialization uses a separate root-scoped +Fresh-graph initialization uses the separate root-scoped `__init_claim.json`; it is not a recovery-v9 sidecar. Strict and `force` init both acquire it with create-if-absent and repeat target preflight while holding -the claim. `force` may replace orphan schema artifacts only when no graph -manifest exists; it never rebinds an existing graph or purges data datasets. +the claim. The genesis manifest Create includes the contract row. Both modes +ignore and preserve orphan legacy schema files and staging. `force` changes +the existing-manifest conflict behavior; it does not replace a committed graph +or purge data datasets. -A failure proven to precede physical initialization may clean up schema files -owned by that claim. Once a Lance dataset Create may have started, the result is -acknowledgement-unknown and OmniGraph probes the exact attempt-local genesis: +Once a Lance dataset Create may have started, initialization probes the exact +attempt-local genesis before deciding the outcome: - an exact committed genesis resumes final validation; - a later validation failure returns `InitializationCommitted` and preserves the committed graph; -- an unavailable or mismatched proof returns `InitializationIndeterminate` and - preserves the schema artifacts and claim. +- an unavailable or mismatched proof returns `InitializationIndeterminate` + and preserves initialization artifacts and the claim. Do not retry initialization or remove an indeterminate claim until every initializer for that root is quiescent and the root has been inspected. The -claim prevents a concurrent force attempt from overwriting another attempt's -schema contract or racing delayed cleanup. +claim coordinates competing initializers even though schema files are no +longer installed. Read-write open of a local root still performs its temporary +create-if-absent capability probe; removing contract installation does not +make that entire entrypoint read-only. ## Graph branch controls @@ -160,11 +169,13 @@ The v8 storage fence keeps older binaries from exposing retired branches. - `crates/omnigraph/tests/detached_commit_matrix.rs` owns the writer × window × fault × recovery-actor matrix under one oracle. - `crates/omnigraph/tests/schema_apply.rs` and `system_column_upgrade.rs` own - the staged-contract outcomes. -- `crates/omnigraph/tests/recovery.rs` owns what is left of open-time - recovery: a clean open creates nothing, a legacy sidecar refuses a - read-write open and not a read-only one, and a read-only open never touches - schema staging. + atomic contract publication, pre-publication refusal and complete published + outcomes, including later errors. +- `crates/omnigraph/src/db/upgrade/tests.rs` owns intent and receipt recovery, + contract-byte drift refusal and every legacy-file deletion boundary. +- `crates/omnigraph/tests/recovery.rs` owns manifest-only contract admission, + ignored orphan schema artifacts, read-only opens without writes, and legacy + sidecars refusing read-write but not read-only open. - The initialization cells in `failpoints.rs` own exact-genesis recovery, committed-versus-indeterminate outcomes, and claim retention. - `crates/omnigraph/tests/lance_surface_guards.rs` owns the Lance @@ -178,4 +189,6 @@ The design rationale is [RFC 0067](../rfcs/0067-detached-table-commits.md), which supersedes the sidecar protocol of [RFC 0022](../rfcs/0022-unified-write-path.md), and [RFC: Detached-only tables](../rfcs/2026-09-21-detached-only-tables.md), -which removes RFC 0067's promotion. +which removes RFC 0067's promotion. The +[schema-contract RFC](../rfcs/2026-09-30-schema-contract-in-manifest.md) +replaces its separate contract installation. diff --git a/docs/dev/testing.md b/docs/dev/testing.md index 3da165469..eb2696050 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -58,7 +58,7 @@ The engine integration suite is grouped by behavior, not implementation module: | Maintenance and substrate fences | `maintenance.rs`, `lance_surface_guards.rs`, `lance_version_columns.rs`, `forbidden_apis.rs` | | Export and lineage | `export.rs`, `lineage_projection.rs` | | Legacy-vintage graphs (`id`/`src`/`dst` spellings, born at the current stamp) | `legacy_columns.rs` — load, query, export round trip, evolution; needs `--features failpoints` | -| System-column upgrade (RFC 0040 step 3: respelling in place on a supported standalone graph; vintage is independent of the storage stamp) | `system_column_upgrade.rs` — check and execute, preflight refusals, every window before the manifest commit leaving no residue, a post-commit failure finished by the next read-write open or the same handle's next write, the control-object cost; needs `--features failpoints`. Route composition and the default target: `upgrade/tests.rs` | +| System-column upgrade (RFC 0040 step 3: respelling in place on a supported standalone graph; vintage is independent of the storage stamp) | `system_column_upgrade.rs`: check and execute, preflight refusals, every window before the manifest commit leaving no residue, a complete contract and table state after a post-commit failure, same-handle retry, the control-object cost; needs `--features failpoints`. Route composition and the default target: `upgrade/tests.rs` | | Cost and benchmark contracts | `write_cost.rs`, `write_cost_s3.rs`, `warm_read_cost.rs`, `branch_control_cost.rs`, `merge_cost.rs`, `changes_cost.rs`, the checkpoint/head lookup instruments, the ignored `manifest_history_curve.rs` instrument (requests, bytes and retained `__manifest` bytes as history grows), and `benchmark_scenario_contract.rs` | Use `tests/helpers/mod.rs` for the standard graph, snapshots, row reads, Blob selectors, and bounded Blob collection. Recovery helpers belong in `tests/helpers/recovery.rs`; object-store counters belong in `tests/helpers/cost.rs`. @@ -72,7 +72,7 @@ Crash tests must cover the writer and the user-visible reopening behavior: - `tests/failpoints.rs` owns crash windows around durable effects: after a detached effect and before publication, where the graph is unchanged, and after publication, where the pin is complete; - `tests/detached_commit_matrix.rs` owns the writer × window × fault × recovery-actor matrix under one oracle; -- `tests/recovery.rs` owns what is left of open-time recovery: a clean open creates nothing, a sidecar from an older build refuses a read-write open and not a read-only one, and a read-only open never touches schema staging; +- `tests/recovery.rs` owns manifest-only contract admission, ignored orphan schema artifacts and read-only opens without writes; a sidecar from an older build refuses read-write but not read-only open. Read-write open of a local root retains its temporary create-if-absent capability probe; - `tests/lance_surface_guards.rs` owns the Lance detached-commit facts the pin and the collector depend on; - the writer's normal integration owner proves pre-effect failures leave no residue. @@ -134,20 +134,44 @@ OMNIGRAPH_V5_BIN=/target/debug/omnigraph cargo test --locked -p omnigraph-c The older seams work the same way with released binaries: `OMNIGRAPH_OLD_BIN` (0.7.2) and `OMNIGRAPH_PREVIOUS_BIN` (0.8.1). `OMNIGRAPH_V6_BIN` (the 0.10.0 release) owns the v6↔v10 fence. RFC 0062 introduced v7's registration clock, RFC 0042's native-ref retirement metadata requires v8, RFC 0040's system columns stamped new graphs v9, and RFC 0067's detached table commits stamp every graph v10. The v0.9 journey is a different case, a fully exercised v6 graph — branches, edges, vectors, full-text and blobs — that the current binary refuses and that is rebuilt from a 0.9 export; `Test Workspace` runs both on every pull request that changes engine input, with the releases it installs. The separate `Storage Upgrade Compatibility` CI job requires genuine v0.9 and -v0.10 local standalone journeys: the v6 → v7 → v8 route with `--to-format 8` -first, then the default route to v10 on the same branched fixture, which the -journey asserts keeps every branch and every table byte. It fails -missing predecessor binaries, missing cases and skipped required cases. Engine -storage-upgrade tests own direct v7 → v8 conversion, exact pending v6 → v7 -recovery before composition, explicit target 7, deferred check reporting, -v8 no-op admission with retained retired refs, the v8 and v9 → v10 stamp -step (`storage_upgrade_default_route_takes_a_legacy_v8_graph_to_v10`, -`storage_upgrade_default_route_takes_a_v9_graph_to_v10`) and the synthetic -v6/v7 → v10 composition (`storage_upgrade_default_route_takes_a_synthetic_v6_graph_to_v10`; -no genuine predecessor binary executes that step yet). Keep the normal-open -format fences: explicit conversion does not grant serving support for -v6/v7/v8/v9. -See the [support matrix](versioning.md#storage-upgrade-support-matrix). +v0.10 local standalone journeys. Each first takes v6 → v7 → v8 with +`--to-format 8`, then the default v8 → v10 → v11 → v13 route on the same +branched fixture. The owners compare exports, table payload files, branch +ancestry, commit IDs, numeric snapshots and Blob bytes, and verify that the +accepted source/IR text moves into each branch's contract row before the root +files disappear. Missing predecessors, missing cases, empty runs and skipped +required cases fail the job; a local environment-gated skip is not evidence. + +For local storage-upgrade journeys, put verified released executables at +`target/storage-upgrade-binaries/v0.9.0/omnigraph` and +`target/storage-upgrade-binaries/v0.10.0/omnigraph` in the workspace, then run: + +```bash +cargo test --workspace --locked --test crossversion_upgrade --features omnigraph-engine/failpoints,omnigraph-cluster/failpoints storage_upgrade -- --test-threads=1 +``` + +The existing CI environment inputs take precedence. Without those inputs, +the presence of `target/storage-upgrade-binaries` requires both local fixtures; +a missing file or unexpected exact release version fails. The harness logs +the selected executable, and `check-storage-upgrade-ci.py --check-log +crossversion ` rejects a skipped required journey. + +`db/upgrade/tests.rs` owns route composition and intermediate targets, deferred +check reporting, intent recovery, retired ancestry, v10 pin promotion and +cleanup, and the v13 conversion protocol. Its fixtures must include actual +flat-v11 and packed-v12 sources without contract rows, both mixed branch +directions, zero-effect checks, exact contract identity and text digests, +foreign movement and every interruption boundary, including each legacy-file +delete. The upgrade-only v10 constructor must not admit ordinary opens while +its handle exists, even for an unexpectedly row-bearing source. Current-v13 +no-op cases preserve ignored orphan files/staging and allow the former +sentinel name as a user branch. Keep ordinary-open refusal for all pre-v13 +stamps; an explicit intermediate stop does not grant serving support. +`schema_apply.rs`, `system_column_upgrade.rs` and historical-read owners cover +atomic contract publication, first-touch retry and current-contract historical +reads. The catalog tests own row uniqueness, projection and validation; +`lance_surface_guards.rs` owns the filtered packed-record scan. See the +[support matrix](versioning.md#storage-upgrade-support-matrix). The system tests start workspace binaries on ephemeral localhost ports. Set `OMNIGRAPH_SKIP_SYSTEM_E2E=1` only in constrained local sandboxes; CI's configured owners must not skip. diff --git a/docs/dev/versioning.md b/docs/dev/versioning.md index ba310b36f..66885c37d 100644 --- a/docs/dev/versioning.md +++ b/docs/dev/versioning.md @@ -15,23 +15,21 @@ version axes. Never derive one axis from another. ## Current storage contract -The current binary serves **internal manifest schema v11 and v12**: -`MIN_SUPPORTED_INTERNAL_SCHEMA_VERSION` is 11 and -`INTERNAL_MANIFEST_SCHEMA_VERSION` is 12. v12 changes only how a `__manifest` -row is stored (one packed `record` column, see the v12 bullet below); a v11 -graph opens as it is and each Lance branch of its `__manifest` converts on its -next publish, with no command. v10 is the [RFC 0067](../rfcs/0067-detached-table-commits.md) -stamp: a table registration may name a detached Lance version whose linear -target is published before it exists, which an older binary would open as -reclaimed history, so the stamp refuses it before any open. Both system -column vintages of [RFC 0040](../rfcs/0040-system-column-namespace.md), -`id`/`src`/`dst` and `__id`/`__src`/`__dst`, are supported within this range: -the vintage is read from the schema IR's feature set, never from the stamp, and -`omnigraph schema upgrade-system-columns` converts it on a supported standalone -graph without a separate vintage-specific stamp. Its publication follows the -ordinary format conversion rule, so a v11 main becomes v12. A v8, v9 or v10 graph -needs one explicit `omnigraph upgrade`, a route that keeps its branches and its spellings, -before this binary serves it. +The current binary serves **internal manifest schema v13**. Both +`MIN_SUPPORTED_INTERNAL_SCHEMA_VERSION` and +`INTERNAL_MANIFEST_SCHEMA_VERSION` are 13. Each live branch's `__manifest` +carries one `schema_contract` row containing the accepted schema source, +compiled IR and identity. Fresh initialization writes it in the genesis +Create; schema apply and the system-column upgrade replace it in the same +publication as their table references. Normal serving has no root-file +fallback. See [Schema contract in the manifest](../rfcs/2026-09-30-schema-contract-in-manifest.md). + +Both system-column vintages of [RFC 0040](../rfcs/0040-system-column-namespace.md), +`id`/`src`/`dst` and `__id`/`__src`/`__dst`, remain supported. The schema IR's +feature set determines the vintage, never the storage stamp. +`omnigraph schema upgrade-system-columns` converts the spellings on a +supported standalone graph without changing its storage stamp. Admitted +older formats require explicit storage conversion before normal open. - v4 was the last released pre-identity format, used by OmniGraph 0.8.x. - v5 was an unreleased development format that introduced SchemaIR v2, @@ -58,8 +56,8 @@ before this binary serves it. `omnigraph schema upgrade-system-columns` converted it in place (RFC 0040 Rollout step 3: stamp advance first, one rename-only commit per table, schema promotion last). Since v10 the vintage has no separate stamp and the - upgrade stages detached renames like schema apply (RFC 0067); its publication - follows the ordinary format conversion rule above. + upgrade stages detached renames like schema apply (RFC 0067). At v13 it + publishes the renamed table references and replacement contract row together. - v10 preserves v9's layout and lets a table registration carry `omnigraph.staged_version` and `omnigraph.transaction_uuid` (RFC 0067): the detached Lance version a pin was staged as and the transaction promotion @@ -75,26 +73,35 @@ before this binary serves it. `detached-only-v10-to-v11`, which promotes every pending pin once, reaps the proven copies, records the key on every current registration of every live branch, and restamps. -- v12 keeps v11's logical rows (`manifest_schema()`, ten fields: the unused - `base_objects` list is gone) and stores them as three columns: `object_id`, +- v12 kept the ten logical fields present at v11 (the unused `base_objects` + list was removed) and stored them as three columns: `object_id`, `object_type` and `record`, a Lance packed struct (`lance-encoding:packed`) holding the eight remaining fields row-major plus a `present` bitmask, one bit per field that is null in the logical row (Lance 11 refuses a null value inside a packed struct child, so every child is declared non-null and a null is stored as `""` or `0` with its bit set). A scan reads one column's pages for the record instead of one per field, and always projects `record` - whole (Lance 11 cannot project a fixed-width child of a packed struct on - its own). Conversion is the branch's next publish: the copy-on-write - overwrite writes the packed shape and the v12 stamp in one commit; an idle - branch stays v11 and reads as v11; earlier versions keep their shape for - time travel. A graph holds v11 and v12 across its branches until every - branch has published, and every reader, `omnigraph upgrade` and - `omnigraph schema upgrade-system-columns` accept that state. `omnigraph - upgrade` has no v12 step and `--to-format 12` is an unsupported target; an - offline route that converts every branch at once is a separate change (see - §Changing an axis). A Lance directory-namespace client, which reads - `__manifest` by its own catalog column names, fails at `location` on a - stamp-12 manifest; it found no OmniGraph table at stamp 11 either. + whole. Lance 11's default filtered-scan materialization can split packed + children between early and late reads; the contract-row scan uses + `MaterializationStyle::AllEarly` to keep the projected record together. + Isolated packed-child projection remains unsupported. The original v12 + transition converted a v11 branch on its next publish, so predecessor roots + can mix v11 and v12. These are now conversion inputs and retained-history + layouts, not served formats. `--to-format 12` remains unsupported. A Lance + directory-namespace client, which reads `__manifest` by its own catalog + column names, fails at `location` on a stamp-12 manifest; it found no + OmniGraph table at stamp 11 either. +- v13 preserves the packed record and adds nullable `LargeUtf8` columns + `schema_source` and `schema_ir`. Only the `schema_contract` row carries + these texts; its `metadata` records `schema_ir_hash`, + `schema_identity_version` and `schema_identity_domain`. Serving scans capture + the texts, identity and table state together from the exact branch and + version. Contract acceptance validates the IR, source shape and captured + table identities; catalog reuse requires the same validated image or row + content. Historical and migration folds may read the identity alone. + Historical queries keep using the accepted + live contract, with aliases rebound by stable identity to the retained + table image; an older row does not add historical schema-language semantics. - the unreleased v7–v19 stamps of the rejected MemWAL experiment never shipped and are not supported migration inputs. Reuse of a numeric stamp by another design (RFC 0062, RFC 0042 or RFC 0040) does not make an experimental graph @@ -102,22 +109,34 @@ before this binary serves it. rebuild at a fresh root. Normal open refuses lower and higher stamps before recovery or table decoding; -neither served stamp is rewritten on open. -`omnigraph upgrade` defaults to v11: qualified standalone v6 graphs run the -registered v6 → v7 → v8 → v10 → v11 route, qualified v7 graphs run -v7 → v8 → v10 → v11, v8 and v9 graphs run v10 → v11, and v10 graphs run the -v11 step alone. The v10 step (`detached-pins-v8-v9-to-v10`) restamps main and -every live branch under one intent and touches nothing else; retired -ancestors keep their stamp. The v11 step (`detached-only-v10-to-v11`) is -engine-backed: it needs every writer stopped and runs on standalone roots -only. `--to-format 7`, `--to-format 8` and `--to-format 10` retain those -intermediate targets for a compatible older executable; the current binary -refuses normal open of each result, and v9 is not a target. The system-column respelling is a separate operation on -a served graph, `omnigraph schema upgrade-system-columns`. Original retained -snapshots remain unchanged; historical v6 registrations use an explicit -legacy decoder after main-root admission. A pending upgrade marker refuses -normal opens until every branch validates and main activation completes. -See [RFC 0064](../rfcs/0064-explicit-storage-upgrades.md) for the offline protocol. +it never migrates a graph. `omnigraph upgrade` defaults to v13 and composes +`v6 → v7 → v8 → v10 → v11 → v13`; qualified v7 inputs start at v7, +v8 and v9 inputs enter the v10 step, and v10 inputs enter the v11 step. +The v10 step (`detached-pins-v8-v9-to-v10`) restamps main and every live +branch under one intent; retired ancestors keep their stamp. The v11 step +(`detached-only-v10-to-v11`) promotes every pending v10 pin once, reaps proven +copies and records the last linear version. Both require the existing offline, +standalone admission. Its upgrade-only v10 handle receives a validated legacy +contract without relaxing ordinary-open admission. + +The final `schema-contract-v11-v12-to-v13` step accepts actual flat v11 and +packed v12 sources, including either mixture across main and named branches. +Protocol 5's intent pins the semantic contract identity and raw-text SHA-256 +digests of the source and IR. Each branch overwrite preserves its logical rows +and writes the contract, target stamp and receipt together; it introduces no +graph commit. All branches must validate before the three legacy contract +files are deleted. Main retains its pending marker until cleanup finishes, +then activates last. After all branches convert, retries read the contract +from main's row and tolerate any subset of those files already being absent. + +Explicit targets 7, 8, 10 and 11 remain intermediate stops for compatible +executables; the current binary refuses normal open of their results. Neither +9 nor 12 is a target. Check mode has no effects and reports downstream +preflights that require intermediate output as deferred. Original retained +snapshots remain unchanged; historical v6 registrations use the explicit +legacy decoder after main-root admission. See +[RFC 0064](../rfcs/0064-explicit-storage-upgrades.md) for the offline protocol +and [the upgrade guide](../user/operations/upgrade.md) for execution and retry. ## Recovery sidecars @@ -169,14 +188,14 @@ see the [admission limits](../user/operations/upgrade.md). | Source executable / format | Normal open | Default explicit route | Required coverage owner | |---|---|---|---| -| 0.9.0 / v6 | Refused | v6 → v7 → v8 → v10 → v11 (`--to-format 8` stops at v8; branches are kept) | `crossversion_upgrade.rs::genuine_v09_explicit_storage_upgrade_preserves_history` (v8 stop, then the default route on its branched fixture); `upgrade/tests.rs::storage_upgrade_default_route_takes_a_synthetic_v6_graph_to_v11` | -| 0.10.0 / v6 | Refused | v6 → v7 → v8 → v10 → v11 | `crossversion_upgrade.rs::genuine_v010_explicit_storage_upgrade_preserves_history` | -| Qualified development / v7 | Refused | v7 → v8 → v10 → v11 | Engine storage-upgrade tests: metadata-only conversion, history and retry (pinned to `--to-format 8`), plus the synthetic v6 default-route test above | -| Legacy vintage / v8 | Refused | v8 → v10 → v11 (`detached-pins-v8-v9-to-v10`, then `detached-only-v10-to-v11`; `--to-format 8` is an already-current no-op) | `upgrade/tests.rs::storage_upgrade_default_route_takes_a_legacy_v8_graph_to_v11`, `storage_upgrade_current_v8_preserves_retired_ancestry_and_recreated_name`; `tests/system_column_upgrade.rs` (the respelling on a served graph, refusals, crash points) | -| 0.11.x / v9 | Refused | v9 → v10 → v11 | `upgrade/tests.rs::storage_upgrade_default_route_takes_a_v9_graph_to_v11` | -| 0.11.x (detached table commits) / v10 | Refused | v10 → v11 (`detached-only-v10-to-v11`) | `upgrade/tests.rs::storage_upgrade_default_route_takes_a_v10_graph_to_v11` | -| Current / v11 | Accepted; converts to v12 on each branch's next publish | Already current (also while the graph's branches mix v11 and v12); a lower target is refused; `--to-format 12` is an unsupported target, no route writes v12 | `omnigraph-catalog` `stamp_11_manifest_converts_on_its_next_publish` (no released binary wrote v11, so the synthetic conversion test is the predecessor evidence); `upgrade/tests.rs::storage_upgrade_accepts_a_graph_converting_branch_by_branch`; CLI `crossversion_upgrade.rs::genuine_v010_explicit_storage_upgrade_preserves_history` (a genuine 0.10.x graph upgraded to v11 converts branch by branch under the new binary's mutate and merge) | -| Current / v12 | Accepted | Already current; a lower target is refused | `upgrade/tests.rs::storage_upgrade_current_vintage_is_already_current_without_a_route`; stamp tests in `migrations.rs` | +| 0.9.0 / v6 | Refused | v6 → v7 → v8 → v10 → v11 → v13 (`--to-format 8` retains the intermediate stop) | `crossversion_upgrade.rs::genuine_v09_explicit_storage_upgrade_preserves_history`; default-route owners in `upgrade/tests.rs` | +| 0.10.0 / v6 | Refused | v6 → v7 → v8 → v10 → v11 → v13 | `crossversion_upgrade.rs::genuine_v010_explicit_storage_upgrade_preserves_history` | +| Qualified development / v7 | Refused | v7 → v8 → v10 → v11 → v13 | `upgrade/tests.rs`: metadata conversion, history, retry and route composition | +| Legacy vintage / v8 | Refused | v8 → v10 → v11 → v13 (`--to-format 8` is an already-current no-op before conversion) | `upgrade/tests.rs`: default route and retained retired ancestry; `system_column_upgrade.rs`: subsequent respelling on the served graph | +| 0.11.x / v9 | Refused | v9 → v10 → v11 → v13 | Default-route owners in `upgrade/tests.rs` | +| 0.11.x (detached table commits) / v10 | Refused | v10 → v11 → v13 | `upgrade/tests.rs`: pending-pin promotion, reap interruption, last-linear-version preservation and cleanup | +| Predecessor / v11, v12, or mixed branches | Refused | v11/v12 → v13 (`schema-contract-v11-v12-to-v13`) | `upgrade/tests.rs`: actual flat/packed layouts, both mixed directions, exact contract bytes, history, refusal and cleanup retry | +| Current / v13 | Accepted | Already current for default or target 13; lower registered targets refuse | Current-format no-op owners in `upgrade/tests.rs` and CLI `crossversion_upgrade.rs`; stamp tests in `migrations.rs` | | Older, future or unqualified experimental format | Refused | No route; source-compatible export/rebuild | Existing format fences and engine refusal tests | Source v6/v7 admission rejects any reserved native-ref retirement metadata. @@ -257,10 +276,10 @@ change before the release that ships the stamp; for a stamp no released binary wrote, the synthetic conversion test is the predecessor evidence and the matrix row says so. Steps 3 and 5 apply unchanged. -### Pins and staged contracts +### Pins and schema contracts -1. A change to what a pin or a staged schema contract records is a manifest - or contract format change: it takes a stamp or contract version and the +1. A change to what a pin or a schema-contract row records is a manifest + format change: it takes a stamp or contract version and the graph-storage checklist above. 2. A new detached transaction kind needs its twin-replay surface guard first. 3. Never infer missing lifetime identity from aliases. diff --git a/docs/dev/writes.md b/docs/dev/writes.md index b0c30ccac..4df9475c2 100644 --- a/docs/dev/writes.md +++ b/docs/dev/writes.md @@ -25,7 +25,7 @@ acquire shared schema permit → branch → sorted-table gates, recheck the comp ↓ commit each participant as a detached version of its pin ↓ -publish every pin (published_dataset_version, staged_version, transaction_uuid) + lineage in one __manifest CAS +publish every pin + accepted schema contract + lineage in one __manifest CAS ``` An error before the manifest CAS leaves the graph unchanged: the detached @@ -36,9 +36,9 @@ effect has post-publication work. A lost acknowledgement is resolved against the exact attempted manifest; unavailable readback stays indeterminate. No writer arms a recovery record, and no write replans around a partial state, because no partial state is ever visible. Schema apply and the system-column upgrade -additionally stage and install the schema contract around their CAS; only -they can report `RecoveryRequired`, naming a manifest commit that landed -while the contract installation did not. +include the replacement schema-contract row in the same publication as their +table references. A later `RecoveryRequired` outcome can identify a committed +publication; it does not leave durable contract files to install. ## Captured authority @@ -47,8 +47,9 @@ schema/catalog, target graph branch, optional graph head, native branch identity, table-incarnation identities, and expected table versions. Every planning and validation step uses that view. -Branch merge also uses the captured target for physical table opens and -publication. It never changes the `Omnigraph` handle's active branch while the +Ordinary non-bound branch writes retain their capture in the same one-entry +coordinator cache used by branch merge. Branch merge also uses the captured +target for physical table opens and publication. It never changes the `Omnigraph` handle's active branch while the merge runs. Publication reuses the active coordinator, or takes the cached non-active target coordinator, only when its branch identity, graph head, and manifest version match the captured transaction; otherwise it opens the target @@ -67,8 +68,8 @@ path. Captures share immutable lineage and the Lance session, copy current table state, and leave the handle's active branch unchanged. Create admission lists native ref names and reads bodies only for the source, -the target's incarnations and logical ancestors or descendants, and the -schema-apply sentinel. Physical path checks include retired ref names. The +the target's incarnations and logical ancestors or descendants. Physical path +checks include retired ref names. The exact target ref must be absent; an empty listing of its exact native tree allows creation without forced reclamation. A nonempty tree and clone-only recovery keep Lance's full dependency and tag checks. @@ -83,7 +84,7 @@ it does not protect an input from concurrent cleanup. Ref-name listing still grows with the physical ref inventory. Cold recreation of the same logical name reads its retained incarnations to validate retirement -and liveness; source, hierarchy and sentinel incarnations also remain relevant. +and liveness; source and hierarchy incarnations also remain relevant. Unrelated retired ref bodies add no reads to an ordinary fresh create. After a content publication, the publisher returns the projection (the @@ -94,17 +95,29 @@ cache has adopted the published lineage. A foreign advance, unsupported base, or failure before lineage adoption leaves the full-refresh fallback armed. Registration replacement, rename, tombstone, and same-version physical-owner handoff use the existing complete fold. This is disposable process memory; -`__manifest` remains the only durable graph authority. Publication still scans -history for collision, expected-version, and lineage validation. - -The capture itself (`open_write_txn`) takes no permit on the common path: -when a schema-apply sentinel stands and an apply in this process holds the -exclusive permit, it parks on the shared side until the apply releases, then -recaptures under the promoted contract. A sentinel under a free gate is -another process's apply and gets the typed refusal at once. Only `mutate` -and `load` reach the park: merge, index maintenance, optimize, cleanup and -repair call `ensure_schema_apply_idle` before their capture, so a standing -sentinel refuses them one call earlier. +`__manifest` remains the only durable graph authority. + +`GraphNamespacePublisher` retains at most one complete stored row batch from +its last acknowledged publication. Every attempt opens the target branch's +latest `__manifest` again. Reuse requires matching native manifest location, +version, naming scheme, complete Lance manifest and schema metadata; known +ETag or size mismatches reject reuse. Dictionary-bearing schemas bypass this +cache. Cached rows pass through the same decoder and collision, expected-version, +lineage and exact-head checks as scanned rows. The zero-retry version CAS and +lost-acknowledgement readback remain unchanged. Foreign movement, missing rows +or an image above the 8 MiB accounted Arrow-buffer/key budget use a full scan. +This budget bounds retained cache accounting, not total process memory. The +copy-on-write publication still rewrites the complete live row set. + +The capture (`open_write_txn`) obtains the accepted contract identity from +the selected manifest snapshot. It validates that snapshot's complete contract +row before reusing a catalog with the same identity. An unchanged captured +manifest image reuses its validation; newly observed images carry the contract +from their state scan, and acknowledged publications reuse their retained rows. +The cache is disposable; a missing or invalid row +cannot be repaired from root files. Captures and finalization retain the +process-local schema gate and complete-authority checks without a sentinel +branch or `ensure_schema_apply_idle` pass. Finalization acquires the root-shared gate order: @@ -130,7 +143,7 @@ physical-effect proofs: | Writer | Physical adapter | Publication | |---|---|---| | Mutation / Load | One exact staged keyed, overwrite, or delete transaction per touched table | One graph commit | -| SchemaApply | Exact existing-table rewrites plus new-type dataset creation (a linear `v1` create) and the complete schema/manifest delta | One main-branch graph commit | +| SchemaApply | Existing-table detached rewrites; original added-type Create or qualified reuse/detached replacement; complete table delta and replacement schema-contract row | One main-branch graph commit | | BranchMerge | Onto main: a pointer switch, main's registration taking the source's pin. Into a named branch: a chain of detached chunk commits (proven insertion chain or bounded ordered diff) published as one pin per table (`exec/merge.rs`) | One target-branch graph commit | | EnsureIndices / full-text rebuild | One detached `CreateIndex` batch per productive table, published as a pin like a mutation's effect (RFC 0067); ordinary ensure leaves untrainable vector work pending, explicit FTS rebuild replaces postings from rows | One graph publication when work lands | | Optimize | One detached compaction `Rewrite` per productive table, chained with a detached whole rebuild of each index whose coverage lags and a detached build of each declared-but-unbuilt index, published as pins (RFC 0067) | One main-branch graph commit with an exact CAS on the pins the batch was planned from | @@ -178,10 +191,12 @@ records `omnigraph.last_linear_version`, the highest linear version a pin ever reached (`1` for a table created under v11), and every writer that rebuilds the row copies it forward (`TableVersionMetadata`, `crates/omnigraph-core/src/metadata.rs`). A writer whose captured snapshot predates another -publication loses the CAS and reprepares (`ReadSetChanged`). No writer reads -the linear HEAD: a foreign linear commit above -`omnigraph.last_linear_version` is what `repair` reports as `foreign_drift` -(`repair.rs`, `judge_against_last_linear_version`), never a writer's concern. +publication loses the CAS and reprepares (`ReadSetChanged`). Existing-table +writes do not read linear HEAD; `repair` reports a foreign linear commit above +`omnigraph.last_linear_version` as `foreign_drift` +(`repair.rs`, `judge_against_last_linear_version`). Added-type admission can +inspect an unregistered deterministic path's HEAD only to prove it remains +its qualified original empty version 1; it never adopts an advanced HEAD. Detached manifests are reclaimed by the tracing collector `cleanup` runs (`db/omnigraph/collector.rs`; `optimize.rs`, `cleanup_detached_only`). Its @@ -213,23 +228,25 @@ including full-text search through the batch's certificate, serve from the staged version. Schema apply stages each existing-table rewrite as a detached Overwrite of -the table's pin and publishes it as a pin one past the published version. An -added type is a linear -version-one create at its identity path; that path is a deterministic -function of the accepted identity allocator, so an attempt that died after -creating the dataset left it exactly where the retry creates it, and the -retry reclaims the unregistered leftover under the schema sentinel before -creating. The schema contract is staged before the manifest commit with the -graph commit it publishes recorded in `__schema_state.json.staging`, and the -writer installs the live contract from memory after the commit. No sidecar is -armed: a failure before the commit leaves detached versions, a created -dataset and a staged contract that the next read-write open discards; a -failure after it leaves a published manifest whose contract installation the -same handle's next write, or the next read-write open, completes because the -recorded commit is in main's lineage. A read-only open refuses that state and serves -an unpublished staging as if it were absent. The open also reclaims a -sentinel left by a crashed apply, under the same one-mutation-process -boundary as every other open-time recovery decision. +the table's pin and publishes it one logical version past the published +version. An added type uses its identity-derived path, which a retry derives +again from the accepted allocator. Retry preserves that path and every +existing version. It can reuse an original empty, stable-row-ID version-one +Create with the desired physical schema. If that original Create has another +schema from an uncommitted attempt, it stages an empty detached Overwrite +based on version one. The replacement publishes at logical version 2 with its +detached version, transaction UUID and `last_linear_version = 1`, against +expected absent table version 0. Fresh or matching original Creates remain +logical version 1; nonempty or otherwise unqualified leftovers refuse. + +One main-branch manifest CAS publishes the table delta, graph-lineage change +and replacement contract row. A competing stale attempt cannot delete an +existing table path; the final CAS decides publication. A pre-publication +failure leaves the previous contract and graph visible. After publication, +both are complete and only the disposable in-memory view may need refresh. +There are no staged root contract files, schema-apply sentinel or durable +post-publication install. The system-column upgrade uses the same publication +boundary. See [Schema contract in the manifest](../rfcs/2026-09-30-schema-contract-in-manifest.md). Branch merge follows it too. A merge onto main is a pointer switch: main's registration takes the source's pin, and the merge stages no fenced insert, @@ -347,7 +364,7 @@ adoption does no source I/O. See [blob.md](blob.md). | Strict read-set movement | `ReadSetChanged` | | Exact duplicate on strict insert | `KeyConflict` | | Any writer fails before publication, after any detached effect | Typed error; no graph movement; the detached staging is unpublished and the collector reclaims it once its recorded authority is gone | -| Schema apply or the system-column upgrade fails after publication, before its contract is installed | `RecoveryRequired` naming the published commit; the next read-write open, `refresh`, or that handle's next write installs the staged contract | +| Schema apply or the system-column upgrade reports an error after proven publication | `RecoveryRequired` can name the committed outcome; its tables and contract are durable, and refresh/reopen rebuilds only the in-memory view | | A foreign linear commit lands above a table's `omnigraph.last_linear_version` | No read or write resolves it; `repair` reports the table as `foreign_drift` and never adopts the commit; the collector deletes neither its manifest nor its files and lists it under `foreign_versions` | | A sidecar from a build that predates detached commits is present | A read-write open and the storage upgrade refuse until that build has resolved it | diff --git a/docs/releases/v0.12.0.md b/docs/releases/v0.12.0.md index ff2d2bbfe..7d0933d7b 100644 --- a/docs/releases/v0.12.0.md +++ b/docs/releases/v0.12.0.md @@ -99,7 +99,7 @@ Unreleased. above it came from outside OmniGraph and are resolved by no read or write; `repair` classifies such a table `foreign_drift`, prints the last linear version and the HEAD, takes no action and exits 0. `--confirm` and - `--force --confirm` publish nothing on a v11 graph, and `cleanup` never + `--force --confirm` publish nothing on a v13 graph, and `cleanup` never deletes a foreign version. - Inserting into a node type with a camelCase `@unique` property no longer fails with `No field named `: the committed-uniqueness probe @@ -350,7 +350,7 @@ Unreleased. maintenance, optimize, cleanup, repair, branch delete, the system-column upgrade's `--check` and read-view captures take a shared permit on the schema gate; schema apply, the system-column upgrade, open, refresh, - settle, reload, `sync_branch` and branch create take it exclusive. A + reload, `sync_branch` and branch create take it exclusive. A writer on one branch no longer waits for a writer on another, and a read no longer waits behind another handle's publish hold (a read and a write on the same handle and branch still wait for each other while it @@ -396,9 +396,11 @@ Unreleased. non-idempotent retry could double-apply; only an outcome it genuinely cannot confirm surfaces as an in-doubt error. A failed operation never wedges its own live handle: once the fault source stops, the same handle's next write - succeeds without reopening. No writer reads a table's linear HEAD, so a - foreign linear commit blocks nothing; `omnigraph repair` reports it as - `foreign_drift`. `ensure_indices` and the explicit full-text rebuild + succeeds without reopening. Writes to registered tables do not read their + linear HEAD, so a foreign linear commit does not block those writes; + `omnigraph repair` reports it as `foreign_drift`. Schema apply checks an + unregistered added-type path before reuse and requires its original empty + version 1. `ensure_indices` and the explicit full-text rebuild follow: each index batch is a detached commit published as a pin, a rebuild interrupted after publication keeps serving search through the staged batch, and no `EnsureIndices` sidecar is written. Branch merge @@ -411,14 +413,15 @@ Unreleased. detached on the dataset its inherited registration names, two branches writing one table produce two detached chains in one dataset, and forks created by earlier builds stay valid and are reclaimed by `cleanup` once - unreferenced. Schema apply follows: an existing-table rewrite is a - detached commit published as a pin, an added type is a linear `v1` create - whose leftover from a crashed attempt the retry reclaims, the staged - schema contract records the graph commit that publishes it so the next - read-write open installs or discards it, and no `SchemaApply` sidecar is - written; a failure before the manifest commit returns the plain error, and - only a failure after it reports `RecoveryRequired`, naming the published - commit. Optimize follows: each table's compaction is one detached rewrite + unreferenced. Schema apply follows: existing-table rewrites are detached + commits published as pins. An added type's original empty `v1` Create can + be reused after a failed attempt; a different qualified original schema + uses a detached replacement, preserving the path and its versions. The + table delta and replacement schema-contract row publish together. No + `SchemaApply` sidecar or root-contract installation remains. A failure + before publication leaves the previous graph visible; a post-publication + error identifies a complete published outcome, including its contract. + Optimize follows: each table's compaction is one detached rewrite of its pin, a scalar or vector index whose coverage lags is rebuilt whole as a detached commit chained on it, the batch publishes once with an exact check on every pin it planned from, and no `Optimize` sidecar is written; @@ -459,11 +462,12 @@ Unreleased. writes one. A read-write open, and `omnigraph upgrade`, refuse a graph whose `__recovery/` directory holds a sidecar, naming it, until the 0.11.x binary that wrote it has opened the graph read-write and finished its own - recovery; a read-only open is unaffected. `RecoveryRequired` now has one - meaning: a schema apply or system-column upgrade published its manifest - commit but its schema files were not installed, and reopening read-write - installs them. A long-lived handle's refresh no longer runs any recovery - pass besides installing such a published schema contract. + recovery; a read-only open is unaffected. Refresh validates the published + schema-contract row and updates the in-memory view; it does not install + root schema files. A schema or system-column operation can still report + `RecoveryRequired` after a proven publication, but the named commit already + contains its tables and contract. Explicit storage-upgrade recovery retains + its own pending-intent protocol. Test seams `recovery.*`, `optimize.post_recovery_check_pre_main_gate`, `cleanup.post_recovery_check_pre_gates` and `system_column_upgrade.after_lock_reclaim` are gone, and @@ -471,46 +475,50 @@ Unreleased. The GQ logic test format loses its `--- known_failure` section, which existed only to pin known recovery-sidecar defects. -- **Storage format v12, v11 served; every graph from the 0.11.x line needs one explicit upgrade.** - Normal open now serves internal manifest schema v11 and v12. Fresh graphs - are born at v12, and a v11 graph converts to v12 branch by branch, each - branch on its next publish, with no command; v12 changes only how a - `__manifest` row is stored (one packed `record` column, fewer object-store - requests per publish and reopen), and its logical rows are v11's. A v8, v9 or v10 - graph from the 0.11.x line is refused until `omnigraph upgrade ` - (run `--check` first) takes it to v11: the metadata-only v10 stamp step - where the source needs it, then `detached-only-v10-to-v11`, which judges - every pin on every live branch and refuses before any write when a foreign - linear commit blocks one (finding `blocked_promotion`), promotes every - pending v10 pin once (the last promotion the engine runs), reaps the - copies that promotion proves, records `omnigraph.last_linear_version` on - every registration of every live branch in one publication per branch, - and restamps. It keeps every branch, every table byte and the graph's - system column spellings, runs on standalone roots with every writer - stopped, and refuses a graph with a pending recovery sidecar until the - 0.11.x binary that wrote it has resolved it; a cluster-managed graph has - no route yet. v11 exists because a registration's - `published_dataset_version` no longer names a Lance version: an older - binary would open it and misread the table as reclaimed history, so the - stamp refuses it before any open. Older binaries refuse v11 and v12 graphs. - v10 is - a conversion source, and `--to-format 10` an intermediate target for a - compatible older executable, for this release only; both are removed at - the next release boundary. `--to-format 9` is not a target: the - system-column respelling that v9 used to mark is `omnigraph schema - upgrade-system-columns` on a served graph and does not change the stamp; - a legacy-vintage graph is born at v12 too. That upgrade has schema apply's - shape: each table's rename commits detached and is published as a pin, - the staged contract names the publishing graph commit, and no recovery - sidecar is written; a run interrupted before its manifest commit leaves - the graph unchanged and is simply rerun, and one interrupted after it is - completed by the next read-write open. Test seam - `schema_apply.post_sidecar_pre_effect` is now +- **Storage format v13; older graphs need explicit conversion.** Normal open + serves v13 only and never migrates. Fresh graphs of either system-column + vintage start at v13. `omnigraph upgrade ` defaults to 13; run + `--check` first. Qualified v6 inputs compose through + v6 → v7 → v8 → v10 → v11 → v13; v7 starts at v7, v8/v9 at the v10 step, + and v10 at the v11 step. The v11 step retains its pending-pin checks: + `blocked_promotion` refuses a foreign linear target before effects; accepted + pins are promoted once, proven copies reaped and the last linear version + recorded on every live branch's registrations. + + The final `schema-contract-v11-v12-to-v13` step accepts actual flat v11, + packed v12 and either mixture across branches. Its intent pins the full + contract identity and exact source/IR text digests. Each branch receives + its contract row, stamp and receipt in one overwrite without a graph commit. + Branch ancestry, table identities, retained snapshots and table payload + files are preserved. The three legacy contract files are deleted after + every branch validates; main activates last. Retry can finish after any + subset of those deletes. A completed v13 no-op needs no legacy files and + preserves ignored orphan files/staging and legitimate user branches named + after the former sentinel. Normal serving uses only the row. + + This remains an offline standalone operation: stop all processes touching + the root and retain a verified whole-root backup. Legacy schema staging, + residual schema-apply sentinels and unresolved recovery sidecars refuse + conversion. Cluster-managed roots remain outside this route. Explicit + targets 7, 8, 10 and 11 are intermediate stops for compatible executables; + normal open with this binary refuses their results. Neither 9 nor 12 is a + target. v10 source/target support remains limited to this release boundary. + Rollback restores the complete pre-upgrade backup with the compatible + executable; post-upgrade writes are absent from that backup. Formats with + no route use source-compatible export and rebuild. + + `omnigraph schema upgrade-system-columns` separately respells a served + graph without changing v13. Its detached table renames and replacement + contract publish together. An interruption before publication leaves the + graph unchanged; afterward both contract and table references are complete. + Test seam `schema_apply.post_sidecar_pre_effect` is now `schema_apply.post_lock_pre_effect`, and `system_column_upgrade.after_stamp_advance` is gone. The - `V5 ↔ V9 Format Fence` CI job is now `V5 ↔ V10 Format Fence`. Downgrade - is an export with this binary followed by `init` and `load` with the older - one; history and branches are lost. + `V5 ↔ V9 Format Fence` CI job is now `V5 ↔ V10 Format Fence`. + Historical queries retain current-contract semantics; older contract rows + do not add a schema history API. Export with this binary followed by `init` + and `load` with an older one is a separate logical rebuild; it loses history + and branches. - **Engine v1 is retired from production; engine v2 answers every read.** The session setting `engine` has the one value `v2`, its default: `set engine = v1;`, `--set engine=v1`, `{"settings": {"engine": "v1"}}` and diff --git a/docs/rfcs/0037-deterministic-simulation-harness.md b/docs/rfcs/0037-deterministic-simulation-harness.md index 9675bc7c5..4d7da7a7c 100644 --- a/docs/rfcs/0037-deterministic-simulation-harness.md +++ b/docs/rfcs/0037-deterministic-simulation-harness.md @@ -274,7 +274,7 @@ exactly: | # | Name | Imitated reality | Who sees the fault | What must survive | Positions indexed by | Mechanism | |---|---|---|---|---|---|---| -| 1 | injected storage faults | the store misbehaves on a call | the operation (an error or wrong bytes) | its retry and error paths | the call | call-level | +| 1 | injected operation failures | the store misbehaves on a call, or the engine fails before publication | the operation (an error or wrong bytes) | its retry and error paths | the storage call or pre-publication seam | call-level or named decision seam | | 2 | crash windows | the process dies at a phase boundary | nobody; the process is gone | recovery, from a marked place | code location (the 71 cataloged windows) | line-level | | 3 | crash-state enumeration | the process dies between durable writes | nobody; the process is gone | recovery, from every write cut | write ordinal (W writes = W states) | call-level (a write counter) | @@ -284,7 +284,9 @@ Per model, what is built and measured: inputs") injects clean errors and latency, corruption on the read path (value-aware bit rot, truncated reads, persistent latent sector errors), lost and misdirected writes, ack-loss (the effect - durable, the confirmation lost), and bounded staleness. + durable, the confirmation lost), and bounded staleness. The separate + `engine_effect_error_pct` stream returns failures at existing + pre-publication seams while the process stays alive; it is not a crash. 2. The judged contract is two-sided: a failed operation is invisible XOR fully applied, and committed writes never roll back. 3. Since memory dies with the process, storage is the only surviving @@ -694,6 +696,17 @@ by attribution: a damage ledger records what was injected, so engine-born detection errors are legalized by overlap instead of guessed at. +Coverage follows the methods the workload actually calls. Current-format +schema contracts are Lance rows. Adapter text-read corruption and staleness +are qualified against offline legacy upgrade admission; rejected attempts +must preserve the adapter and Lance objects, followed by a successful clean +conversion. Binary graph-index writes support loss and misdirection but not +the adapter's text-only corruption. Lance-call weather supports errors, +latency and lost acknowledgements and remains outside strict report replay. +The separate engine-effect stream preserves strict replay, requires counted +seam delivery, and is scoped to workload operations under serialized scenario +ownership; the live-handle owner also requires subsequent successful writes. + ### Provenance Every recorded execution is a numbered run whose meta names the diff --git a/docs/rfcs/2026-09-18-shared-schema-gate.md b/docs/rfcs/2026-09-18-shared-schema-gate.md index 44e66a8e3..64dc1264c 100644 --- a/docs/rfcs/2026-09-18-shared-schema-gate.md +++ b/docs/rfcs/2026-09-18-shared-schema-gate.md @@ -19,6 +19,8 @@ blocked_on: [] > A term set in ***bold italics*** is being defined at that exact spot; it is > used plain everywhere after. +The process-local gate remains in force. [Schema contract in the manifest](2026-09-30-schema-contract-in-manifest.md) replaces root-file contract installation and the schema-apply sentinel with atomic manifest publication. + ## Summary Resolve the schema-gate half of [RFC 0067](0067-detached-table-commits.md)'s diff --git a/docs/rfcs/2026-09-21-detached-only-tables.md b/docs/rfcs/2026-09-21-detached-only-tables.md index f2fc22da7..7b88f49b9 100644 --- a/docs/rfcs/2026-09-21-detached-only-tables.md +++ b/docs/rfcs/2026-09-21-detached-only-tables.md @@ -31,6 +31,8 @@ part of it, promotion, and replaces what promotion was buying. RFC 0067's probe table, excluding probe 14, and its Decision log; draft [RFC 0068](0068-graph-commit-record.md). +The added-type retry rule is extended by [Schema contract in the manifest](2026-09-30-schema-contract-in-manifest.md#apply-and-table-creation): an empty original version may be reused or replaced by a detached candidate at logical version 2, preserving linear version 1. + ## Summary Under RFC 0067 a write stages each table effect as a ***detached commit***, diff --git a/docs/rfcs/2026-09-30-schema-contract-in-manifest.md b/docs/rfcs/2026-09-30-schema-contract-in-manifest.md new file mode 100644 index 000000000..5ec745287 --- /dev/null +++ b/docs/rfcs/2026-09-30-schema-contract-in-manifest.md @@ -0,0 +1,398 @@ +--- +rfc: "2026-09-30-schema-contract-in-manifest" +title: "Schema contract in the manifest" +track: maintainer +status: draft +implementation: in-progress +authors: + - azimafroozeh +created: 2026-09-30 +updated: 2026-09-30 +discussion: null +supersedes: [] +superseded_by: [] +blocked_on: + - "Obtain the designated human verification of the measured DST cost golden; no such approval is recorded." + - "Obtain the designated human review of the required durable-call registry classifications; no such approval is recorded." +--- + +# RFC: Schema contract in the manifest + +> A term set in ***bold italics*** is being defined at that exact spot. + +## Summary + +Store the ***schema contract***, the accepted schema source, compiled schema +and their identity, in one live row of `__manifest`, the Lance dataset that +publishes a graph's table references and commit history. Schema apply replaces +that row in the same commit as its table references and graph head. A reader +therefore accepts a complete published contract together with the graph state +that uses it. + +This replaces the root schema files and the schema-apply sentinel with the +existing manifest publication boundary. The process-local schema gate stays. +The format change requires an explicit offline conversion; normal open does +not migrate a graph. Historical queries keep their current-contract behavior, +described under User and operational behavior. + +## Motivation + +The previous protocol publishes table changes first and installs three schema +files afterward. A crash between those steps leaves a published graph whose +contract still needs installation. Open, refresh and write entry must decide +whether staged files belong to a published apply or an unfinished one. A +sentinel branch announces that an apply is in progress, but the process-local +gate cannot fence another process's decisions about those files. + +The graph already has a ***compare-and-swap (CAS)***, a publication that succeeds +only against the captured manifest version. Bringing the contract into that +publication removes the separate installation decision. It also removes the +root-file reads and sentinel checks used to coordinate it. This has a byte +cost: the current manifest publisher rewrites its live rows, so the contract +text is copied on ordinary publications too. Design states that tradeoff. + +## User and operational behavior + +The schema commands and schema language keep their existing interfaces. +Schema apply retains its main-only, single-live-branch restriction and existing +policy checks. A successful apply makes its schema and table changes visible +together. A failure before publication leaves the previous graph and contract +visible. A lost acknowledgement still requires the existing publication-outcome +check; it is not proof that the apply lost. + +A live read uses the contract of its resolved manifest version. A historical +query uses the accepted live contract, with table aliases rebound by stable +identity to the selected historical image. For example, after renaming a type, +the current name still addresses its retained data. Retaining an older +`schema_contract` row does not introduce historical schema-language semantics +or a schema-history API. An unrefreshed handle must observe a completed apply +when it next captures the live contract for such a query; that regression is +an explicit evidence gate below. + +`omnigraph schema upgrade-system-columns` uses the same atomic contract +publication as schema apply. The schema IR, the compiled representation of the +schema, still determines system-column spelling. Storage conversion does not +rename user data or change its system-column vintage. + +Operators use `omnigraph upgrade --check --json`, then execute the same +command without `--check` while all readers, writers and maintenance processes +are stopped and a restorable whole-root backup is retained. The new target and +route are specified under Compatibility and reversibility. Interrupted +conversion is resumed with the upgrade-capable executable; removing a pending +marker by hand is not a recovery procedure. Cluster-managed conversion remains +outside the supported boundary of the existing offline protocol. + +## Design + +### Definitions and existing authorities + +The schema contract consists of its exact source text, serialized SchemaIR and +small identity fields. The graph head identifies the accepted graph commit. +A table registration names the table identity and exact physical version used +by that graph state. A ***detached commit*** is a Lance table version that can +be opened by its identifier without advancing that table's linear history. +These table effects become graph-visible only when a manifest registration +names them. + +The design follows the replace-in-place `graph_head` row and existing manifest +publisher. It keeps the ordering of the +[shared schema gate](2026-09-18-shared-schema-gate.md) and the table-version +rules of [Detached-only tables](2026-09-21-detached-only-tables.md). It changes +the contract-installation part of those protocols. It introduces neither a +second catalog dataset nor a new lock or identity namespace. + +### Row and validation + +The canonical row has `object_id = "schema_contract"` and +`object_type = "schema_contract"`. Its `metadata` contains `schema_ir_hash`, +`schema_identity_version` and `schema_identity_domain`. The exact source and IR +texts occupy nullable `LargeUtf8` columns, `schema_source` and `schema_ir`, +beside the packed `record` column. Both texts are required on this row and null +on every other row. The row has no table identity or table-version fields. +There is exactly one live contract row per converted manifest branch. + +Serving captures read contract text with the state in one manifest scan. +Historical and migration readers may project only the small identity. +Content loading pins both native branch and manifest version; equal version +numbers on different branches are not interchangeable. The engine checks the +identity envelope, parses and validates the IR, recomputes its compiler-defined +hash and proves that the source compiles to the IR's semantic shape. The IR +hash is not a hash of the raw JSON formatting. It then validates table +identities and paths against the captured catalog. + +The accepted-catalog cache retains the complete validated row and its exact +captured manifest image. A new image must supply its row before a catalog with +the same semantic identity can be reused. Unchanged images reuse validation; +an acknowledged publication can supply its exact retained rows without I/O. +A cache hit cannot supply a contract from another identity domain, or accept +changed source or IR merely because its head is unchanged. +A missing, duplicate, malformed or mismatching row fails admission; +normal serving has no root-file fallback. + +Fresh initialization includes the row in its genesis manifest Create. It +retains the existing `__init_claim.json` ownership and exact-genesis outcome +checks. The claim remains necessary to coordinate competing initializers; +removing schema files does not remove that ownership problem. + +### Apply and table creation + +Schema apply stages existing-table rewrites as detached commits. Its one main +publication contains the new registrations and tombstones, the graph-lineage +update and replacement contract row. It checks the captured graph head, branch +identity and expected table versions. The system-column upgrade submits its +renamed table references and replacement contract through the same publisher. +Installing a new in-memory view afterward is cache maintenance, not another +durable contract installation. A published schema needs no file promotion on +open or refresh. + +Added-type creation needs special care because retries derive the same path +from the accepted identity allocator. The rule is to preserve that +path and every existing version. An original empty, stable-row-ID version-one +dataset with the desired physical schema can be reused. If its schema differs +because an earlier uncommitted apply requested another shape, stage an empty +detached Overwrite based on that original version. Publish that result at +logical table version 2, with its detached version, transaction UUID and +`last_linear_version = 1`, against expected absent table version 0. A fresh or +matching original Create remains logical version 1. The detached replacement +must be above the last linear version so readers open it rather than the old +linear image. Nonempty or otherwise unqualified leftovers refuse. + +The final manifest CAS decides which competing apply may publish. A stale +attempt must not delete or overwrite a path another attempt may already have +registered. The schema-apply owners cover matching reuse, changed desired +schema and a stale competing apply that loses without deleting the path. + +### Projection and write cost + +The content columns remain outside the packed record so ordinary catalog folds +can omit their bytes. A cache miss reads the contract row once for the accepted +identity and validates it; later captures can reuse the accepted catalog. +This removes contract-file requests, not all manifest I/O. + +Cold open captures state, lineage and contract content in one scan of a pinned +image. It preserves the later freshness refresh: a changed manifest incarnation +discards the captured content and reads the refreshed image. Content errors are +held until that choice, so a concurrent repair is judged at the same boundary. +Routine state scans continue to omit the content columns. + +Ordinary open retains the metadata image used for format admission before the +process-local schema queue. It consumes that image for the combined scan, then +checks the serving format of the image held after the freshness refresh. If a +captured scan fails, one fresh metadata open may replace it only when the +complete image changed; an unchanged failed image remains an error. + +The combined scan can use a temporary reader for direct data files whose known +encoded sizes total at most 64 KiB, when the existing read block is smaller. +Each eligible file is fetched through a bounded request and its response size +is validated. Body failures retain Lance's normal download-retry behavior. +The coordinator keeps its original reader. Large files, indirect layouts, +pending upgrades and raw in-memory bindings retain their existing path. This +bounds the encoded files eligible for this policy, not total decoded memory. + +Named-branch queries may borrow an existing matching write coordinator after a +fresh incarnation probe; queries do not insert or refresh that cache. Deletion +takes a matching cached coordinator after its branch gate and validates it after +the table gates. A stale or absent capture uses the ordinary coherent opener. +Native-ref retirement and catalog validation remain required. + +The publisher may retain the exact stored rows of its last acknowledged +publication. A fresh open must match the complete immutable Lance manifest, +its native location and schema metadata before reuse. Its one-entry cache has +an 8 MiB accounted Arrow-buffer/key budget; misses use the same full scan. +The same row decoder, logical preconditions and version CAS still run. Exact +source and IR bytes are never reconstructed from the accepted-catalog cache. + +The existing copy-on-write publisher rewrites the live row set. Keeping the +contract inline therefore carries its source and IR payload on each publication +and in retained manifest versions. Physical bytes and request counts depend on +Lance encoding, compression and range-read block sizes. Evaluation includes +request counts and bytes across schema sizes and warm and cold operations. +Retained manifest versions may retain older contract text under ordinary +retention; this proposal adds no separate audit-retention guarantee. + +A synthetic measurement against base `7f1233bb` uses 140 node types, four +properties per type and one populated row. Its source is 14,839 bytes and its +pretty SchemaIR is 311,064 bytes. Fresh debug executables run the same seeded +case on the in-memory object store with a 4 KiB read block size, the unit cost +model and two-process replay: + +| Executing operation | Requests, base → candidate | Bytes read, base → candidate | Bytes written, base → candidate | +|---|---:|---:|---:| +| Warm query before or after the write | 1 → 1 | 0 → 0 | 0 → 0 | +| Update one populated row on main | 25 → 21 | 79,988 → 407,859 | 143,808 → 472,046 | + +This fixture measures the first write after setup, so its publisher has no +retained prior publication. It does not measure a second warm write. + +These measurements cover Lance object-store traffic, excluding setup and the +runner's checks. Root schema-file requests are measured separately by the +engine cost owners. The write adds two manifest-data requests and removes six +sentinel-ref requests; carrying the contract costs about 328 KB more read and +written per publication in this fixture. Request savings do not imply byte +savings. Small files may cross a range-read threshold and require more GETs. +The experiment measures neither real cloud latency nor CPU parsing time. + +The workspace pins Lance 11.0.0 and writes stable file version V2_2. For a +filtered scan of the whole packed record, Lance's default materialization +heuristic can split fixed-width and variable-width children between early and +late reads. The contract-row scan uses `MaterializationStyle::AllEarly`, which +keeps the projected record together. The exact filtered-shape guard also +passes with `AllLate`; neither explicit style splits the record as the default +heuristic does in this fixture. This does not make isolated packed-child projection supported, +and it makes no claim that every row-address read is broken. The storage +surfaces are described in [the Lance contract](../dev/lance.md); the relevant +locked implementation is `lance-11.0.0`'s scanner materialization and strict +overwrite paths. + +### Explicit conversion and cleanup + +The migration extends the intent, per-branch receipt and final activation +protocol of [RFC 0064](0064-explicit-storage-upgrades.md). + +1. Inspect each live native branch at its captured version. Accept actual flat + v11 and packed v12 layouts, including either mixture across main and named + branches. Preserve exact branch identities, table registrations, commit IDs, + ancestry, unrelated metadata, retirement records and retained versions. + Refuse incompatible physical schemas, legacy contract staging, a residual + schema-apply sentinel and unresolved recovery sidecars before effects. +2. Read and validate `_schema.pg`, `_schema.ir.json` and `__schema_state.json` + through the upgrade-only legacy reader. Preserve the chosen source and IR + bytes exactly. Pin their raw-text SHA-256 digests and the semantic contract + identity in protocol 5's existing durable intent. Comment or JSON-format + changes must be detected even when the semantic identity stays equal. +3. Fence main using the existing pending-upgrade metadata. Convert each branch + from its own captured version. One strict overwrite writes its preserved + logical rows, the contract row, target stamp and completion receipt. A + schema-format conversion creates no artificial graph commit. Strict + overwrite uses zero Lance commit retries; foreign movement is not rebased + into an accepted conversion. +4. Validate every receipt and converted row against the pinned contract, + including the exact text digests, and revalidate the complete branch + inventory. Convert main after named branches and keep its pending marker. + Before completion, retry still needs the intact legacy contract. Once all + branches are converted, main's validated row supplies the canonical bytes. +5. Delete the three legacy contract files only after that validation. Missing + files are tolerated at this stage, so retry can finish after any subset of + deletes. Revalidate ownership and activate main last by clearing the pending + marker. A cleanup failure leaves normal open fenced until retry finishes. + +The earlier v10-to-v11 step needs an engine handle before the source has a +contract row. A narrow upgrade-only constructor takes a validated contract, +requires the existing v10 conversion admission and supplies it only when that +handle's source snapshot lacks a row. It cannot hide a malformed present row +or relax normal open. The override is immutable per-handle state, not a new +global contract hook. An ordinary open must remain refused while such a +conversion handle exists. + +## Invariants + +- A published schema change has one manifest version containing its contract, + table references and graph-lineage change; there is no published contract + installation left for another process to finish. +- The table references and accepted catalog used by an operation come from one + captured authority. A retry discards the previous attempt's captured state. +- Ordinary serving requires the row and validates its full identity. Root + schema files are inputs only to explicit conversion. +- An added-type attempt preserves an existing path and its versions. Only the + manifest publication grants its candidate graph authority. +- Conversion preserves logical graph history and data identities. Before + activation, main remains fenced; after activation, every live branch has the + same validated contract bytes and no legacy contract file is required. +- A local schema permit provides process ordering. The manifest publication + remains the cross-process authority, and offline upgrade still requires + operator-enforced quiescence. + +## Compatibility and reversibility + +Set both the current internal manifest stamp and serving floor to **13**. +The physical decoder retains the distinction between flat v11 and packed v12 +for conversion and retained history. Ordinary open refuses both source stamps +and every future stamp before serving. + +Register `schema-contract-v11-v12-to-v13` and make 13 the default upgrade +target. Qualified older inputs compose through the existing steps: +`6 → 7 → 8 → 10 → 11 → 13`; v9 starts with the existing step to v10. Preserve +explicit intermediate targets 7, 8, 10 and 11 for compatible executables. +Format 12 remains an unsupported explicit target. A valid v13 graph reports +`already_current` without requiring deleted legacy files. Check mode has no +effects and reports later checks that depend on intermediate output as deferred. + +Source formats without an admitted route still require export with a compatible +binary and rebuild. Conversion does not grant serving admission to an +intermediate format. Rollback restores the complete pre-upgrade backup with +the compatible executable; retained old manifest versions are not a downgrade +mechanism. Writes after conversion are absent from that backup. + +## Alternatives + +**Keep the root files and remove only the sentinel.** This removes an advisory +mechanism but leaves the manifest commit and file replacements separate. A +crash after publication still needs a durable way to identify and finish the +contract installation. It does not achieve the stated publication invariant. + +**Put only identity in the manifest and store immutable text by hash.** This +keeps the manifest small, but creates contract objects that must exist before +publication and remain available for every referencing version. It adds their +lookup and reclamation rules. Inline content uses the existing manifest +lifetime and accepts the byte tradeoff, subject to the measurements above. + +**Store the contract in a separate dataset.** A second dataset does not share +the manifest CAS. A pointer to an immutable version could bridge the two, at +the cost of another dataset, version reference and retention dependency. +The one-row design avoids that additional lifetime. + +**Keep the new row but omit exact-text ownership from upgrade intent.** The +semantic IR identity permits source-comment and JSON-format changes. If those +files change between branch conversions, branches could receive different +bytes while every semantic check passes. Raw-text digests in the existing +intent reject that concrete partial-conversion case. They are conversion +evidence, not another serving authority. + +**Delete an unregistered added-type path on retry.** Another apply can register +that deterministic path after the retry's inspection. Deletion then destroys a +published table before the retry loses its final CAS. Reusing the original +empty version or staging a detached replacement avoids that destructive step. + +## Evidence and tests + +The following owners define the required proof. Local database unit tests +cover the converter's layouts, exact contract ownership and interrupted cleanup. +Both released-predecessor journeys and the system-column and Blob composition +owners pass locally. The packed guard confirms default failure and identical +AllEarly/AllLate results. Final workspace correctness and cost gates, and +designated human review, remain prerequisites for complete implementation. + +| Boundary | Existing owner to extend | Required proof | +|---|---|---| +| Row format and publication | `omnigraph-catalog` tests | Exact text round trip; one row through ordinary publishes; duplicate, missing, null and identity mismatch refusals; branch-and-version pinning; atomic replacement with graph changes. | +| Schema lifecycle | `tests/schema_apply.rs`, `tests/system_column_upgrade.rs`, `tests/failpoints.rs`, `tests/detached_commit_matrix.rs` | Before/after publication windows; fresh read-only and read-write open; same-handle progress after faults; no contract staging or sentinel; existing system-column behavior. | +| Added-type retry | `tests/schema_apply.rs` and failure-window owners | Matching reuse; changed desired schema; stale competing loser; original nonempty/foreign-state refusals; reads, cleanup, repair and change discovery of logical version 2 with last linear version 1. | +| Read binding | `tests/point_in_time.rs` and schema owners | Unrefreshed handle after another handle's rename; current aliases on retained images; full-identity cache key; live freshness and retained old storage layouts. | +| Packed scan | `tests/lance_surface_guards.rs` | Default heuristic, AllLate and AllEarly against the same filtered whole-record request; exact AllEarly output with fixed-width and variable-width children. | +| Conversion | `db/upgrade/tests.rs` | Real flat and packed source layouts; both mixed directions; byte/identity/history preservation; source-text drift with equal semantic identity; strict branch movement; every fence, conversion, delete and activation interruption; no-op retry and check mode; ordinary-open isolation from the upgrade constructor. | +| Released predecessors | CLI `tests/crossversion_upgrade.rs` | `genuine_v09_explicit_storage_upgrade_preserves_history` and `genuine_v010_explicit_storage_upgrade_preserves_history` create sources with the released executables and complete the new route with branches and retained history. Missing executables and skipped cases do not count. | +| Cost and structural gates | `write_cost.rs`, `warm_read_cost.rs`, `branch_control_cost.rs`, `forbidden_apis.rs`, `check-storage-upgrade-ci.py` | Contract request removal, manifest-byte cost across schema sizes, reviewed durable-call ownership and required test execution. Backend qualification remains specific to each tested environment. | + +## Rollout + +Complete the catalog and engine changes, then qualify the explicit converter +and its earlier-route dependency. Extend existing test owners rather than +adding a second upgrade harness. Run the required predecessor and correctness +gates, then record the public cost evidence and remaining backend limits. + +Update the current versioning, upgrade, writes, recovery, invariants and test +ownership documents together with the release note. The current docs must +describe implemented behavior; they must not gain authority from this draft. +Link the changed contract-installation and added-type cases from the earlier +RFCs without marking their entire protocols superseded. A maintainer decision +on this RFC remains separate from implementation progress. + +## Unresolved questions + +There are no unresolved design choices within the selected scope. The evidence +and designated human-review gates are listed in the frontmatter and Evidence +and tests. + +## Decision log + +No external review outcome or maintainer acceptance is recorded. diff --git a/docs/rfcs/README.md b/docs/rfcs/README.md index ddfba33ab..34fd01e14 100644 --- a/docs/rfcs/README.md +++ b/docs/rfcs/README.md @@ -238,3 +238,4 @@ then dated RFCs by date. | [2026-09-30](2026-09-30-v012-http-admission.md) | v0.12 HTTP admission | maintainer | accepted | complete | | [2026-09-30](2026-09-30-exact-merge-receipts.md) | Exact merge receipts | maintainer | accepted | complete | | [2026-09-30](2026-09-30-owned-server-operations.md) | Owned server operations | maintainer | accepted | complete | +| [2026-09-30](2026-09-30-schema-contract-in-manifest.md) | Schema contract in the manifest | maintainer | draft | in-progress | diff --git a/docs/user/operations/upgrade.md b/docs/user/operations/upgrade.md index 2893abb78..d58e1e4e4 100644 --- a/docs/user/operations/upgrade.md +++ b/docs/user/operations/upgrade.md @@ -1,24 +1,24 @@ # Upgrading OmniGraph -Normal open accepts storage formats v11 and v12; nothing migrates on open, but -a v11 branch becomes v12 on its next publish. A v8, v9 or v10 graph from the -0.11.x line takes one explicit `omnigraph upgrade` (below), which keeps its -branches and spellings; use export/import with a source-compatible binary when -no route exists. Storage formats, release versions and full-text index formats -are separate; check the [release notes](../../releases/) before upgrading. +Normal open accepts storage format v13 and never migrates a graph. Admitted +older formats, including v11 and v12, require explicit `omnigraph upgrade`. +Use export/import with a source-compatible binary when no route exists. +Check the [release notes](../../releases/) for storage and index compatibility. ## Explicit storage migration -`omnigraph upgrade` defaults to v11: the conversions to v8 where the source -needs them, the v10 stamp step, then the v11 step, which promotes every pending -table pin once, reaps proven copies oldest link first, records the last linear version on -every registration of every live branch and restamps. Qualified standalone v6 -graphs run v6 → v7 registration conversion, then metadata-only v7 → v8; -qualified v7 development graphs run only the latter. Both reuse table data and -preserve branch ancestry, IDs, property values, schema identity, retained -commit IDs and numeric snapshots. Historical v6 snapshots keep their original -metadata and use an explicit legacy decoder after root admission; historical -v7 snapshots keep their registration-clock interpretation. +`omnigraph upgrade` defaults to v13. Qualified standalone v6 graphs run +`v6 → v7 → v8 → v10 → v11 → v13`; v7 inputs start at v7, v8 and v9 inputs +enter the v10 step, and v10 inputs enter the v11 step. The v11 step promotes +each pending v10 table pin once, reaps proven copies oldest first and records +the last linear version on every live branch's registrations. + +The final step accepts v11, v12 and mixed branch formats. It preserves the +schema source and compiled schema, data, identities, ancestry, retained commit +IDs and numeric snapshots. It creates no graph commit or graph-table payload +rewrite. Legacy schema files are removed only after every branch validates. +System-column spellings stay unchanged. See [storage versioning](../../dev/versioning.md) +for physical layouts and retained-history decoding. 1. Stop every server, embedded writer, maintenance process and cluster apply that could touch the graph or its shared dependencies. A process-local lock @@ -32,9 +32,9 @@ v7 snapshots keep their registration-clock interpretation. ``` 4. Inspect `outcome`, `findings`, `route` and `work`. A passing check is advisory; - execution repeats validation. For v6 → v8, `work.deferred_checks` identifies - checks requiring the intermediate v7 output. Execution validates these before - the v7 → v8 handler has effects; a passing check does not pre-approve them. + execution repeats validation. `work.deferred_checks` identifies downstream + preflights that need intermediate output. Execution validates each before + its handler has effects; a passing check does not pre-approve them. Resolve source recovery with the compatible source executable before retrying. Shared Lance files outside the root refuse. `work.external_blob_exclusions` lists external URI bytes whose immutability @@ -56,48 +56,47 @@ v7 snapshots keep their registration-clock interpretation. with the old executable; old bytes remaining in the upgraded root do not make downgrading safe. Post-upgrade writes are absent from that backup. -`--to-format` defaults to 11. Explicit `--to-format 7`, `8` or `10` stops there -for a compatible older executable; the current binary refuses each result. v9 -is not a target: the respelling it marked is a separate operation on a served -graph (below). A v11 or v12 graph, mixed or not, reports `already_current` for -the default and `--to-format 11`; a lower target is refused as a downgrade, -`--to-format 12` as unsupported. The v11 step refuses, before any write, a pin -whose linear target a foreign commit occupies (`blocked_promotion`, reported by -`--check` too); after the switch no pin can reach that state. Unsupported -sources and targets refuse; there is no automatic data-moving fallback. Check -and execution return zero only for success (`check_passed`, `completed` or -`already_current`); a repeated success is a no-write no-op. A source with a -pending recovery sidecar refuses until the executable that wrote it resolves it. - -After each handler's early fence, ordinary opens refuse until every branch is -converted and validated and main activates that handler's target. If interrupted, -retain the backup and rerun the same requested target without `--check`, using -this upgrade-capable executable. An earlier pending v6 → v7 attempt keeps its -exact original protocol, target and attempt identity; requesting v8 first -finishes that attempt, then runs v7 → v8. Requesting v7 cannot downgrade or -resume a pending v7 → v8 attempt. The report identifies the last durable boundary -and required recovery action. Unknown ownership or foreign branch movement -requires investigation; never delete the pending marker to force serving or -point the source executable at it. - -Source v6/v7 graphs containing reserved native-ref retirement metadata refuse -conversion. On a v8 graph, admission validates that metadata and excludes -valid retired refs from logical branch enumeration while retaining their physical -ancestry. Upgrade neither retires branches nor reclaims their storage. Branch -naming must also be unambiguous: a native name ending in a ULID-shaped suffix -could be either a v0.9 logical name or a newer branch incarnation, so the -handler requires a logical-head commit written after that native branch's fork -to prove the interpretation and otherwise refuses before writing, including -unused suffixed branches without that evidence. Duplicate logical names and -incarnation-shaped inner path segments also refuse. Resolve the branch naming -with the source executable or use the export/rebuild fallback; do not rename +`--to-format` defaults to 13. Explicit targets 7, 8, 10 and 11 stop there for +a compatible executable; the current binary refuses normal open of each +result. Neither 9 nor 12 is a target. A valid v13 graph reports +`already_current` for the default or explicit target 13, and refuses lower +registered targets as downgrades. A v13 no-op uses its contract row: it neither +reads orphan legacy schema files nor treats the former schema-apply sentinel +name as a system branch. Ignored orphan schema staging remains untouched. + +The v11 step refuses a pin whose linear target a foreign commit occupies +(`blocked_promotion`, reported by `--check` too). Legacy contract conversion +refuses schema staging, a residual schema-apply sentinel, invalid contract +files or unresolved recovery sidecars before effects. Resolve a sidecar with +the executable that wrote it. Unsupported sources and targets refuse without +an automatic data-moving fallback. Check and execution return zero only for +success (`check_passed`, `completed` or `already_current`); a repeated success +is a no-write no-op. + +An interrupted upgrade keeps ordinary opens fenced. Retain the backup and +rerun the same requested target without `--check`, using this upgrade-capable +executable. Earlier pending steps finish before later steps begin; requesting +a lower target cannot downgrade or resume a pending higher step. The report +identifies the last durable boundary and recovery action. The final step +activates only after every branch validates and all legacy schema files are +removed. Retry tolerates files already removed and checks the chosen contract +bytes and branch ownership. Investigate changed bytes or foreign movement; +never delete the pending marker or open the root with the source executable. + +Source v6/v7 graphs with reserved branch-retirement metadata refuse conversion. +On v8, valid retired branches retain their physical ancestry. Upgrade never +retires branches or reclaims their storage. Ambiguous branch names, duplicate +logical names and unproven incarnation-shaped names refuse before effects. +Resolve names with the source executable or use export/rebuild; do not rename native Lance refs or edit their metadata manually. Upgrade admission bounds each retained manifest to 1,000,000 rows and 64 MiB of decoded batch metadata, 1,024 native branches and 100,000 retained versions per branch. Version references are counted before historical manifests are -loaded. Exceeding a bound refuses before conversion. Payload bytes copied -and rewritten are zero; managed Blob validation can still read substantial data. +loaded. Exceeding a bound refuses before conversion. The final step rewrites +manifest metadata and copies the schema source and IR into it; graph-table +payload files are not rewritten. Managed Blob validation can still read +substantial data. Server and cluster selectors, cluster profiles and recognized cluster-layout roots refuse until a cluster upgrade protocol is qualified. Local paths, file URIs and symlink aliases are resolved before the cluster ownership check. Direct path access @@ -239,35 +238,37 @@ mapping is: | v5 | the exact unreleased development build that wrote it | | v6 | latest 0.10.x (the refusal names 0.9.x or 0.10.x) | | v7 | the exact unreleased development build that wrote it | -| v8, v9, v10 | the 0.11.x line (v8: development builds before the system-column namespace change and conversions completed with `omnigraph upgrade --to-format 8`; v10: detached table commits); `omnigraph upgrade` takes each to v11 without export/import | -| v11, v12 | current line: fresh graphs are born v12 and a v11 branch converts on its next publish, so a graph may mix the two until every branch has published; entity export/import normally not required within this generation | +| v8, v9, v10 | the 0.11.x line (v8: development builds before the system-column namespace change and conversions completed with `omnigraph upgrade --to-format 8`; v10: detached table commits); `omnigraph upgrade` takes each to v13 without export/import | +| v11, v12 | the compatible predecessor development build; explicit `omnigraph upgrade` converts flat, packed or mixed branches to v13 | +| v13 | current binary; entity export/import is not required within this storage generation | If the graph's generation is newer than the binary, upgrade the binary instead. ## System-column upgrade (legacy spellings) `omnigraph schema upgrade-system-columns ` respells a served graph's -legacy system columns in place (`id`/`src`/`dst` to `__id`/`__src`/`__dst`); -a v11 main becomes v12. Columns are renamed by field id: no table rows are -rewritten, indexes survive, and data, history, and commit ids are unchanged; +legacy system columns in place (`id`/`src`/`dst` to `__id`/`__src`/`__dst`). +Run storage conversion first if normal open refuses the graph. The respelling +keeps v13 and publishes the replacement contract with the renamed table +references in one new graph commit. Columns are renamed by field id: no table +rows are rewritten, indexes survive, and data and existing history/commit IDs are preserved; `--check` runs the preflight and writes nothing; `--json` prints the report. The preflight refuses a graph with any non-main branch (merge what you need, then delete the branches: a merge alone leaves the source live) and a property whose name starts with `_` (rename it with `@rename_from` first), naming every offender under `system_columns_preflight`. Edge constraints such as `@unique(src, dst)` become `@unique(@src, @dst)`. Stop every server -serving the graph and retain a verified backup first: a write from a server -still running lands the graph in a refusal only a restore clears, and there -is no reverse operation. A run interrupted before it publishes changes nothing: rerun -it. One interrupted after is completed by the next read-write open (any CLI command but -`--check`; a rerun reports `already_current`); a read-only handle refuses until then. A graph that +serving the graph and retain a verified backup first. Concurrent graph movement +can refuse the attempt; stop writers and rerun from a fresh capture. +There is no reverse operation. A run interrupted before it publishes changes nothing: rerun +it. Once publication succeeds, fresh read-only and read-write opens use the +accepted schema; a rerun reports `already_current`. A graph that already spells `__id`/`__src`/`__dst` reports `already_current`; cluster-managed graphs are refused. ## Rebuild -Keep the old and new executables separate. Use a different target URI so the -source remains recoverable throughout verification. +Keep separate executables and target URIs so the source remains recoverable. When an export already carries top-level `id`, the rewrite preserves it and any user property named `id`. For predecessor schemas, change endpoint constraint references from `src`/`dst` to `@src`/`@dst` before the new `init`. See [ingestion](../../dev/ingestion.md#strict-graph-batch) for the envelope contract. @@ -291,7 +292,6 @@ omnigraph schema show s3://bucket/graph-new.omni ``` For another branch: - ```bash old-omnigraph export --branch review s3://bucket/graph.omni \ > review.jsonl diff --git a/scripts/check-storage-upgrade-ci.py b/scripts/check-storage-upgrade-ci.py index 6b4c4717e..dddbf39fa 100644 --- a/scripts/check-storage-upgrade-ci.py +++ b/scripts/check-storage-upgrade-ci.py @@ -13,12 +13,24 @@ CONTEXT = "Storage Upgrade Compatibility" FEATURES = "omnigraph-engine/failpoints,omnigraph-cluster/failpoints" CASES = ( + "storage_upgrade_current_binary_reports_already_current_on_a_fresh_graph", "genuine_v09_explicit_storage_upgrade_preserves_history", "genuine_v010_explicit_storage_upgrade_preserves_history", "storage_upgrade_refuses_cluster_path_aliases", "genuine_v09_storage_upgrade_refuses_ambiguous_branch_names", ) ENGINE_CASES = ( + "storage_upgrade_schema_contract_converts_mixed_branch_formats", + "storage_upgrade_schema_contract_cleanup_retries_after_each_delete", + "storage_upgrade_schema_contract_resumes_every_partial_cleanup_state", + "storage_upgrade_schema_contract_refuses_forged_historical_fence", + "storage_upgrade_refuses_restamped_current_layout_and_injected_legacy_contract", + "storage_upgrade_current_contract_ignores_legacy_orphans_and_former_sentinel_name", + "storage_upgrade_schema_contract_interruption_boundaries_retry", + "storage_upgrade_schema_contract_refuses_contract_drift_on_retry", + "storage_upgrade_schema_contract_refuses_invalid_contract_without_effects", + "storage_upgrade_schema_contract_refuses_legacy_artifacts_without_effects", + "storage_upgrade_legacy_constructor_does_not_admit_ordinary_opens", "storage_upgrade_check_has_no_local_store_effects", "storage_upgrade_interruption_boundaries_retry_without_mixed_visibility", "storage_upgrade_recovery_refuses_foreign_head_movement", From 6452ff1d319af4461e51a45fc6eca967c46df45f Mon Sep 17 00:00:00 2001 From: azim afroozeh <13484327+azimafroozeh@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:15:07 +0200 Subject: [PATCH 2/5] rebase --- crates/omnigraph-gqt/tests/runner_dispatch.rs | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/crates/omnigraph-gqt/tests/runner_dispatch.rs b/crates/omnigraph-gqt/tests/runner_dispatch.rs index afdd292ff..59ae8b2d9 100644 --- a/crates/omnigraph-gqt/tests/runner_dispatch.rs +++ b/crates/omnigraph-gqt/tests/runner_dispatch.rs @@ -154,21 +154,16 @@ fn explicit_environment_selection_and_lifetime_evidence() { .unwrap() .iter() .filter(|event| event["kind"] == "engine_lifetime") + .map(|event| event["value"].clone()) .collect::>(); - assert_eq!(lifetimes.len(), 3); - let mut opens = 0; - for event in lifetimes { - let before = event["value"]["before"].as_array().unwrap(); - let after = event["value"]["after"].as_array().unwrap(); - assert_eq!( - before[0], after[0], - "ordinary steps cannot initialize a graph" - ); - opens += after[1].as_u64().unwrap() - before[1].as_u64().unwrap(); - } assert_eq!( - opens, 1, - "the engine's real open hook must fire only for restart" + lifetimes, + vec![ + serde_json::json!({"before": [1, 0], "after": [1, 0]}), + serde_json::json!({"before": [1, 0], "after": [1, 1]}), + serde_json::json!({"before": [1, 1], "after": [1, 1]}), + ], + "the engine must initialize once and reopen only at the restart step" ); assert_eq!( events From fd61cf74d6b4b44fca14ba1a6b38686dd133df88 Mon Sep 17 00:00:00 2001 From: azim afroozeh <13484327+azimafroozeh@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:34:51 +0200 Subject: [PATCH 3/5] hmm --- crates/omnigraph/tests/forbidden_apis.rs | 105 ++++++++++++++--------- 1 file changed, 63 insertions(+), 42 deletions(-) diff --git a/crates/omnigraph/tests/forbidden_apis.rs b/crates/omnigraph/tests/forbidden_apis.rs index a289d0f66..683f2019e 100644 --- a/crates/omnigraph/tests/forbidden_apis.rs +++ b/crates/omnigraph/tests/forbidden_apis.rs @@ -190,7 +190,6 @@ enum WriteProtocol { EphemeralScratch, TestOnly, Bootstrap, - SchemaContractInstall, ReadOnlyAccess, ReadTaskCancellation, } @@ -206,7 +205,6 @@ impl WriteProtocol { Self::EphemeralScratch => "ephemeral scratch".into(), Self::TestOnly => "test/failpoint-only".into(), Self::Bootstrap => "bootstrap".into(), - Self::SchemaContractInstall => "staged schema contract installation (RFC 0067)".into(), Self::ReadOnlyAccess => "read-only raw snapshot access".into(), Self::ReadTaskCancellation => { "cancel a queued Tokio read worker; no storage effect".into() @@ -217,9 +215,10 @@ impl WriteProtocol { const MUTATION_V9: WriteProtocol = WriteProtocol::Exact("Mutation v9"); const LOAD_V9: WriteProtocol = WriteProtocol::Exact("Load v9"); -const SCHEMA_V9: WriteProtocol = WriteProtocol::Exact("SchemaApply v9"); +const SCHEMA_V9: WriteProtocol = + WriteProtocol::Exact("schema apply: table pins and schema contract in one manifest CAS"); const SYSTEM_COLUMNS_V9: WriteProtocol = - WriteProtocol::Exact("system-column upgrade (RFC 0040, detached renames per RFC 0067)"); + WriteProtocol::Exact("system-column upgrade: detached renames and schema contract in one CAS"); const MERGE_V9: WriteProtocol = WriteProtocol::Exact("BranchMerge v9"); const INDICES_V9: WriteProtocol = WriteProtocol::Exact("EnsureIndices v9"); const OPTIMIZE_V9: WriteProtocol = @@ -242,13 +241,13 @@ macro_rules! write_surfaces { write_surfaces! { "db/omnigraph.rs" => WriteProtocol::Bootstrap => ["init", "init_with_options", "init_with_storage"], - "db/omnigraph.rs" => WriteProtocol::SchemaContractInstall => ["open", "open_with_storage", "refresh"], + "db/omnigraph.rs" => WriteProtocol::Composed("read-write admission + owned local-root create-if-absent capability probe") => ["open", "open_with_storage"], "exec/mutation.rs" => MUTATION_V9 => ["mutate", "mutate_with_receipt", "mutate_as", "mutate_as_with_receipt", "mutate_as_with_expected_head", "mutate_as_with_expected_head_receipt"], "loader/mod.rs" => LOAD_V9 => ["load_jsonl", "load_jsonl_file", "load", "load_with_receipt", "load_file", "load_graph_batch"], "loader/mod.rs" => WriteProtocol::Composed("optional branch create, then Load v9") => ["load_as", "load_as_with_receipt", "load_file_as", "load_file_as_with_receipt", "load_graph_batch_as", "load_graph_batch_as_with_receipt"], "loader/mod.rs" => WriteProtocol::Composed("branch create when absent, then Load v9 alias") => ["ingest", "ingest_as", "ingest_file", "ingest_file_as"], - "db/omnigraph.rs" => WriteProtocol::Composed("SchemaApply v9 + sentinel ref + optional hard-drop GC") => ["apply_schema", "apply_schema_with_options", "apply_schema_as", "apply_schema_as_with_catalog_check"], - "db/omnigraph.rs" => WriteProtocol::Composed("SchemaApply v9 system-column upgrade + stamp advance + sentinel ref") => ["upgrade_system_columns", "upgrade_system_columns_as"], + "db/omnigraph.rs" => SCHEMA_V9 => ["apply_schema", "apply_schema_with_options", "apply_schema_as", "apply_schema_as_with_catalog_check"], + "db/omnigraph.rs" => SYSTEM_COLUMNS_V9 => ["upgrade_system_columns", "upgrade_system_columns_as"], "exec/merge.rs" => MERGE_V9 => ["branch_merge", "branch_merge_as"], "db/omnigraph.rs" => INDICES_V9 => [ "ensure_indices", "ensure_indices_on", @@ -268,6 +267,7 @@ write_surfaces! { // cannot evade discovery. const READ_ONLY_SURFACES: &[(&str, &str)] = &[ ("db/omnigraph.rs", "open_read_only"), + ("db/omnigraph.rs", "refresh"), ("db/omnigraph.rs", "cleanup_plan"), ("db/omnigraph.rs", "cleanup_plan_missing_paths"), ("db/omnigraph.rs", "cleanup_plan_path_snapshot"), @@ -317,6 +317,46 @@ const READ_ONLY_SURFACES: &[(&str, &str)] = &[ // crate-visible wrapper would otherwise create a new internal route to the // manifest publisher without changing the durable gateway count. const LOW_LEVEL_READ_ONLY_SURFACES: &[(&str, &str, &str)] = &[ + ( + "omnigraph-catalog/lib.rs", + "ManifestCoordinator", + "prepare_open_with_contract", + ), + ( + "omnigraph-catalog/lib.rs", + "ManifestCoordinator", + "open_prepared_with_lineage_and_contract", + ), + ( + "db/graph_coordinator.rs", + "GraphCoordinator", + "open_with_contract", + ), + ( + "omnigraph-catalog/lib.rs", + "ManifestCoordinator", + "open_with_lineage_and_contract", + ), + ( + "db/graph_coordinator.rs", + "GraphCoordinator", + "read_schema_contract", + ), + ( + "omnigraph-catalog/lib.rs", + "ManifestCoordinator", + "read_schema_contract", + ), + ( + "omnigraph-catalog/lib.rs", + "ManifestCoordinator", + "read_schema_contract_at", + ), + ( + "omnigraph-catalog/lib.rs", + "ManifestCoordinator", + "read_schema_contract_for_snapshot", + ), ( "db/graph_coordinator.rs", "GraphCoordinator", @@ -364,11 +404,6 @@ const LOW_LEVEL_READ_ONLY_SURFACES: &[(&str, &str, &str)] = &[ "refresh_for_live_read", ), ("db/graph_coordinator.rs", "GraphCoordinator", "branch_list"), - ( - "db/graph_coordinator.rs", - "GraphCoordinator", - "schema_apply_locked", - ), ( "db/graph_coordinator.rs", "GraphCoordinator", @@ -495,11 +530,6 @@ const LOW_LEVEL_READ_ONLY_SURFACES: &[(&str, &str, &str)] = &[ "ManifestCoordinator", "branch_identifier", ), - ( - "omnigraph-catalog/lib.rs", - "ManifestCoordinator", - "schema_apply_locked", - ), ( "omnigraph-catalog/lib.rs", "ManifestCoordinator", @@ -525,12 +555,6 @@ const LOW_LEVEL_WRITE_SURFACES: &[(&str, &str, &str, WriteProtocol)] = &[ "branch_create", WriteProtocol::NativeRefControl, ), - ( - "db/graph_coordinator.rs", - "GraphCoordinator", - "branch_delete", - WriteProtocol::NativeRefControl, - ), ( "db/graph_coordinator.rs", "GraphCoordinator", @@ -623,16 +647,21 @@ macro_rules! gateway_surfaces { const FREE_FUNCTION_OWNER: &str = "(free fn)"; /// Files whose crate-visible free `fn`s the registry pins: the index and staging -/// helpers that were `TableStore` methods before the crate split. +/// helpers that were `TableStore` methods before the crate split, plus the +/// exact module containing the bounded manifest scan reader. const FREE_FUNCTION_GATEWAY_FILES: &[&str] = &[ "omnigraph-core/dataset_index.rs", "omnigraph-core/staging.rs", + "omnigraph-core/instrumentation/small_manifest_reads.rs", ]; // Closed callable surface for the primitive/gateway types themselves. The raw // call inventory below catches body growth; this registry catches a new wrapper // or an entirely new primitive name before a crate-internal caller can use it. gateway_surfaces! { + "omnigraph-core/instrumentation/small_manifest_reads.rs" => "(free fn)" => GatewayDisposition::ReadOrPure => [ + "manifest_scan_dataset", + ], "omnigraph-core/storage.rs" => "StorageAdapter" => GatewayDisposition::ReadOrPure => [ "read_text", "read_text_if_exists", "read_text_if_exists_bounded", "read_bytes_if_exists_bounded", "exists", @@ -652,6 +681,7 @@ gateway_surfaces! { "write_text_if_match", "delete_prefix", ], "storage_layer.rs" => "TableStorage" => GatewayDisposition::ReadOrPure => [ + "validate_initial_empty_table", "transaction_identity", "open_snapshot_at_entry", "open_snapshot_at_table", "open_dataset_head", "branch_identifier", "list_native_branches", @@ -690,6 +720,7 @@ gateway_surfaces! { "promote_detached", ], "table_store.rs" => "TableStore" => GatewayDisposition::ReadOrPure => [ + "validate_initial_empty_table", "transaction_identity", "new", "root_uri", "dataset_uri", "open_snapshot_table", "open_at_entry", "open_at_entry_verified", "open_dataset_head", "list_native_branches", @@ -747,6 +778,9 @@ gateway_surfaces! { "omnigraph-core/staging.rs" => "(free fn)" => GatewayDisposition::ReadOrPure => [ "is_detached_version", ], + "omnigraph-catalog/publisher.rs" => "ManifestBatchPublisher" => GatewayDisposition::ReadOrPure => [ + "cached_rows", + ], "omnigraph-catalog/publisher.rs" => "ManifestBatchPublisher" => GatewayDisposition::Durable(WriteProtocol::Exact("manifest publisher gateway")) => [ "publish_with_precondition", ], @@ -776,6 +810,7 @@ macro_rules! durable_calls { // manifest implementations are included; only standalone test-only sources // whose parent cfg is invisible to this file walker are excluded. durable_calls! { + ("db/upgrade/legacy_schema_files.rs", ".delete(", 1, WriteProtocol::Exact("protocol 5: validate every converted branch, remove exact legacy contract files, then activate main")), ("db/upgrade.rs", "CommitBuilder::new(", 3, WriteProtocol::Exact("offline storage upgrade with main-owned intent")), ("db/upgrade.rs", "InsertBuilder::new(", 1, WriteProtocol::Exact("manifest-only conversion under durable upgrade ownership")), ("db/upgrade.rs", ".execute_uncommitted_stream(", 1, WriteProtocol::Exact("manifest-only conversion under durable upgrade ownership")), @@ -833,7 +868,7 @@ durable_calls! { ("storage_layer.rs", ".promote_detached(", 1, WriteProtocol::Exact("sealed TableStorage forwarding")), ("db/omnigraph/promotion.rs", ".promote_detached(", 1, WriteProtocol::Exact("RFC 0067 promotion replay")), ("db/omnigraph/promotion.rs", "SnapshotHandle::new(", 1, WriteProtocol::ReadOnlyAccess), - ("storage_layer.rs", ".dataset()", 31, WriteProtocol::Composed("sealed TableStorage forwarding")), + ("storage_layer.rs", ".dataset()", 32, WriteProtocol::Composed("sealed TableStorage forwarding")), ("storage_layer.rs", ".into_arc()", 6, WriteProtocol::Composed("sealed TableStorage forwarding")), ("storage_layer.rs", "SnapshotHandle::new(", 4, WriteProtocol::Composed("sealed TableStorage forwarding")), ("table_store.rs", ".raw_dataset_append(", 1, WriteProtocol::EphemeralScratch), @@ -845,14 +880,12 @@ durable_calls! { ("table_store.rs", ".create_index_builder(", 5, WriteProtocol::Composed("staged index primitive and RFC 0067 whole-rebuild fold")), ("table_store.rs", ".execute_uncommitted(", 10, WriteProtocol::Composed("staged physical primitive")), ("db/omnigraph/system_column_upgrade.rs", ".commit_staged_detached(", 1, WriteProtocol::Exact("RFC 0067 detached rename-only system-column effect")), - ("db/omnigraph/system_column_upgrade.rs", ".write_text(", 4, WriteProtocol::Exact("RFC 0067 staged and live schema contract source/IR/state")), ("db/omnigraph/system_column_upgrade.rs", ".commit_changes_with_intent_and_expected(", 1, SYSTEM_COLUMNS_V9), ("db/omnigraph/system_column_upgrade.rs", ".dataset()", 1, SYSTEM_COLUMNS_V9), ("exec/merge.rs", ".commit_staged_detached(", 1, WriteProtocol::Exact("RFC 0067 detached merge chain")), ("exec/merge.rs", ".dataset()", 4, WriteProtocol::Exact("RFC 0067 detached merge chain")), ("db/omnigraph/schema_apply.rs", ".commit_staged_create_exact(", 1, SCHEMA_V9), - ("db/omnigraph/schema_apply.rs", ".commit_staged_detached(", 1, WriteProtocol::Exact("RFC 0067 detached schema rewrite")), - ("db/omnigraph/schema_apply.rs", ".delete_prefix(", 1, WriteProtocol::Exact("RFC 0067 reclaim of an unregistered add-type leftover under the schema sentinel")), + ("db/omnigraph/schema_apply.rs", ".commit_staged_detached(", 2, WriteProtocol::Exact("detached schema rewrite + incompatible original-empty-table retry")), ("db/omnigraph/table_ops.rs", ".commit_staged(", 1, WriteProtocol::Composed("shared merge/Optimize index tail")), ("db/omnigraph/table_ops.rs", ".commit_staged_detached(", 1, WriteProtocol::Exact("RFC 0067 detached index batch")), ("exec/staging.rs", ".commit_staged_detached(", 1, WriteProtocol::Exact("Mutation/Load detached staging (RFC 0067)")), @@ -872,16 +905,9 @@ durable_calls! { ("db/graph_coordinator.rs", ".commit_changes_with_lineage_and_precondition(", 1, WriteProtocol::Exact("lowest manifest publisher gateway")), ("omnigraph-catalog/lib.rs", ".publish_with_precondition(", 1, WriteProtocol::Exact("lowest manifest publisher gateway")), ("db/omnigraph/table_ops.rs", ".commit_updates_with_actor_with_expected(", 2, WriteProtocol::TestOnly), - ("db/omnigraph.rs", ".write_text_if_absent(", 3, WriteProtocol::Composed("bootstrap init claim + strict `_schema.pg` defence + bind-time create-if-absent probe")), - ("db/omnigraph.rs", ".write_text(", 1, WriteProtocol::Bootstrap), - ("db/schema_state.rs", ".write_text(", 2, WriteProtocol::Composed("schema state publication")), - ("db/omnigraph/schema_apply.rs", ".write_text(", 4, WriteProtocol::Exact("RFC 0067 staged and live schema contract source/IR/state")), - ("db/omnigraph.rs", ".delete(", 3, WriteProtocol::Composed("bootstrap init cleanup + claim release + create-if-absent probe removal")), - ("db/schema_state.rs", ".delete(", 3, WriteProtocol::Composed("schema staging cleanup")), - ("db/schema_state.rs", ".rename_text(", 1, WriteProtocol::Composed("schema staging promotion")), + ("db/omnigraph.rs", ".write_text_if_absent(", 2, WriteProtocol::Composed("bootstrap init claim + owned local-root create-if-absent probe")), + ("db/omnigraph.rs", ".delete(", 2, WriteProtocol::Composed("bootstrap init claim release + owned local-root probe removal")), ("db/omnigraph.rs", "GraphCoordinator::init_commit_with_session(", 1, WriteProtocol::Bootstrap), - ("db/omnigraph.rs", "recover_schema_state_files(", 3, WriteProtocol::SchemaContractInstall), - ("db/schema_state.rs", "promote_exact_schema_staging(", 1, WriteProtocol::SchemaContractInstall), ("db/omnigraph/optimize.rs", "compact_files(", 1, WriteProtocol::PhysicalOnly), ("db/omnigraph/optimize.rs", ".commit_staged_detached(", 3, WriteProtocol::Exact("RFC 0067 detached compaction rewrite, index fold and deferred index build")), ("db/omnigraph/optimize.rs", "commit_updates_on_branch_with_expected(", 1, OPTIMIZE_V9), @@ -889,11 +915,7 @@ durable_calls! { ("db/omnigraph/schema_apply.rs", "cleanup_old_versions(", 1, WriteProtocol::Composed("SchemaApply hard-drop GC")), ("db/omnigraph.rs", ".branch_create(", 2, WriteProtocol::NativeRefControl), ("db/omnigraph.rs", ".branch_delete_captured(", 1, WriteProtocol::NativeRefControl), - ("db/omnigraph/schema_apply.rs", ".branch_create(", 1, SCHEMA_V9), - ("db/omnigraph/schema_apply.rs", ".branch_delete(", 1, SCHEMA_V9), - ("db/omnigraph.rs", ".branch_delete(", 2, WriteProtocol::NativeRefControl), ("db/graph_coordinator.rs", ".create_branch(", 1, WriteProtocol::NativeRefControl), - ("db/graph_coordinator.rs", ".delete_branch(", 1, WriteProtocol::NativeRefControl), ("db/graph_coordinator.rs", ".delete_branch_with_expected(", 1, WriteProtocol::NativeRefControl), ("omnigraph-core/branch_control.rs", ".create_branch(", 1, WriteProtocol::Composed("graph/data native refs")), ("omnigraph-core/lance_clone.rs", ".create_branch(", 1, WriteProtocol::Composed("scoped native clone index-origin forwarding")), @@ -2679,7 +2701,6 @@ fn graph_manifest_writer_methods_are_not_public_escape_hatches() { let coordinator_methods = [ "init_commit_with_session", "branch_create", - "branch_delete", "branch_delete_captured", "commit_updates_with_actor", "commit_updates_with_actor_with_expected", From de486a4879bd14212856718dbcc26ac99a78a007 Mon Sep 17 00:00:00 2001 From: azim afroozeh <13484327+azimafroozeh@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:14:50 +0200 Subject: [PATCH 4/5] fix --- crates/omnigraph-gqt/README.md | 31 +++++---- crates/omnigraph-gqt/src/dst_runner.rs | 54 ++++++++++++++++ crates/omnigraph-gqt/src/lib.rs | 32 +++++----- crates/omnigraph-gqt/src/measure.rs | 8 ++- crates/omnigraph-gqt/tests/runner_dispatch.rs | 64 +++++++------------ 5 files changed, 112 insertions(+), 77 deletions(-) diff --git a/crates/omnigraph-gqt/README.md b/crates/omnigraph-gqt/README.md index 41ba243ca..c34a81922 100644 --- a/crates/omnigraph-gqt/README.md +++ b/crates/omnigraph-gqt/README.md @@ -321,10 +321,12 @@ the fault wrapper's own), in both of the engine's realms. The Lance realm: the measuring store is a decorator on the engine's `object_store_seam`, the seam the DST fault decorator uses, so every store the registry builds is wrapped, `__manifest` and table traffic alike. The control -realm: the schema contract (`_schema.pg`, `_schema.ir.json`, -`__schema_state.json` and their `.staging` twins), the init claim and probe, -the legacy `__recovery/` listing and the graph-index artifact go through the -engine's `StorageAdapter`, whose DST store is a second in-memory object store +realm: the init claim and probe, manifest-root preflight and graph-index +artifacts go through the engine's `StorageAdapter`. The classifier also +recognizes legacy schema files (`_schema.pg`, `_schema.ir.json`, +`__schema_state.json` and their `.staging` twins) and `__recovery/` paths. +Format 13 stores the live schema contract inline in `__manifest`, so its I/O +belongs to the Lance realm. The adapter's DST store is a second in-memory object store the registry never builds; the worker wraps the adapter it hands the engine and logs each call as the requests the in-memory adapter makes for it: a text read one `get`, a bounded read one `get` of `0-(max+1)`, a write one `put`, @@ -344,9 +346,8 @@ control realm's limits below are the exceptions). Per group it reports: twice; objects are told apart by their real names, uuids included, and a control object never meets a Lance object of the same name); - `after_publish`, the requests of either realm after the group's last - `__manifest` version put, the publish CAS: the crash window, where a crash - leaves a published operation unfinished (a schema apply's contract write - after its publish is in it); absent when the group published nothing; + `__manifest` version put, the publish CAS; absent when the group published + nothing. A schema apply includes its contract in that same publication; - a count per `_.` class (`manifest_meta.put`, `table_data.get`, `control_schema.head`, …) where the realm is the dataset (`__manifest`, the table, the recovery root) and the kind its Lance @@ -419,17 +420,15 @@ Direct-engine environments record nothing: on a `file` root Lance bypasses the wrapped store for data files, so only the DST in-memory object store sees every request. The control realm's counts are the in-memory adapter's: under DST its store never holds a Lance object, so the engine's probes of a -dataset root through the adapter (init's `__manifest` preflight, schema -apply's leftover-dataset probe) always miss, `head_failed` then `list`, and -the `delete_prefix` reclaim behind the second is never reached (it is logged -as its listing alone, the deletes that follow being as many as it found); an +dataset root through the adapter (such as init's `__manifest` preflight) +always miss, `head_failed` then `list`. `delete_prefix` logs only its listing; +deletes performed inside the adapter are not counted. An `exists` the store refused is `head_failed` whether the head or the list after it failed. An adapter the engine builds for itself instead of using -the handle's (`ensure_no_pending_recovery`, the storage upgrade, the -graph-index load of a historical read) is outside the wrapped one; no gqt -step reaches one today. A contract file probed then read in one step is a repeat -read, `head` and whole-object `get` sharing a key, and so is every load of -the contract after a step's first (a write's revalidation after its capture). +the handle's (the storage upgrade or the graph-index load of a historical +read) is outside the wrapped one; no gqt step reaches one today. An adapter +object probed then read in one step is a repeat read: `head` and whole-object +`get` share a key. Repeated loads in that step also count as repeat reads. Every case measures the same way; nothing in a case declares it, and the invocation takes several case paths or directories, anywhere on disk. `--artifacts ` puts the report and the TSV in that diff --git a/crates/omnigraph-gqt/src/dst_runner.rs b/crates/omnigraph-gqt/src/dst_runner.rs index 15f7252d6..73f1d70dc 100644 --- a/crates/omnigraph-gqt/src/dst_runner.rs +++ b/crates/omnigraph-gqt/src/dst_runner.rs @@ -2097,6 +2097,60 @@ mod action_tests { ); } + #[tokio::test] + async fn concurrent_replay_compares_outcomes_without_background_measurements() { + async fn report( + sessions: &[serde_json::Value], + block: &crate::concurrent::Outcome, + ) -> super::WorkerReport { + super::capture("same-input".into(), async { + crate::record_concurrent_outcome(3, sessions, block); + Ok(()) + }) + .await + .unwrap() + } + + let mut sessions = vec![serde_json::json!({ + "label": "r1", "outcome": "ok", "message": null, "script": null, + })]; + let mut block = crate::concurrent::Outcome { + stuck_at: None, + failure: None, + unattributed: 2, + log: Vec::new(), + wall_ms: 5, + }; + let expected = super::comparable(&report(&sessions, &block).await).unwrap(); + block.unattributed = 3; + block.wall_ms = 9; + block.log.push(serde_json::json!({"wall_ms": 7})); + let measured = super::comparable(&report(&sessions, &block).await).unwrap(); + assert!( + expected == measured, + "background I/O and timing measurements must not change replay equality" + ); + + block.stuck_at = Some(0); + assert_ne!( + expected, + super::comparable(&report(&sessions, &block).await).unwrap() + ); + block.stuck_at = None; + block.failure = Some("session starved".into()); + assert_ne!( + expected, + super::comparable(&report(&sessions, &block).await).unwrap() + ); + block.failure = None; + sessions[0]["outcome"] = "failed".into(); + sessions[0]["message"] = "query failed".into(); + assert_ne!( + expected, + super::comparable(&report(&sessions, &block).await).unwrap() + ); + } + fn measured(environment: &str, seed: &str, step: u64, requests: u64) -> super::MeasureRow { super::MeasureRow { environment: environment.into(), diff --git a/crates/omnigraph-gqt/src/lib.rs b/crates/omnigraph-gqt/src/lib.rs index 1675938a4..88d750945 100644 --- a/crates/omnigraph-gqt/src/lib.rs +++ b/crates/omnigraph-gqt/src/lib.rs @@ -2975,34 +2975,32 @@ async fn run_concurrent_step( }) }) .collect(); + record_concurrent_outcome(step.ordinal, &sessions_evidence, &block); + if let Some(failure) = block.failure { + return Err(fail(failure)); + } + for (op, (outcome, _)) in step.sessions.iter().zip(&results) { + if let Err(message) = outcome { + return Err(fail(format!("session `{}`: {message}", op.label))); + } + } + Ok(()) +} + +fn record_concurrent_outcome(ordinal: usize, sessions: &[Value], block: &concurrent::Outcome) { dst_runner::record( "concurrent_block", serde_json::json!({ - "sessions": sessions_evidence, + "sessions": sessions, "stuck_at": block.stuck_at.map(|at| at + 1), "failure": block.failure, }), ); measure::push_detail(serde_json::json!({ "slot": "concurrent", - "step": step.ordinal, + "step": ordinal, "value": {"grants": block.log, "wall_ms": block.wall_ms, "unattributed": block.unattributed}, })); - dst_runner::observe(|| { - format!( - "concurrent block: stuck_at={:?} failure={:?} unattributed={}", - block.stuck_at, block.failure, block.unattributed - ) - }); - if let Some(failure) = block.failure { - return Err(fail(failure)); - } - for (op, (outcome, _)) in step.sessions.iter().zip(&results) { - if let Err(message) = outcome { - return Err(fail(format!("session `{}`: {message}", op.label))); - } - } - Ok(()) } /// One session of a block: the statement on its branch through the case diff --git a/crates/omnigraph-gqt/src/measure.rs b/crates/omnigraph-gqt/src/measure.rs index c820708e8..9096321c8 100644 --- a/crates/omnigraph-gqt/src/measure.rs +++ b/crates/omnigraph-gqt/src/measure.rs @@ -6,9 +6,11 @@ //! decorator uses and the GQT worker otherwise leaves empty, so every store //! the engine's registry builds is wrapped: `__manifest` and table traffic, //! spawned tasks and cached handles all pass through one ledger. The control -//! realm: the schema contract, its staging twins, the init claim and probe, -//! the legacy `__recovery/` listing and the graph-index artifact go through -//! the engine's `StorageAdapter`, whose DST store is a second in-memory +//! realm: init claims, capability probes, manifest-root preflights and +//! graph-index artifacts go through the engine's `StorageAdapter`; legacy +//! schema and recovery paths also have control classes. The live schema +//! contract is inline in `__manifest`, in the Lance realm. The adapter's +//! DST store is a second in-memory //! object store the registry never builds; the worker wraps the adapter it //! hands the engine ([`wrap_adapter`], the `control` module) and logs each //! call as the requests that adapter makes for it, under `control_` diff --git a/crates/omnigraph-gqt/tests/runner_dispatch.rs b/crates/omnigraph-gqt/tests/runner_dispatch.rs index 59ae8b2d9..727907cac 100644 --- a/crates/omnigraph-gqt/tests/runner_dispatch.rs +++ b/crates/omnigraph-gqt/tests/runner_dispatch.rs @@ -543,29 +543,11 @@ fn measure_counts_schema_contract_requests_issue_817() { files.sort(); files }; - let mut setup = contract_files("setup", 0); - setup.dedup(); - assert_eq!( - setup, - ["__schema_state.json", "_schema.ir.json", "_schema.pg"] - ); - for step in [1, 2] { - assert_eq!( - contract_files("step", step), - [ - "__schema_state.json", - "__schema_state.json", - "__schema_state.json", - "__schema_state.json", - "__schema_state.json", - "_schema.ir.json", - "_schema.ir.json", - "_schema.ir.json", - "_schema.ir.json", - "_schema.pg", - "_schema.pg", - ], - "insert {step}: capture loads the contract and re-reads the state, revalidation loads it again" + assert!(contract_files("setup", 0).is_empty()); + for step in [1, 2, 3] { + assert!( + contract_files("step", step).is_empty(), + "step {step}: the schema contract is inline in the catalog" ); } let io_counts = |slot: &str, step: u64| -> serde_json::Value { @@ -588,27 +570,27 @@ fn measure_counts_schema_contract_requests_issue_817() { .map(|(class, count)| (class.clone(), count.as_u64().unwrap())) .collect() }; - for step in [1, 2] { + assert_eq!( + control_classes(&io_counts("setup", 0)), + [ + ("control_claim.delete".to_string(), 1), + ("control_claim.put".to_string(), 1), + ("control_manifest.head_failed".to_string(), 2), + ("control_manifest.list".to_string(), 2), + ("control_probe.delete".to_string(), 1), + ("control_probe.put".to_string(), 1), + ], + "setup measures the init claim, capability probe and two manifest preflights" + ); + for step in [1, 2, 3] { let counts = io_counts("step", step); - assert_eq!( - control_classes(&counts), - [ - ("control_schema.get".to_string(), 7), - ("control_schema.head".to_string(), 4), - ], - "insert {step}: the engine's pin is 7 read_text + 4 exists (tests/write_cost.rs)" + assert!( + control_classes(&counts).is_empty(), + "step {step}: the inline contract needs no control-adapter requests" ); assert_eq!( - counts["repeat_reads"], 8, - "insert {step}: 11 contract reads on three files; no Lance object is read twice" + counts["repeat_reads"], 0, + "step {step}: neither the inline contract nor table data needs repeated reads" ); } - assert_eq!( - control_classes(&io_counts("step", 3)), - [ - ("control_schema.get".to_string(), 3), - ("control_schema.head".to_string(), 2), - ], - "a warm query reads the contract once (tests/warm_read_cost.rs)" - ); } From 89a8de6c7a55e9e64c083c2245a5ecd36c7b1a83 Mon Sep 17 00:00:00 2001 From: azim afroozeh <13484327+azimafroozeh@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:42:44 +0200 Subject: [PATCH 5/5] fix --- crates/omnigraph-gqt/README.md | 4 + crates/omnigraph-gqt/src/dst_runner.rs | 186 ++++++++++++++++++++++++- crates/omnigraph-gqt/src/lib.rs | 1 + 3 files changed, 187 insertions(+), 4 deletions(-) diff --git a/crates/omnigraph-gqt/README.md b/crates/omnigraph-gqt/README.md index c34a81922..52b662607 100644 --- a/crates/omnigraph-gqt/README.md +++ b/crates/omnigraph-gqt/README.md @@ -175,6 +175,10 @@ engine spawned) can never be named. The `--- expect` after the block is bare, one `