From b5a12083dc17a4d36a962879e10372a4af68c7a9 Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Thu, 10 Sep 2026 06:27:44 +0000 Subject: [PATCH 1/4] test(storage): preserve permanent publishing and codec assessments --- Cargo.lock | 1 + .../tests/permanent_storage_budgets.rs | 154 ++++++++++++++++++ crates/graphforge-storage/Cargo.toml | 1 + crates/graphforge-storage/src/writer.rs | 65 ++++++++ 4 files changed, 221 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index bd6b26114..aae675b6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2496,6 +2496,7 @@ dependencies = [ "unicode-normalization", "uuid", "wait-timeout", + "zstd", ] [[package]] diff --git a/crates/graphforge-api/tests/permanent_storage_budgets.rs b/crates/graphforge-api/tests/permanent_storage_budgets.rs index 29caf64f4..74e911f40 100644 --- a/crates/graphforge-api/tests/permanent_storage_budgets.rs +++ b/crates/graphforge-api/tests/permanent_storage_budgets.rs @@ -2496,3 +2496,157 @@ fn exploratory_coalesced_routes_reject_oversized_encoding_before_publication() { let reopened = GraphForge::new(source.to_str()).unwrap(); verify_graph(&reopened, fixture, &nodes, &[]); } + +fn publishing_parquet_inventory(source: &Path) -> Value { + let selected = graphforge_storage::resolve_project_generation(source).unwrap(); + let inventory = selected.graph_files_inventory().unwrap().unwrap(); + let generation_owned = selected.declared_graph_files_inventory().unwrap().is_some(); + let mut files = Vec::new(); + let mut payload = 0_u64; + let mut allocated = 0_u64; + for entry in inventory + .files + .iter() + .filter(|entry| entry.relative_path.ends_with(".parquet")) + { + let path = if generation_owned { + selected.graph_tree_root().join(&entry.relative_path) + } else { + graphforge_storage::graph_object_path(source, &entry.content_sha256).unwrap() + }; + let input = File::open(&path).unwrap(); + let physical = graphforge_filesystem::file_space_usage(&input) + .unwrap() + .allocated_bytes; + let reader = ParquetRecordBatchReaderBuilder::try_new(input).unwrap(); + let groups = reader.metadata().row_groups().iter().map(|group| { + json!({"rows":group.num_rows(),"columns":group.columns().iter().map(|column| { + json!({"path":column.column_path().string(),"codec":format!("{:?}", column.compression()), + "encodings":format!("{:?}", column.encodings().collect::>())}) + }).collect::>()}) + }).collect::>(); + let edge_order = if reader.schema().index_of("edge_id").is_ok() { + let ids = reader + .build() + .unwrap() + .flat_map(|batch| { + let batch = batch.unwrap(); + batch + .column_by_name("edge_id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect::>(); + json!({"rows":ids.len(), "first":ids.first(), "last":ids.last(), + "first_inversion":ids.windows(2).position(|pair| pair[0] >= pair[1]), + "prefix":ids.iter().take(12).collect::>()}) + } else { + Value::Null + }; + payload += entry.byte_length; + allocated += physical; + files.push(json!({"path":entry.relative_path,"bytes":entry.byte_length, + "allocated_bytes":physical,"row_groups":groups, "edge_id_order":edge_order})); + } + json!({"ownership":if generation_owned { "generation_graph_tree" } else { "cas" }, "parquet_bytes":payload,"parquet_allocated_bytes":allocated,"files":files}) +} + +#[test] +fn permanent_publishing_policy_construction_mutation_and_compaction() { + use graphforge_storage::{ + GraphDeltaCompactionLimits, GraphDeltaCompactionRequest, ProjectRetentionLimits, + ProjectRetentionPolicy, + }; + let root = tempfile::tempdir().unwrap(); + let source = root.path().join("source"); + let fixture = Fixture { + name: "publishing_policy", + nodes: 1025, + edges: 4097, + routes: 2, + identifiers: Identifiers::Random, + properties: true, + adjacency: false, + heterogeneous: true, + }; + let (mut nodes, edges) = rows(fixture); + construct(&source, fixture, &nodes, &edges); + let construction = publishing_parquet_inventory(&source); + let graph = GraphForge::new(source.to_str()).unwrap(); + let before = verify_graph(&graph, fixture, &nodes, &edges); + let started = Instant::now(); + graph + .execute("MATCH (n) WHERE n.score IS NOT NULL SET n.score = n.score + 1") + .unwrap(); + let mutation_ns = started.elapsed().as_nanos(); + for node in &mut nodes { + node.2 = node.2.map(|score| score + 1); + } + let after = verify_graph(&graph, fixture, &nodes, &edges); + assert_ne!(before, after); + drop(graph); + let mutation = publishing_parquet_inventory(&source); + let portable = root.path().join("mutation-portable"); + std::fs::create_dir(&portable).unwrap(); + round_trip(&portable, &source, fixture, &nodes, &edges); + + println!( + "PUBLISHING_PRE_DELTA {}", + json!({"construction":construction, "mutation":mutation}) + ); + use graphforge_api::{ + COMPOSITE_TRANSACTION_CONTRACT_VERSION, CompositeGraphMutation, + CompositeKnowledgeParticipants, CompositeTransactionRequest, PropValue, WriteContext, + }; + let graph = GraphForge::new(source.to_str()).unwrap(); + graph + .publish_composite_transaction(CompositeTransactionRequest { + contract_version: COMPOSITE_TRANSACTION_CONTRACT_VERSION, + context: WriteContext { + operation_uuid: OperationId(Uuid::now_v7()), + actor_uuid: None, + }, + graph_mutations: vec![CompositeGraphMutation::SetNodeProperty { + node_uuid: nodes[0].0, + property: "score".into(), + value: PropValue::Int(123), + }], + knowledge: CompositeKnowledgeParticipants::default(), + }) + .unwrap(); + drop(graph); + nodes[0].2 = Some(123); + let graph = GraphForge::new(source.to_str()).unwrap(); + verify_graph(&graph, fixture, &nodes, &edges); + let started = Instant::now(); + let report = graph + .compact_graph_delta( + &GraphDeltaCompactionRequest { + transaction_uuid: Uuid::from_u128(121305), + generation_uuid: Uuid::from_u128(121306), + through_run_sequence: None, + limits: GraphDeltaCompactionLimits::default(), + cleanup_after_commit: false, + cleanup_policy: ProjectRetentionPolicy::default(), + cleanup_limits: ProjectRetentionLimits::default(), + }, + None, + ) + .unwrap(); + let compaction_ns = started.elapsed().as_nanos(); + drop(graph); + let compaction = publishing_parquet_inventory(&source); + let portable = root.path().join("compaction-portable"); + std::fs::create_dir(&portable).unwrap(); + round_trip(&portable, &source, fixture, &nodes, &edges); + println!( + "PERMANENT_PUBLISHING_POLICY {}", + json!({"construction":construction, + "mutation":mutation,"compaction":compaction,"mutation_ns":mutation_ns, + "compaction_ns":compaction_ns,"compaction_report":format!("{report:?}")}) + ); +} diff --git a/crates/graphforge-storage/Cargo.toml b/crates/graphforge-storage/Cargo.toml index 3e7bfa5fe..187ef76b7 100644 --- a/crates/graphforge-storage/Cargo.toml +++ b/crates/graphforge-storage/Cargo.toml @@ -49,6 +49,7 @@ rustix = { version = "1.1", features = ["fs"] } named-lock = "0.4.1" [dev-dependencies] +zstd = { version = "=0.13.3", default-features = false } divan = { workspace = true } wait-timeout = "0.2" diff --git a/crates/graphforge-storage/src/writer.rs b/crates/graphforge-storage/src/writer.rs index 95606e65b..f8234e9ea 100644 --- a/crates/graphforge-storage/src/writer.rs +++ b/crates/graphforge-storage/src/writer.rs @@ -9657,4 +9657,69 @@ mod tests { let declared = Field::new("declared", DataType::UInt64, false); assert!(decode_value(&wrong_dynamic, &declared, 0).is_err()); } + + #[test] + fn replay_zstd_context_memory_assessment() { + assert_eq!( + zstd::zstd_safe::version_string(), + "1.5.7", + "review the codec memory bound when upgrading Zstd" + ); + let decompressor = zstd::zstd_safe::DCtx::create(); + let mut active_decoder = zstd::zstd_safe::DCtx::create(); + let mut reused = zstd::bulk::Compressor::new(1).unwrap(); + let empty_compressor_bytes = reused.context_mut().sizeof(); + let mut measurements = Vec::new(); + let mut maximum_source = 0_usize; + for size in [ + 0, 1, 7, 16, 128, 512, 513, 1024, 4096, 16384, 16385, 32768, 32769, 65536, 131072, + 262144, 1048576, 2097152, 0, 1, 16384, 513, 2097152, 7, 65536, + ] { + let input = (0..size) + .map(|index| { + // Incompressible-looking deterministic data; workspace also checked + // with a reusable context that has retained earlier allocations. + let mut value = index as u64 + 0x9e37_79b9_7f4a_7c15; + value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + (value ^ (value >> 31)).to_le_bytes()[0] + }) + .collect::>(); + let mut fresh = zstd::bulk::Compressor::new(1).unwrap(); + let compressed = fresh.compress(&input).unwrap(); + assert_eq!(zstd::bulk::decompress(&compressed, size).unwrap(), input); + let mut decoded = Vec::with_capacity(size); + active_decoder + .decompress(&mut decoded, &compressed) + .unwrap(); + assert_eq!(decoded, input); + assert!(active_decoder.sizeof() <= 128 * 1024); + let reused_output = reused.compress(&input).unwrap(); + assert_eq!(compressed, reused_output); + // Pinned Zstd1 fast strategy: fixed contexts/workspace, at most + // 32768 hash entries, and B + 11 * floor(B/4) token storage. + // Retained state is bounded by the lifetime maximum source size. + // The dormant DCtx is part of this encoder only, not an input reader. + maximum_source = maximum_source.max(size); + let block = maximum_source.min(128 * 1024); + let hash = maximum_source.max(64).next_power_of_two().min(16384) * 8; + let envelope = 128 * 1024 + hash + block + 11 * (block / 4); + assert!(fresh.context_mut().sizeof() + decompressor.sizeof() <= envelope); + assert!(reused.context_mut().sizeof() + decompressor.sizeof() <= envelope); + + measurements.push(serde_json::json!({"source_bytes":size, "maximum_source_bytes":maximum_source, "sampled_envelope":envelope, + "compressed_len":compressed.len(), "compressed_capacity":compressed.capacity(), + "fresh_compressor_bytes":fresh.context_mut().sizeof(), + "reused_compressor_bytes":reused.context_mut().sizeof(), + "active_decoder_bytes":active_decoder.sizeof(), "decoded_capacity":decoded.capacity()})); + } + println!( + "REPLAY_ZSTD_MEMORY {}", + serde_json::json!({ + "zstd_version":zstd::zstd_safe::version_string(), + "empty_compressor_bytes":empty_compressor_bytes, + "empty_decompressor_bytes":decompressor.sizeof(), + "measurements":measurements}) + ); + } } From 60ffdca983ed1ad4b2acd4edfe2605f3e85d6e7e Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Thu, 10 Sep 2026 06:36:39 +0000 Subject: [PATCH 2/4] test(storage): compare permanent codec profiles before changing defaults --- .../tests/permanent_storage_budgets.rs | 240 +++++++++++++++++- 1 file changed, 239 insertions(+), 1 deletion(-) diff --git a/crates/graphforge-api/tests/permanent_storage_budgets.rs b/crates/graphforge-api/tests/permanent_storage_budgets.rs index 74e911f40..4d67468fd 100644 --- a/crates/graphforge-api/tests/permanent_storage_budgets.rs +++ b/crates/graphforge-api/tests/permanent_storage_budgets.rs @@ -2547,10 +2547,25 @@ fn publishing_parquet_inventory(source: &Path) -> Value { } else { Value::Null }; + let codec_pairs = if std::env::var_os("GF_PARQUET_CODEC_PAIRS").is_some() { + let reader = + ParquetRecordBatchReaderBuilder::try_new(File::open(&path).unwrap()).unwrap(); + let schema = reader.schema().clone(); + let batches = reader + .build() + .unwrap() + .collect::, _>>() + .unwrap(); + Some(paired_codec_profiles( + &arrow::compute::concat_batches(&schema, &batches).unwrap(), + )) + } else { + None + }; payload += entry.byte_length; allocated += physical; files.push(json!({"path":entry.relative_path,"bytes":entry.byte_length, - "allocated_bytes":physical,"row_groups":groups, "edge_id_order":edge_order})); + "allocated_bytes":physical,"row_groups":groups, "edge_id_order":edge_order,"codec_pairs":codec_pairs})); } json!({"ownership":if generation_owned { "generation_graph_tree" } else { "cas" }, "parquet_bytes":payload,"parquet_allocated_bytes":allocated,"files":files}) } @@ -2650,3 +2665,226 @@ fn permanent_publishing_policy_construction_mutation_and_compaction() { "compaction_ns":compaction_ns,"compaction_report":format!("{report:?}")}) ); } + +// Codec-only comparisons keep schema, dictionary policy, row groups and write +// batches identical within each pair. Whole publishing costs are measured by +// the public fixture separately; ArrowWriter::memory_size excludes native Zstd. +fn paired_codec_profiles(batch: &RecordBatch) -> Value { + let mut profiles = Vec::new(); + for (name, dictionary, row_group_rows, current) in [ + ( + "construction", + true, + 1_048_576, + Compression::ZSTD(ZstdLevel::try_new(1).unwrap()), + ), + ("canonical_staging", true, 65_536, Compression::UNCOMPRESSED), + ("replay", false, 8_192, Compression::UNCOMPRESSED), + ( + "other_permanent", + true, + 1_048_576, + Compression::UNCOMPRESSED, + ), + ] { + let mut outputs = Vec::new(); + let mut retained = Vec::new(); + let mut allocated_peak = 0_u64; + for codec in [current, Compression::ZSTD(ZstdLevel::try_new(1).unwrap())] { + let file = tempfile::NamedTempFile::new().unwrap(); + let properties = WriterProperties::builder() + .set_compression(codec) + .set_dictionary_enabled(dictionary) + .set_max_row_group_row_count(Some(row_group_rows)) + .build(); + let start = Instant::now(); + let mut writer = + ArrowWriter::try_new(file.reopen().unwrap(), batch.schema(), Some(properties)) + .unwrap(); + let mut arrow_writer_peak = 0; + for offset in (0..batch.num_rows()).step_by(127) { + writer + .write(&batch.slice(offset, 127.min(batch.num_rows() - offset))) + .unwrap(); + arrow_writer_peak = arrow_writer_peak.max(writer.memory_size()); + } + writer.close().unwrap(); + let encode_ns = start.elapsed().as_nanos(); + let bytes = file.as_file().metadata().unwrap().len(); + let allocated = graphforge_filesystem::file_space_usage(file.as_file()) + .unwrap() + .allocated_bytes; + allocated_peak += allocated; + let start = Instant::now(); + let reader = ParquetRecordBatchReaderBuilder::try_new(file.reopen().unwrap()).unwrap(); + assert_eq!(reader.schema().as_ref(), batch.schema().as_ref()); + let metadata = reader.metadata(); + assert_eq!( + metadata.num_row_groups(), + batch.num_rows().div_ceil(row_group_rows) + ); + for group in metadata.row_groups() { + assert!(group.num_rows() <= row_group_rows as i64); + for column in group.columns() { + assert_eq!(column.compression(), codec); + if !dictionary { + assert!( + !column.encodings().any( + |encoding| encoding == parquet::basic::Encoding::RLE_DICTIONARY + ) + ); + } + } + } + let decoded = reader + .with_batch_size(127) + .build() + .unwrap() + .collect::, _>>() + .unwrap(); + let decoded = arrow::compute::concat_batches(&batch.schema(), &decoded).unwrap(); + let decode_ns = start.elapsed().as_nanos(); + assert_eq!(decoded, *batch); + outputs.push(json!({"codec":format!("{codec:?}"),"bytes":bytes,"allocated_bytes":allocated, + "encode_ns":encode_ns,"decode_ns":decode_ns,"arrow_writer_peak_bytes":arrow_writer_peak})); + retained.push(file); + } + profiles.push(json!({"profile":name,"dictionary":dictionary,"row_group_rows":row_group_rows, + "write_batch_rows":127,"outputs_current_candidate":outputs,"pair_temporary_allocated_peak_bytes":allocated_peak})); + } + json!({"rows":batch.num_rows(),"arrow_input_bytes":batch.get_array_memory_size(),"profiles":profiles}) +} + +#[test] +fn permanent_codec_pairs_cover_wide_nullable_and_full_width_values() { + use arrow::array::{BooleanArray, Float64Array, StructArray, UInt8Array, UInt64Array}; + let rows = 257; + let mut fields = vec![ + Field::new("uuid", DataType::FixedSizeBinary(16), false), + Field::new("ordinal", DataType::UInt64, false), + ]; + let mut arrays = vec![ + uuids((0..rows).map(|row| id(17, row, true))), + Arc::new(UInt64Array::from( + (0..rows) + .map(|row| u64::MAX - row as u64) + .collect::>(), + )) as ArrayRef, + ]; + for column in 0..32 { + fields.push(Field::new( + format!("nullable_{column}"), + DataType::Int64, + true, + )); + arrays.push(Arc::new(Int64Array::from( + (0..rows) + .map(|row| (!row.is_multiple_of(3)).then_some((row * 37 + column) as i64)) + .collect::>(), + ))); + } + let strings = (0..rows) + .map(|row| { + if row.is_multiple_of(3) { + None + } else { + let length = if row == 1 { 256 * 1024 } else { 1024 }; + Some( + (0..length) + .map(|index| char::from(b'!' + ((index * 31 + row * 17) % 90) as u8)) + .collect::(), + ) + } + }) + .collect::>(); + fields.push(Field::new("large_nullable_text", DataType::Utf8, true)); + arrays.push(Arc::new(StringArray::from(strings))); + let tagged_fields: arrow::datatypes::Fields = vec![ + Field::new("tag", DataType::UInt8, false), + Field::new("int", DataType::Int64, true), + Field::new("float", DataType::Float64, true), + Field::new("str", DataType::Utf8, true), + Field::new("bool", DataType::Boolean, true), + ] + .into(); + let tagged = StructArray::new( + tagged_fields.clone(), + vec![ + Arc::new(UInt8Array::from( + (0..rows).map(|row| (row % 4) as u8).collect::>(), + )), + Arc::new(Int64Array::from( + (0..rows) + .map(|row| (row % 4 == 0).then_some(row as i64)) + .collect::>(), + )), + Arc::new(Float64Array::from( + (0..rows) + .map(|row| (row % 4 == 1).then_some(row as f64 / 3.0)) + .collect::>(), + )), + Arc::new(StringArray::from( + (0..rows) + .map(|row| (row % 4 == 2).then_some("tagged")) + .collect::>(), + )), + Arc::new(BooleanArray::from( + (0..rows) + .map(|row| (row % 4 == 3).then_some(true)) + .collect::>(), + )), + ], + None, + ); + fields.push(Field::new("tagged", DataType::Struct(tagged_fields), false)); + arrays.push(Arc::new(tagged)); + let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), arrays).unwrap(); + let small = RecordBatch::try_new( + batch.schema(), + batch + .columns() + .iter() + .map(|column| { + arrow::compute::take( + column.as_ref(), + &arrow::array::UInt32Array::from(vec![0]), + None, + ) + .unwrap() + }) + .collect(), + ) + .unwrap(); + let narrow_rows = 65_537; + let narrow = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("uuid", DataType::FixedSizeBinary(16), false), + Field::new("ordinal", DataType::UInt64, false), + ])), + vec![ + uuids((0..narrow_rows).map(|row| id(19, row, true))), + Arc::new(UInt64Array::from( + (0..narrow_rows) + .map(|row| u64::MAX - row as u64) + .collect::>(), + )), + ], + ) + .unwrap(); + for selected in [small, batch, narrow] { + let result = paired_codec_profiles(&selected); + for profile in result["profiles"].as_array().unwrap() { + assert!( + profile["pair_temporary_allocated_peak_bytes"] + .as_u64() + .unwrap() + <= 8 * 1024 * 1024 + ); + for output in profile["outputs_current_candidate"].as_array().unwrap() { + assert!(output["arrow_writer_peak_bytes"].as_u64().unwrap() <= 8 * 1024 * 1024); + assert!(output["bytes"].as_u64().unwrap() <= 2 * 1024 * 1024); + } + } + println!("PERMANENT_CODEC_PAIRS {result}"); + } +} From c16397d2cd7aeb01411f5cbb51c433154d3c3ef9 Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Thu, 10 Sep 2026 07:49:23 +0000 Subject: [PATCH 3/4] fix(storage): share permanent Parquet policy with admitted replay codecs --- cargo-bazel-lock.json | 9 +- crates/graphforge-api/BUILD.bazel | 6 +- .../graphforge-api/src/belief_projection.rs | 3 + crates/graphforge-api/src/checkpoints.rs | 16 + crates/graphforge-api/src/knowledge.rs | 9 +- crates/graphforge-api/src/lib.rs | 2 + .../src/permanent_parquet_test_support.rs | 88 +++ crates/graphforge-api/src/provenance.rs | 10 +- crates/graphforge-api/src/search_index.rs | 3 + .../tests/multi_ontology_certification.rs | 7 + .../tests/permanent_storage_budgets.rs | 62 +- .../src/graph_construction.rs | 2 +- .../src/graph_construction_encoding.rs | 12 +- .../src/graph_delta_journal.rs | 12 +- .../src/graph_projection.rs | 7 +- crates/graphforge-storage/src/lib.rs | 3 + .../src/permanent_parquet.rs | 337 ++++++++ .../src/project_checkpoints.rs | 3 +- .../src/property_overlay.rs | 516 ++++++++++-- .../src/runtime_entity_labels.rs | 9 +- .../src/semantic_bindings.rs | 4 +- crates/graphforge-storage/src/staging.rs | 7 +- crates/graphforge-storage/src/vector_store.rs | 8 +- crates/graphforge-storage/src/writer.rs | 529 +++++++++++-- .../tests/graph_delta_journal.rs | 34 +- .../permanent-storage-assessment.md | 118 +++ .../permanent-parquet-1213-baseline.json | 735 ++++++++++++++++++ .../drift/cargo_feature_fingerprint.json | 11 +- 28 files changed, 2407 insertions(+), 155 deletions(-) create mode 100644 crates/graphforge-api/src/permanent_parquet_test_support.rs create mode 100644 crates/graphforge-storage/src/permanent_parquet.rs create mode 100644 docs/development/evidence/permanent-parquet-1213-baseline.json diff --git a/cargo-bazel-lock.json b/cargo-bazel-lock.json index dcdf6c13e..0c1fe7764 100644 --- a/cargo-bazel-lock.json +++ b/cargo-bazel-lock.json @@ -1,5 +1,5 @@ { - "checksum": "6a40a06e42157027df4fb5b43e623e477cd821b17bd6a2670363a11b508a094b", + "checksum": "594b6ba3c63b9701d278637b815cac9903f9c34e2906b60fd6018eb89b58575b", "crates": { "adler2 2.0.1": { "name": "adler2", @@ -14087,6 +14087,10 @@ { "id": "wait-timeout 0.2.1", "target": "wait_timeout" + }, + { + "id": "zstd 0.13.3", + "target": "zstd" } ], "selects": {} @@ -34676,7 +34680,8 @@ "codspeed-divan-compat 5.0.1", "cucumber 0.21.1", "insta 1.48.0", - "wait-timeout 0.2.1" + "wait-timeout 0.2.1", + "zstd 0.13.3" ], "unused_patches": [] } diff --git a/crates/graphforge-api/BUILD.bazel b/crates/graphforge-api/BUILD.bazel index 26279a77b..402534013 100644 --- a/crates/graphforge-api/BUILD.bazel +++ b/crates/graphforge-api/BUILD.bazel @@ -476,7 +476,11 @@ gf_rust_integration_test( gf_rust_integration_test( name = "multi_ontology_certification", - srcs = ["tests/multi_ontology_certification.rs"], + srcs = [ + "tests/multi_ontology_certification.rs", + "src/permanent_parquet_test_support.rs", + ], + crate_root = "tests/multi_ontology_certification.rs", crate = ":graphforge_api", data = _API_TEST_DATA + ["//tests/fixtures/multi-ontology-v1:certification_files"], rustc_env = { diff --git a/crates/graphforge-api/src/belief_projection.rs b/crates/graphforge-api/src/belief_projection.rs index ff5c9cbb8..93f5d66af 100644 --- a/crates/graphforge-api/src/belief_projection.rs +++ b/crates/graphforge-api/src/belief_projection.rs @@ -1708,6 +1708,9 @@ mod tests { policy: policy.clone(), }) .unwrap(); + crate::permanent_parquet_test_support::assert_file( + &projection.graph.dir.join("topology/nodes.parquet"), + ); assert_ne!(projection.source_generation_uuid(), Uuid::nil()); assert!(!projection.policy_bytes().is_empty()); assert_ne!(projection.policy_fingerprint(), [0; 32]); diff --git a/crates/graphforge-api/src/checkpoints.rs b/crates/graphforge-api/src/checkpoints.rs index 6f29b220f..f7b11932a 100644 --- a/crates/graphforge-api/src/checkpoints.rs +++ b/crates/graphforge-api/src/checkpoints.rs @@ -3053,6 +3053,22 @@ mod tests { actor_uuid: None, }) .unwrap_or_else(|error| panic!("{shape}: {error:?}")); + crate::permanent_parquet_test_support::assert_participants( + directory.path(), + graphforge_storage::WORKSPACE_CAPABILITY_ID, + ); + if matches!(shape, "knowledge" | "epistemic") { + crate::permanent_parquet_test_support::assert_participants( + directory.path(), + "knowledge", + ); + } + if shape == "epistemic" { + crate::permanent_parquet_test_support::assert_participants( + directory.path(), + "epistemic", + ); + } drop(graph); let reopened = GraphForge::new(Some(directory.path().to_str().unwrap())).unwrap(); diff --git a/crates/graphforge-api/src/knowledge.rs b/crates/graphforge-api/src/knowledge.rs index 9f17da3ff..c959a8a41 100644 --- a/crates/graphforge-api/src/knowledge.rs +++ b/crates/graphforge-api/src/knowledge.rs @@ -2984,8 +2984,12 @@ pub(crate) fn snapshot_to_participant( } fn write_parquet(batch: &RecordBatch, schema: &SchemaRef) -> Result, GfError> { - let mut writer = ArrowWriter::try_new(Vec::new(), Arc::clone(schema), None) - .map_err(|error| GfError::Storage(error.to_string()))?; + let mut writer = ArrowWriter::try_new( + Vec::new(), + Arc::clone(schema), + Some(graphforge_storage::permanent_parquet::writer_properties().build()), + ) + .map_err(|error| GfError::Storage(error.to_string()))?; writer .write(batch) .map_err(|error| GfError::Storage(error.to_string()))?; @@ -3729,6 +3733,7 @@ mod tests { let created = graph.create_assertion(request.clone()).unwrap(); assert_eq!(created.stats.rows_produced, 1); + crate::permanent_parquet_test_support::assert_participants(root.path(), "knowledge"); let cancelled = CancellationToken::new(); cancelled.cancel(); assert_eq!( diff --git a/crates/graphforge-api/src/lib.rs b/crates/graphforge-api/src/lib.rs index 0d5faa1c7..ab52e90b2 100644 --- a/crates/graphforge-api/src/lib.rs +++ b/crates/graphforge-api/src/lib.rs @@ -110,6 +110,8 @@ mod multi_ontology; mod mutation_transaction; #[cfg(test)] mod mutation_transaction_fault_tests; +#[cfg(test)] +mod permanent_parquet_test_support; pub use multi_ontology::{ ActivationProfileChangeRequest, BridgeAdoptionRequest, BridgeCandidate, BridgeDeleteRequest, BridgeUpdateRequest, CompositionValidationReceipt, ModuleAdoptionRequest, ModuleCandidate, diff --git a/crates/graphforge-api/src/permanent_parquet_test_support.rs b/crates/graphforge-api/src/permanent_parquet_test_support.rs new file mode 100644 index 000000000..c516bb6a9 --- /dev/null +++ b/crates/graphforge-api/src/permanent_parquet_test_support.rs @@ -0,0 +1,88 @@ +//! Inspect authenticated published payloads in existing public lifecycle tests. + +use std::path::Path; + +use graphforge_storage::ResolvedProjectGeneration; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use parquet::file::metadata::ParquetMetaData; + +fn assert_metadata(metadata: &ParquetMetaData, name: &str) -> usize { + let mut columns = 0; + for group in metadata.row_groups() { + assert!( + group.num_rows() <= 1_048_576, + "{name}: default row-group ceiling" + ); + for column in group.columns() { + assert!( + matches!(column.compression(), parquet::basic::Compression::ZSTD(_)), + "{name}: permanent column {} lost Zstd", + column.column_path() + ); + columns += 1; + } + } + columns +} + +pub(crate) fn assert_file(path: &Path) { + let reader = + ParquetRecordBatchReaderBuilder::try_new(std::fs::File::open(path).unwrap()).unwrap(); + assert!(assert_metadata(reader.metadata(), &path.display().to_string()) > 0); +} + +pub(crate) fn assert_graph(generation: &ResolvedProjectGeneration) { + let inventory = generation.graph_files_inventory().unwrap().unwrap(); + let owned = generation + .declared_graph_files_inventory() + .unwrap() + .is_some(); + let mut columns = 0; + for entry in inventory + .files + .iter() + .filter(|entry| entry.relative_path.ends_with(".parquet")) + { + let path = if owned { + generation.graph_tree_root().join(&entry.relative_path) + } else { + graphforge_storage::graph_object_path( + generation.container_root(), + &entry.content_sha256, + ) + .unwrap() + }; + let reader = + ParquetRecordBatchReaderBuilder::try_new(std::fs::File::open(path).unwrap()).unwrap(); + columns += assert_metadata(reader.metadata(), &entry.relative_path); + } + assert!(columns > 0, "published graph fixture must contain data"); +} + +pub(crate) fn assert_participants(root: &Path, capability: &str) { + let generation = graphforge_storage::resolve_project_generation(root).unwrap(); + let mut columns = 0; + for participant in generation + .participant_snapshots() + .unwrap() + .into_iter() + .filter(|participant| { + participant.capability_id == capability && participant.encoding == "parquet" + }) + { + let reader = + ParquetRecordBatchReaderBuilder::try_new(bytes::Bytes::from(participant.bytes)) + .unwrap(); + columns += assert_metadata(reader.metadata(), &participant.record_family_id); + if participant.record_family_id == "restoration_transition" { + assert_eq!( + reader.metadata().file_metadata().created_by(), + Some("graphforge-restoration-transition/1") + ); + } + } + assert!( + columns > 0, + "{capability}: public fixture must publish Parquet data" + ); +} diff --git a/crates/graphforge-api/src/provenance.rs b/crates/graphforge-api/src/provenance.rs index c48179391..7baf0388d 100644 --- a/crates/graphforge-api/src/provenance.rs +++ b/crates/graphforge-api/src/provenance.rs @@ -280,8 +280,12 @@ fn participant( } fn write_parquet(batch: &RecordBatch, schema: &SchemaRef) -> Result, GfError> { - let mut writer = ArrowWriter::try_new(Vec::new(), Arc::clone(schema), None) - .map_err(|error| GfError::Storage(error.to_string()))?; + let mut writer = ArrowWriter::try_new( + Vec::new(), + Arc::clone(schema), + Some(graphforge_storage::permanent_parquet::writer_properties().build()), + ) + .map_err(|error| GfError::Storage(error.to_string()))?; writer .write(batch) .map_err(|error| GfError::Storage(error.to_string()))?; @@ -551,6 +555,8 @@ mod tests { .unwrap() .is_some() ); + crate::permanent_parquet_test_support::assert_participants(root.path(), "provenance"); + crate::permanent_parquet_test_support::assert_graph(&generation); let ledger = read_ledger(&generation).unwrap(); assert_eq!(ledger.events.len(), 1); assert_eq!(ledger.events[0].event_kind, EventKind::CreateNode); diff --git a/crates/graphforge-api/src/search_index.rs b/crates/graphforge-api/src/search_index.rs index 1d83fea40..cd74e9bd3 100644 --- a/crates/graphforge-api/src/search_index.rs +++ b/crates/graphforge-api/src/search_index.rs @@ -782,6 +782,9 @@ mod tests { .unwrap(); let replaced = current_search_artifact(&graph.dir, &key).unwrap().unwrap(); assert_ne!(replaced.path, first.path); + crate::permanent_parquet_test_support::assert_file( + &replaced.path.join(graphforge_storage::VECTOR_DATA_FILE), + ); let rows = read_vector_snapshot(&replaced.path, 2, VectorStoreLimits::default(), || Ok(())) .unwrap(); assert_eq!(rows.len(), 1); diff --git a/crates/graphforge-api/tests/multi_ontology_certification.rs b/crates/graphforge-api/tests/multi_ontology_certification.rs index d44873455..fb76c2e6f 100644 --- a/crates/graphforge-api/tests/multi_ontology_certification.rs +++ b/crates/graphforge-api/tests/multi_ontology_certification.rs @@ -1,5 +1,9 @@ //! Public Rust certification for the reproducible six-domain M9 fixture. +#[path = "../src/permanent_parquet_test_support.rs"] +#[allow(dead_code)] +mod permanent_parquet_test_support; + use std::collections::BTreeMap; use graphforge_api::{ @@ -343,6 +347,9 @@ fn retained_data_migrates_atomically_and_exact_identity_reopens() { let receipt = graph .migrate_ontology_module(&request, &preview, None) .expect("publish retained-data migration"); + permanent_parquet_test_support::assert_graph( + &graphforge_storage::resolve_project_generation(project.path()).unwrap(), + ); let replay = graph .migrate_ontology_module(&request, &preview, None) .expect("exact idempotent replay"); diff --git a/crates/graphforge-api/tests/permanent_storage_budgets.rs b/crates/graphforge-api/tests/permanent_storage_budgets.rs index 4d67468fd..514a8b479 100644 --- a/crates/graphforge-api/tests/permanent_storage_budgets.rs +++ b/crates/graphforge-api/tests/permanent_storage_budgets.rs @@ -2519,6 +2519,15 @@ fn publishing_parquet_inventory(source: &Path) -> Value { .unwrap() .allocated_bytes; let reader = ParquetRecordBatchReaderBuilder::try_new(input).unwrap(); + for group in reader.metadata().row_groups() { + for column in group.columns() { + assert!( + matches!(column.compression(), parquet::basic::Compression::ZSTD(_)), + "permanent payload lost shared compression: {}", + entry.relative_path + ); + } + } let groups = reader.metadata().row_groups().iter().map(|group| { json!({"rows":group.num_rows(),"columns":group.columns().iter().map(|column| { json!({"path":column.column_path().string(),"codec":format!("{:?}", column.compression()), @@ -2564,7 +2573,7 @@ fn publishing_parquet_inventory(source: &Path) -> Value { }; payload += entry.byte_length; allocated += physical; - files.push(json!({"path":entry.relative_path,"bytes":entry.byte_length, + files.push(json!({"path":entry.relative_path,"sha256":entry.content_sha256,"bytes":entry.byte_length, "allocated_bytes":physical,"row_groups":groups, "edge_id_order":edge_order,"codec_pairs":codec_pairs})); } json!({"ownership":if generation_owned { "generation_graph_tree" } else { "cas" }, "parquet_bytes":payload,"parquet_allocated_bytes":allocated,"files":files}) @@ -2637,6 +2646,11 @@ fn permanent_publishing_policy_construction_mutation_and_compaction() { nodes[0].2 = Some(123); let graph = GraphForge::new(source.to_str()).unwrap(); verify_graph(&graph, fixture, &nodes, &edges); + let before_compaction = graphforge_storage::resolve_project_generation(&source) + .unwrap() + .graph_files_inventory() + .unwrap() + .unwrap(); let started = Instant::now(); let report = graph .compact_graph_delta( @@ -2655,6 +2669,52 @@ fn permanent_publishing_policy_construction_mutation_and_compaction() { let compaction_ns = started.elapsed().as_nanos(); drop(graph); let compaction = publishing_parquet_inventory(&source); + let changed = compaction["files"] + .as_array() + .unwrap() + .iter() + .filter(|file| { + !before_compaction.files.iter().any(|before| { + file["path"].as_str() == Some(before.relative_path.as_str()) + && file["sha256"].as_str() == Some(before.content_sha256.as_str()) + }) + }) + .collect::>(); + assert!( + !changed.is_empty(), + "compaction must encode permanent payloads" + ); + for file in changed { + for group in file["row_groups"].as_array().unwrap() { + assert!(group["rows"].as_u64().unwrap() <= 8192); + for column in group["columns"].as_array().unwrap() { + assert!( + !column["encodings"] + .as_str() + .unwrap() + .contains("RLE_DICTIONARY") + ); + } + } + } + + // Deterministic published-payload ceilings; CPU/RSS/OS I/O remain measured + // observations, while replay admission and private-stream limits have + // separate exact boundary regressions. + for (name, inventory, logical_kib, allocated_kib) in [ + ("construction", &construction, 400, 448), + ("mutation", &mutation, 416, 480), + ("compaction", &compaction, 352, 400), + ] { + assert!( + inventory["parquet_bytes"].as_u64().unwrap() <= logical_kib * 1024, + "{name} payload budget" + ); + assert!( + inventory["parquet_allocated_bytes"].as_u64().unwrap() <= allocated_kib * 1024, + "{name} allocation budget" + ); + } let portable = root.path().join("compaction-portable"); std::fs::create_dir(&portable).unwrap(); round_trip(&portable, &source, fixture, &nodes, &edges); diff --git a/crates/graphforge-storage/src/graph_construction.rs b/crates/graphforge-storage/src/graph_construction.rs index 71e6e04bd..d57ac5d3a 100644 --- a/crates/graphforge-storage/src/graph_construction.rs +++ b/crates/graphforge-storage/src/graph_construction.rs @@ -6330,7 +6330,7 @@ fn build_runtime_catalog( root, output, &catalog.to_record_batch(), - Some(crate::graph_construction_encoding::permanent_parquet_properties()?), + Some(crate::permanent_parquet::writer_properties().build()), evidence, )?; record_shape_artifact_install(evidence, &receipt)?; diff --git a/crates/graphforge-storage/src/graph_construction_encoding.rs b/crates/graphforge-storage/src/graph_construction_encoding.rs index a00f02567..44195cbd0 100644 --- a/crates/graphforge-storage/src/graph_construction_encoding.rs +++ b/crates/graphforge-storage/src/graph_construction_encoding.rs @@ -31,8 +31,6 @@ use graphforge_ontology::{QualifiedSymbol, SymbolKind}; use graphforge_value::{EntityTypeId, RelationTypeId, TaggedTypeId}; use parquet::arrow::ArrowWriter; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; -use parquet::basic::{Compression, ZstdLevel}; -use parquet::file::properties::WriterProperties; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use uuid::Uuid; @@ -2149,14 +2147,6 @@ fn write_surrogate_tails( Ok(()) } -/// One codec policy for permanent construction payloads, including the catalog -/// that is shaped privately and then published byte-for-byte. -pub(crate) fn permanent_parquet_properties() -> Result { - Ok(WriterProperties::builder() - .set_compression(Compression::ZSTD(ZstdLevel::try_new(1).map_err(storage)?)) - .build()) -} - fn write_parquet( root: &StableDirectory, relative: &str, @@ -2188,7 +2178,7 @@ fn write_parquet( digest: Sha256::new(), }, batch.schema(), - Some(permanent_parquet_properties()?), + Some(crate::permanent_parquet::writer_properties().build()), ) .map_err(storage)?; writer.write(batch).map_err(storage)?; diff --git a/crates/graphforge-storage/src/graph_delta_journal.rs b/crates/graphforge-storage/src/graph_delta_journal.rs index 40adb29d4..1cdcaf8ab 100644 --- a/crates/graphforge-storage/src/graph_delta_journal.rs +++ b/crates/graphforge-storage/src/graph_delta_journal.rs @@ -601,6 +601,12 @@ pub struct GraphDeltaReplayEvidence { pub estimated_replay_memory_bytes: u64, /// Hard Arrow row bound used by every streaming materialization reader. pub materialization_batch_row_bound: u64, + /// Private IPC bytes written and subsequently read once by the admitted + /// low-budget node decoder strategy. Zero for direct streaming. + pub temporary_decode_stream_bytes: u64, + /// Filesystem allocation of that one live private stream, measured before + /// consumption. This excludes the copied graph workspace and other files. + pub temporary_decode_stream_allocated_bytes: u64, /// Canonical reconstructed fingerprint. pub state_fingerprint: [u8; 32], } @@ -1286,7 +1292,11 @@ pub fn materialize_replayed_graph_tree( if evidence.runs_replayed == 0 { return Ok((open_evidence, evidence)); } - crate::writer::write_replay_overlay_streaming(graph_root, inventory, target, &overlay, limits)?; + let spool = crate::writer::write_replay_overlay_streaming( + graph_root, inventory, target, &overlay, limits, + )?; + evidence.temporary_decode_stream_bytes = spool.bytes; + evidence.temporary_decode_stream_allocated_bytes = spool.allocated_bytes; let deltas = target.join(GRAPH_DELTA_DIR); if deltas.exists() { fs::remove_dir_all(&deltas) diff --git a/crates/graphforge-storage/src/graph_projection.rs b/crates/graphforge-storage/src/graph_projection.rs index 9503becc5..98e0dea21 100644 --- a/crates/graphforge-storage/src/graph_projection.rs +++ b/crates/graphforge-storage/src/graph_projection.rs @@ -703,7 +703,12 @@ pub(crate) fn write_parquet(path: &Path, batch: &RecordBatch) -> Result<(), GfEr .ok_or_else(|| validation("graph parquet target has no parent"))?; fs::create_dir_all(parent).map_err(storage)?; let file = File::create(path).map_err(storage)?; - let mut writer = ArrowWriter::try_new(file, batch.schema(), None).map_err(storage)?; + let mut writer = ArrowWriter::try_new( + file, + batch.schema(), + Some(crate::permanent_parquet::writer_properties().build()), + ) + .map_err(storage)?; writer.write(batch).map_err(storage)?; writer.close().map_err(storage)?; Ok(()) diff --git a/crates/graphforge-storage/src/lib.rs b/crates/graphforge-storage/src/lib.rs index 6a6196300..14eb2246e 100644 --- a/crates/graphforge-storage/src/lib.rs +++ b/crates/graphforge-storage/src/lib.rs @@ -427,6 +427,9 @@ pub use runtime_entity_labels::{ }; pub mod parquet_scan; + +/// Shared encoding defaults for published project Parquet payloads. +pub mod permanent_parquet; pub use parquet_scan::{GraphForgeParquetExec, IoConcurrencyExt, ParquetFragment}; pub mod schemas; diff --git a/crates/graphforge-storage/src/permanent_parquet.rs b/crates/graphforge-storage/src/permanent_parquet.rs new file mode 100644 index 000000000..68d5fc61a --- /dev/null +++ b/crates/graphforge-storage/src/permanent_parquet.rs @@ -0,0 +1,337 @@ +//! Encoding policy for Parquet that becomes a published project payload. +//! +//! Lifecycle owners keep their durability, streaming, row-group and dictionary +//! choices. Private construction streams and external query exports are not +//! permanent project payloads. Resource envelopes here target the pinned +//! Parquet 58 / Zstd 1.5.7 implementation, not process RSS or allocator overhead. + +use arrow::datatypes::Schema; +use graphforge_core::{GfError, ProjectErrorCode}; +use parquet::{ + arrow::ArrowSchemaConverter, + basic::{Compression, ZstdLevel}, + file::properties::{ + EnabledStatistics, WriterProperties, WriterPropertiesBuilder, WriterVersion, + }, +}; + +pub(crate) const PAGE_BYTES: usize = 1024 * 1024; +pub(crate) const PAGE_ROWS: usize = 20_000; + +/// Shared permanent-payload encoding defaults. +/// +/// Callers retain measured path-specific dictionary and row-group settings, +/// and domain metadata such as restoration's `created_by` marker. The builder +/// does not own files, publication leases, recovery or cancellation. +#[must_use] +pub fn writer_properties() -> WriterPropertiesBuilder { + WriterProperties::builder() + .set_writer_version(WriterVersion::PARQUET_1_0) + .set_compression(Compression::ZSTD( + ZstdLevel::try_new(1).expect("Zstd level 1 is valid"), + )) + .set_dictionary_enabled(true) + .set_max_row_group_row_count(Some(1_048_576)) + .set_data_page_size_limit(PAGE_BYTES) + .set_dictionary_page_size_limit(PAGE_BYTES) + .set_data_page_row_count_limit(PAGE_ROWS) + .set_write_batch_size(1024) + .set_statistics_enabled(EnabledStatistics::Page) + .set_write_page_header_statistics(false) + .set_column_index_truncate_length(Some(64)) + .set_statistics_truncate_length(Some(64)) + .set_offset_index_disabled(false) + .set_coerce_types(false) +} + +fn overflow() -> GfError { + GfError::Project { + code: ProjectErrorCode::ResourceLimit, + message: "permanent Parquet memory reservation overflow".into(), + } +} + +pub(crate) fn physical_columns(schema: &Schema) -> Result { + ArrowSchemaConverter::new() + .convert(schema) + .map(|schema| schema.num_columns()) + .map_err(|error| GfError::Storage(error.to_string())) +} + +/// One encoder's native CCtx plus Parquet's dormant DCtx. Fast strategy, no +/// trained dictionary, no workers, and the maximum source within this row group. +/// Fixed space includes the pinned structs, block states, scratch, alignment +/// and default ASAN arena redzones. Runtime sizing regression pins this claim. +pub(crate) fn zstd_encoder_workspace(maximum_source: usize) -> usize { + let block = maximum_source.min(128 * 1024); + let hash = maximum_source.clamp(64, 16_384).next_power_of_two() * 8; + 128 * 1024 + hash + block + 11 * (block / 4) +} + +/// Active DCtx plus the codec's dormant CCtx, independently of page buffers. +pub(crate) const ZSTD_DECODER_WORKSPACE: usize = 104 * 1024; +const ZSTD_DORMANT_CODEC_WORKSPACE: usize = 104 * 1024; + +/// Conservative additional encoder reservation for dictionaries-off replay. +/// Includes every physical leaf's native state, encoded row-group chunks kept +/// until flush, page headers, and source/compressed temporary copies. Existing +/// row/snapshot and metadata reservations remain separate. `active_source_bytes` +/// is the aggregate charged payload of the active row group, not a per-row size. +pub(crate) fn replay_encoder_buffers( + schema: &Schema, + active_source_bytes: usize, + active_rows: usize, +) -> Result { + if active_rows == 0 { + return Ok(0); // ArrowWriter creates column codecs lazily on first write. + } + let columns = physical_columns(schema)?; + let levels = active_rows + .checked_mul(columns) + .and_then(|n| n.checked_mul(16)) + .ok_or_else(overflow)?; + let source = active_source_bytes + .checked_mul(3) + .and_then(|n| n.checked_add(levels)) + .ok_or_else(overflow)?; + let pages = columns + .checked_mul(1 + active_rows / PAGE_ROWS) + .and_then(|n| n.checked_add(source / PAGE_BYTES)) + .ok_or_else(overflow)?; + // compressBound <= source + ceil(source/256) + 64 per page. Current + // truncated page statistics keep each encoded page header below 512 bytes. + let copies = source + .checked_mul(4) + .and_then(|n| n.checked_add(source.div_ceil(256).checked_mul(3)?)) + .and_then(|n| n.checked_add(pages.checked_mul(512 + 3 * 64)?)) + .ok_or_else(overflow)?; + let parquet_schema = ArrowSchemaConverter::new() + .convert(schema) + .map_err(|error| GfError::Storage(error.to_string()))?; + let mut maximum_leaf_source = 0; + for leaf in parquet_schema.columns() { + let width = match leaf.physical_type() { + parquet::basic::Type::BOOLEAN => Some(1_usize), + parquet::basic::Type::INT32 | parquet::basic::Type::FLOAT => Some(4), + parquet::basic::Type::INT64 | parquet::basic::Type::DOUBLE => Some(8), + parquet::basic::Type::INT96 => Some(12), + parquet::basic::Type::FIXED_LEN_BYTE_ARRAY => usize::try_from(leaf.type_length()).ok(), + parquet::basic::Type::BYTE_ARRAY => None, + }; + let leaf_source = if leaf.max_rep_level() == 0 { + width + .map(|width| { + width + .checked_add(16) + .and_then(|bytes| bytes.checked_mul(active_rows)) + .ok_or_else(overflow) + }) + .transpose()? + .unwrap_or(source) + .min(source) + } else { + source + }; + maximum_leaf_source = maximum_leaf_source.max(leaf_source); + } + let active_codec = zstd_encoder_workspace(maximum_leaf_source); + let native = if source < PAGE_BYTES && active_rows < PAGE_ROWS { + // No data page can flush during writes below both page thresholds. + // Row-group close compresses and drops leaf writers sequentially, so + // only one CCtx is active alongside the other dormant codec pairs. + columns + .checked_mul(ZSTD_DORMANT_CODEC_WORKSPACE) + .and_then(|n| n.checked_add(active_codec.saturating_sub(ZSTD_DORMANT_CODEC_WORKSPACE))) + } else { + columns.checked_mul(active_codec) + } + .ok_or_else(overflow)?; + native.checked_add(copies).ok_or_else(overflow) +} + +/// Writer structures and both Arrow/Parquet schema ownership. Payload buffers, +/// native codecs and retained row-group metadata are charged separately. +pub(crate) fn replay_schema_bytes(schema: &Schema) -> Result { + let parquet_schema = ArrowSchemaConverter::new() + .convert(schema) + .map_err(|error| GfError::Storage(error.to_string()))?; + let parquet_metadata = parquet::file::metadata::ParquetMetaData::new( + parquet::file::metadata::FileMetaData::new( + 1, + 0, + None, + None, + std::sync::Arc::new(parquet_schema), + None, + ), + Vec::new(), + ); + let arrow_schema = schema + .fields() + .iter() + .fold(0_usize, |bytes, field| bytes.saturating_add(field.size())) + .saturating_add( + schema + .metadata() + .iter() + .fold(0_usize, |bytes, (key, value)| { + bytes + .saturating_add(64) + .saturating_add(key.capacity()) + .saturating_add(value.capacity()) + }), + ); + arrow_schema + .checked_mul(4) + .and_then(|bytes| bytes.checked_add(parquet_metadata.memory_size())) + .ok_or_else(overflow) +} + +pub(crate) fn replay_writer_structure_bytes(schema: &Schema) -> Result { + // Pinned ArrowColumnWriter is 1464 bytes, including its 1416-byte column + // writer. The per-leaf allowance also covers allocation containers and + // one closing-column index conversion; variable schemas are separate. + let schema_bytes = replay_schema_bytes(schema)?; + physical_columns(schema)? + .checked_mul(8 * 1024) + .and_then(|bytes| bytes.checked_add(64 * 1024)) + .and_then(|bytes| bytes.checked_add(schema_bytes)) + .ok_or_else(overflow) +} + +fn capacity_bound(len: usize, minimum: usize) -> usize { + if len == 0 { + 0 + } else { + len.saturating_mul(2).max(minimum) + } +} + +/// Retained chunk statistics, page indexes and container capacities for a +/// dictionaries-off replay file. Unlike ArrowWriter::memory_size(), this +/// includes already flushed row groups. All arithmetic fails closed. +pub(crate) fn replay_metadata_bytes( + schema: &Schema, + groups: usize, + rows_per_group: usize, + maximum_row_bytes: usize, +) -> Result { + use parquet::file::{ + metadata::{ColumnChunkMetaData, RowGroupMetaData}, + page_index::{column_index::ColumnIndexMetaData, offset_index::OffsetIndexMetaData}, + }; + let parquet_schema = ArrowSchemaConverter::new() + .convert(schema) + .map_err(|error| GfError::Storage(error.to_string()))?; + let mut per_group = 0_usize; + for leaf in parquet_schema.columns() { + let width = match leaf.physical_type() { + parquet::basic::Type::BOOLEAN => Some(1_usize), + parquet::basic::Type::INT32 | parquet::basic::Type::FLOAT => Some(4), + parquet::basic::Type::INT64 | parquet::basic::Type::DOUBLE => Some(8), + parquet::basic::Type::INT96 => Some(12), + parquet::basic::Type::FIXED_LEN_BYTE_ARRAY => usize::try_from(leaf.type_length()).ok(), + parquet::basic::Type::BYTE_ARRAY => None, + }; + let levels = [leaf.max_def_level(), leaf.max_rep_level()] + .into_iter() + .filter(|level| *level > 0) + .map(|level| usize::from(level.unsigned_abs()) + 1) + .sum::(); + let source = if leaf.max_rep_level() == 0 { + width + .unwrap_or(maximum_row_bytes) + .saturating_add(16) + .saturating_mul(rows_per_group) + } else { + maximum_row_bytes + .saturating_mul(3) + .saturating_add(16) + .saturating_mul(rows_per_group) + }; + let pages = 1_usize + .saturating_add(rows_per_group.saturating_sub(1) / PAGE_ROWS) + .saturating_add(source / PAGE_BYTES); + let binary = matches!( + leaf.physical_type(), + parquet::basic::Type::BYTE_ARRAY | parquet::basic::Type::FIXED_LEN_BYTE_ARRAY + ); + // Truncating a maximum string can fail when its prefix cannot be + // incremented. Reserve the original value bound, not an assumed 64B. + let extrema = width.unwrap_or(maximum_row_bytes); + let min_max = pages + .saturating_mul(extrema) + .saturating_mul(2) + .saturating_add(if binary { + pages + .saturating_add(1) + .saturating_mul(2 * size_of::()) + } else { + 0 + }); + let fixed = size_of::() + + size_of::>() + + size_of::>() + + size_of::>(); + let statistics = if binary { + extrema.saturating_mul(2).saturating_add(64) + } else { + 0 + }; + let retained = fixed + .saturating_add(statistics) + .saturating_add(levels.saturating_mul(8)) + .saturating_add(capacity_bound(pages, 8)) // null flags + .saturating_add(capacity_bound(pages, 4).saturating_mul(8)) // null counts + .saturating_add(capacity_bound(pages.saturating_mul(levels), 4).saturating_mul(8)) + .saturating_add(min_max) + .saturating_add(pages.saturating_mul(24)) // exact-sized final PageLocation vector + .saturating_add( + if leaf.physical_type() == parquet::basic::Type::BYTE_ARRAY { + capacity_bound(pages, 4).saturating_mul(8) + } else { + 0 + }, + ); + per_group = per_group.saturating_add(retained); + } + let outer = capacity_bound(groups, 4) + .saturating_mul(size_of::() + 3 * size_of::>()); + per_group + .checked_mul(groups) + .and_then(|bytes| bytes.checked_add(outer)) + .ok_or_else(overflow) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn replay_writer_structural_memory_assessment() { + use parquet::{ + arrow::arrow_writer::{ArrowColumnWriter, ArrowRowGroupWriterFactory}, + column::writer::ColumnWriter, + file::{ + metadata::{ColumnChunkMetaData, RowGroupMetaData}, + page_index::{ + column_index::ColumnIndexMetaData, offset_index::OffsetIndexMetaData, + }, + writer::SerializedFileWriter, + }, + }; + let measurements = serde_json::json!({ + "arrow_column_writer": size_of::(), + "column_writer": size_of::>(), + "row_group_factory": size_of::(), + "file_writer": size_of::>(), + "column_metadata": size_of::(), + "row_group_metadata": size_of::(), + "column_index": size_of::(), + "offset_index": size_of::(), + "properties": size_of::(), + }); + println!("REPLAY_WRITER_STRUCTURES {measurements}"); + assert!(size_of::() + size_of::>() < 8 * 1024); + } +} diff --git a/crates/graphforge-storage/src/project_checkpoints.rs b/crates/graphforge-storage/src/project_checkpoints.rs index d9c9d30bf..11c558e0e 100644 --- a/crates/graphforge-storage/src/project_checkpoints.rs +++ b/crates/graphforge-storage/src/project_checkpoints.rs @@ -16,7 +16,6 @@ use arrow::record_batch::RecordBatch; use graphforge_core::canonical::{CANONICAL_CONTRACT_VERSION, CanonicalDomain, fingerprint}; use graphforge_core::{GfError, ProjectErrorCode}; use parquet::arrow::ArrowWriter; -use parquet::file::properties::WriterProperties; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use unicode_normalization::UnicodeNormalization; @@ -1808,7 +1807,7 @@ fn restoration_participant( RESTORATION_CONTRACT_VERSION, ]))); let batch = RecordBatch::try_new(Arc::clone(&schema), columns).map_err(arrow_error)?; - let properties = WriterProperties::builder() + let properties = crate::permanent_parquet::writer_properties() .set_created_by("graphforge-restoration-transition/1".into()) .build(); let mut writer = diff --git a/crates/graphforge-storage/src/property_overlay.rs b/crates/graphforge-storage/src/property_overlay.rs index 283dc5b82..08901a851 100644 --- a/crates/graphforge-storage/src/property_overlay.rs +++ b/crates/graphforge-storage/src/property_overlay.rs @@ -2430,6 +2430,52 @@ pub fn read_authenticated_property_snapshots_for_inventory( ), GfError, > { + read_property_targets(inventory, kind, route, targets, None) +} + +pub(crate) fn read_replay_property_targets( + inventory: &AuthenticatedPropertyInventory, + kind: PropertyRouteKind, + route: &str, + targets: &BTreeSet<[u8; 16]>, + max_memory_bytes: usize, +) -> Result< + ( + BTreeMap<[u8; 16], PropertySnapshotRow>, + PropertyOverlayMetrics, + ), + GfError, +> { + read_property_targets(inventory, kind, route, targets, Some(max_memory_bytes)) +} + +#[allow( + clippy::too_many_lines, + reason = "authenticated targeted read and its resource accounting share one lifecycle" +)] +fn read_property_targets( + inventory: &AuthenticatedPropertyInventory, + kind: PropertyRouteKind, + route: &str, + targets: &BTreeSet<[u8; 16]>, + replay_budget: Option, +) -> Result< + ( + BTreeMap<[u8; 16], PropertySnapshotRow>, + PropertyOverlayMetrics, + ), + GfError, +> { + let mut limits = PropertyOverlayLimits::default(); + if let Some(bytes) = replay_budget { + limits.max_buffered_bytes = bytes as u64; + limits.max_row_bytes = limits.max_row_bytes.min((bytes as u64 / 4).max(1)); + if bytes < 64 * 1024 { + return Err(replay_decoder_limit( + "property replay authentication buffer exceeds budget", + )); + } + } let mut unresolved = targets.clone(); let mut found = BTreeMap::new(); let mut metrics = PropertyOverlayMetrics::default(); @@ -2474,6 +2520,9 @@ pub fn read_authenticated_property_snapshots_for_inventory( metrics.property_authentication_read_calls = metrics .property_authentication_read_calls .saturating_add(opened.authentication_read_calls); + if let Some(bytes) = replay_budget { + admit_target_footer(&opened.file, fragment.entry.byte_length, bytes)?; + } let builder = open_counted_retained_property_builder(fragment, &opened, Arc::clone(&counts))?; validate_fragment_schema( @@ -2483,17 +2532,41 @@ pub fn read_authenticated_property_snapshots_for_inventory( kind, route, )?; - let page_reservation_bytes = validate_parquet_resource_admission( - builder.metadata(), - PropertyOverlayLimits::default(), - opened.file.as_ref(), - &counts, - None, - )?; + let targeted_batch_rows = admitted_batch_rows(limits); + let page_reservation_bytes = if replay_budget.is_some() { + let admission = parquet_resource_admission( + builder.metadata(), + limits, + opened.file.as_ref(), + &counts, + None, + targeted_batch_rows, + true, + replay_decoder_limit, + )?; + // Outer authority builder, validation builder, and active reader + // may coexist; account metadata independently from page buffers. + admission + .with_codec_bytes + .saturating_add((builder.metadata().memory_size() as u64).saturating_mul(3)) + } else { + validate_parquet_resource_admission( + builder.metadata(), + limits, + opened.file.as_ref(), + &counts, + None, + )? + }; + let admission = TargetReadAdmission { + limits, + page_reservation_bytes, + replay: replay_budget.is_some(), + }; + admission.check(0, found.values().map(snapshot_charge).sum())?; metrics.decoder_page_reservation_bytes = metrics .decoder_page_reservation_bytes .max(page_reservation_bytes); - let targeted_batch_rows = admitted_batch_rows(PropertyOverlayLimits::default()); metrics.row_groups_considered = metrics .row_groups_considered .saturating_add(u64::try_from(builder.metadata().num_row_groups()).unwrap_or(u64::MAX)); @@ -2504,8 +2577,9 @@ pub fn read_authenticated_property_snapshots_for_inventory( &unresolved, &counts, &mut metrics, - page_reservation_bytes, + admission, targeted_batch_rows, + found.values().map(snapshot_charge).sum(), )?; let validation_bytes = counts.bytes.load(Ordering::Relaxed); let validation_read_calls = counts.blocks.load(Ordering::Relaxed); @@ -2520,7 +2594,7 @@ pub fn read_authenticated_property_snapshots_for_inventory( kind, row_groups, batch_rows: targeted_batch_rows, - page_reservation_bytes, + admission, }, &counts, &mut unresolved, @@ -2569,8 +2643,9 @@ fn select_target_row_groups( unresolved: &std::collections::BTreeSet<[u8; 16]>, counts: &Arc, metrics: &mut PropertyOverlayMetrics, - page_reservation_bytes: u64, + admission: TargetReadAdmission, targeted_batch_rows: usize, + retained_bytes: u64, ) -> Result, GfError> { let builder = open_counted_retained_property_builder(fragment, opened, Arc::clone(counts))?; let mut selected_groups = Vec::new(); @@ -2585,7 +2660,7 @@ fn select_target_row_groups( let mut selected = false; for batch in validation { let batch = batch.map_err(authenticated_arrow_error)?; - charge_target_batch(metrics, &batch, page_reservation_bytes)?; + charge_target_batch(metrics, &batch, admission, retained_bytes)?; let uuids = batch .column_by_name(kind.uuid_field()) .and_then(|column| column.as_any().downcast_ref::()) @@ -2645,7 +2720,7 @@ struct TargetDecodeOptions<'a> { kind: PropertyRouteKind, row_groups: Vec, batch_rows: usize, - page_reservation_bytes: u64, + admission: TargetReadAdmission, } fn decode_target_row_groups( @@ -2666,14 +2741,29 @@ fn decode_target_row_groups( .map_err(parquet_error)?; for batch in reader { let batch = batch.map_err(authenticated_arrow_error)?; - charge_target_batch(metrics, &batch, options.page_reservation_bytes)?; + charge_target_batch( + metrics, + &batch, + options.admission, + found.values().map(snapshot_charge).sum(), + )?; let decoded = decode_snapshot_batch(&batch, options.kind.uuid_field())?; if decoded .iter() - .any(|row| snapshot_charge(row) > PropertyOverlayLimits::default().max_row_bytes) + .any(|row| snapshot_charge(row) > options.admission.limits.max_row_bytes) { - return Err(corrupt("property snapshot row exceeds byte limit")); + return Err(options + .admission + .error("property snapshot row exceeds byte limit")); } + options.admission.check( + batch.get_array_memory_size() as u64, + decoded + .iter() + .chain(found.values()) + .map(snapshot_charge) + .sum(), + )?; metrics.decoder_peak_bytes = metrics .decoder_peak_bytes .max(decoded.iter().map(snapshot_charge).sum::()); @@ -2691,33 +2781,76 @@ fn decode_target_row_groups( Ok(()) } +#[derive(Clone, Copy)] +struct TargetReadAdmission { + limits: PropertyOverlayLimits, + page_reservation_bytes: u64, + replay: bool, +} + +impl TargetReadAdmission { + fn error(self, message: &str) -> GfError { + if self.replay { + replay_decoder_limit(message) + } else { + corrupt(message) + } + } + + fn check(self, arrow_bytes: u64, retained_bytes: u64) -> Result { + // Existing non-replay targeted callers retain their original budget. + let retained_bytes = if self.replay { retained_bytes } else { 0 }; + let bytes = self + .page_reservation_bytes + .checked_add(arrow_bytes) + .and_then(|bytes| bytes.checked_add(retained_bytes)) + .ok_or_else(|| self.error("targeted property decode memory overflow"))?; + if bytes > self.limits.max_buffered_bytes { + return Err(self.error("targeted property decode exceeds live-byte budget")); + } + Ok(bytes) + } +} + fn charge_target_batch( metrics: &mut PropertyOverlayMetrics, batch: &RecordBatch, - page_reservation_bytes: u64, + admission: TargetReadAdmission, + retained_bytes: u64, ) -> Result<(), GfError> { - let limits = PropertyOverlayLimits::default(); let arrow_bytes = u64::try_from(batch.get_array_memory_size()).unwrap_or(u64::MAX); - let decoded_reservation = limits + let decoded_reservation = admission + .limits .max_row_bytes .saturating_mul(u64::try_from(batch.num_rows()).unwrap_or(u64::MAX)); - if page_reservation_bytes - .checked_add(arrow_bytes) - .and_then(|bytes| bytes.checked_add(decoded_reservation)) - .is_none_or(|bytes| bytes > limits.max_buffered_bytes) - { - return Err(corrupt("targeted property decode exceeds live-byte budget")); - } + let bytes = admission.check( + arrow_bytes.saturating_add(decoded_reservation), + retained_bytes, + )?; metrics.emitted_batches = metrics.emitted_batches.saturating_add(1); - metrics.decoder_peak_rows = metrics - .decoder_peak_rows - .max(u64::try_from(batch.num_rows()).unwrap_or(u64::MAX)); + metrics.decoder_peak_rows = metrics.decoder_peak_rows.max(batch.num_rows() as u64); metrics.decoder_peak_bytes = metrics.decoder_peak_bytes.max(arrow_bytes); - metrics.peak_buffered_bytes = metrics.peak_buffered_bytes.max( - page_reservation_bytes - .saturating_add(arrow_bytes) - .saturating_add(decoded_reservation), - ); + metrics.peak_buffered_bytes = metrics.peak_buffered_bytes.max(bytes); + Ok(()) +} + +fn admit_target_footer(file: &File, length: u64, budget: usize) -> Result<(), GfError> { + if length < 8 { + return Err(corrupt("property Parquet footer is truncated")); + } + let mut footer = [0_u8; 8]; + if retained_read_at(file, &mut footer, length - 8).map_err(io_error)? != footer.len() + || &footer[4..] != b"PAR1" + { + return Err(corrupt("property Parquet footer is invalid")); + } + let encoded = + u32::from_le_bytes(footer[..4].try_into().expect("four-byte footer length")) as usize; + if encoded > 16 * 1024 * 1024 || encoded.saturating_mul(4).saturating_add(64 * 1024) > budget { + return Err(replay_decoder_limit( + "property Parquet footer exceeds replay budget", + )); + } Ok(()) } @@ -2801,19 +2934,108 @@ fn validate_parquet_resource_admission( counts: &Arc, projected_columns: Option<&BTreeSet>, ) -> Result { + Ok(parquet_resource_admission( + metadata, + limits, + file, + counts, + projected_columns, + admitted_batch_rows(limits), + false, + corrupt, + )? + .decoded_bytes) +} + +struct ParquetDecoderMemory { + decoded_bytes: u64, + with_codec_bytes: u64, +} + +fn replay_decoder_limit(message: &str) -> GfError { + GfError::Project { + code: graphforge_core::ProjectErrorCode::ResourceLimit, + message: message.into(), + } +} + +/// Reserve authenticated page, codec and batch exposure before replay decoding. +/// Reuses the same bounded raw-page parser as property admission. +pub(crate) fn replay_parquet_reader_reservation( + metadata: &parquet::file::metadata::ParquetMetaData, + file: &File, + max_memory_bytes: usize, + batch_rows: usize, +) -> Result { + let limits = PropertyOverlayLimits { + max_buffered_bytes: max_memory_bytes as u64, + ..PropertyOverlayLimits::default() + }; + let admission = parquet_resource_admission( + metadata, + limits, + file, + &Arc::new(ReadCounts::default()), + None, + batch_rows, + true, + replay_decoder_limit, + )?; + let bytes = usize::try_from(admission.with_codec_bytes) + .map_err(|_| replay_decoder_limit("replay decoder reservation overflows"))?; + if bytes > max_memory_bytes { + return Err(replay_decoder_limit( + "replay decoder pages and codecs exceed memory budget", + )); + } + Ok(bytes) +} + +#[allow( + deprecated, + reason = "Parquet 58 raw page sizes are exposed through compact Thrift headers" +)] +#[allow( + clippy::too_many_arguments, + clippy::too_many_lines, + reason = "one authenticated page scan computes legacy and replay resource envelopes" +)] +fn parquet_resource_admission( + metadata: &parquet::file::metadata::ParquetMetaData, + limits: PropertyOverlayLimits, + file: &File, + counts: &Arc, + projected_columns: Option<&BTreeSet>, + batch_rows: usize, + include_codec: bool, + limit_error: fn(&str) -> GfError, +) -> Result { const MAX_PAGE_HEADER_BYTES: usize = 64 * 1024; let max_page_bytes = limits.max_buffered_bytes / 4; if max_page_bytes == 0 { - return Err(corrupt("property page byte budget is too small")); + return Err(limit_error("property page byte budget is too small")); } let mut largest_group_exposure = 0_u64; + let mut column_memory = if include_codec { + vec![0_u64; metadata.file_metadata().schema_descr().num_columns()] + } else { + Vec::new() + }; + let mut group_values = Vec::with_capacity(if include_codec { + metadata.num_row_groups() + } else { + 0 + }); for group in metadata.row_groups() { + let mut group_value_bytes = 0_u64; let mut group_exposure = 0_u64; for (column_index, column) in group.columns().iter().enumerate() { let selected = projected_columns.is_none_or(|columns| columns.contains(&column_index)); let mut dictionary_exposure = 0_u64; let mut data_exposure = 0_u64; - let _uncompressed = u64::try_from(column.uncompressed_size()) + let mut compressed_dictionary = 0_u64; + let mut compressed_data = 0_u64; + let uncompressed = u64::try_from(column.uncompressed_size()) .map_err(|_| corrupt("property column chunk has negative uncompressed size"))?; let compressed = u64::try_from(column.compressed_size()) .map_err(|_| corrupt("property column chunk has negative compressed size"))?; @@ -2854,17 +3076,21 @@ fn validate_parquet_resource_admission( #[allow(deprecated, reason = "Parquet 58 page admission requires raw sizes")] let uncompressed_page = u64::try_from(header.uncompressed_page_size) .map_err(|_| corrupt("property page has negative uncompressed size"))?; - if header_bytes == 0 - || (selected && uncompressed_page > max_page_bytes) - || compressed_page > compressed - { + if selected && uncompressed_page > max_page_bytes { + return Err(limit_error( + "property page exceeds pre-decode byte admission", + )); + } + if header_bytes == 0 || compressed_page > compressed { return Err(corrupt("property page exceeds pre-decode byte admission")); } if selected { if header.type_ == parquet::format::PageType::DICTIONARY_PAGE { dictionary_exposure = dictionary_exposure.max(uncompressed_page); + compressed_dictionary = compressed_dictionary.max(compressed_page); } else { data_exposure = data_exposure.max(uncompressed_page); + compressed_data = compressed_data.max(compressed_page); } } position = position @@ -2881,33 +3107,141 @@ fn validate_parquet_resource_admission( )); } if selected { + if include_codec { + let native = match column.compression() { + parquet::basic::Compression::UNCOMPRESSED => 0, + parquet::basic::Compression::ZSTD(_) => { + crate::permanent_parquet::ZSTD_DECODER_WORKSPACE as u64 + } + // Ordinary property callers use only decoded_bytes. Replay admission + // refuses a codec without a justified native-memory bound. + _ => u64::MAX, + }; + // A returned Arrow batch may span multiple pages. Fixed-width + // leaves have a value-count bound; variable and repeated leaves + // conservatively reserve the entire contributing row group. + let descriptor = column.column_descr(); + let values = u64::try_from(column.num_values()) + .map_err(|_| corrupt("negative Parquet value count"))?; + let values = if descriptor.max_rep_level() == 0 { + values.min(batch_rows as u64) + } else { + values + }; + let width = match descriptor.physical_type() { + parquet::basic::Type::BOOLEAN => Some(1_u64), + parquet::basic::Type::INT32 | parquet::basic::Type::FLOAT => Some(4), + parquet::basic::Type::INT64 | parquet::basic::Type::DOUBLE => Some(8), + parquet::basic::Type::INT96 => Some(12), + parquet::basic::Type::FIXED_LEN_BYTE_ARRAY => Some( + u64::try_from(descriptor.type_length()) + .map_err(|_| corrupt("negative fixed binary width"))?, + ), + parquet::basic::Type::BYTE_ARRAY => None, + }; + let decoded = width + .map_or_else( + || { + uncompressed + .saturating_add(dictionary_exposure.saturating_mul(values)) + }, + |width| values.saturating_mul(width), + ) + .saturating_add( + values.saturating_mul(16).saturating_mul( + 1 + u64::try_from(descriptor.max_def_level()) + .map_err(|_| corrupt("negative definition level"))? + + u64::try_from(descriptor.max_rep_level()) + .map_err(|_| corrupt("negative repetition level"))?, + ), + ); + group_value_bytes = group_value_bytes.saturating_add(decoded); + let memory = data_exposure + .saturating_mul(2) + .saturating_add(dictionary_exposure.saturating_mul(2)) + .saturating_add(compressed_data) + .saturating_add(compressed_dictionary) + .saturating_add(native) + .saturating_add(8 * 1024); + let column_max = column_memory.get_mut(column_index).ok_or_else(|| { + corrupt("Parquet row-group column count disagrees with schema") + })?; + *column_max = (*column_max).max(memory); + } group_exposure = group_exposure .checked_add(data_exposure) .and_then(|bytes| { dictionary_exposure - .checked_mul(u64::try_from(admitted_batch_rows(limits)).ok()?) + .checked_mul(u64::try_from(batch_rows).ok()?) .and_then(|decoded_dictionary| bytes.checked_add(decoded_dictionary)) }) .and_then(|bytes| { // Validity, offsets, and values buffers are live together. // Sixteen bytes/value/column deliberately over-reserves the // fixed Arrow bookkeeping before the builder allocates it. - u64::try_from(admitted_batch_rows(limits)) + u64::try_from(batch_rows) .ok()? .checked_mul(16) .and_then(|overhead| bytes.checked_add(overhead)) }) .ok_or_else(|| corrupt("property projected page exposure overflows"))?; } - if group_exposure > max_page_bytes { - return Err(corrupt( + if !include_codec && group_exposure > max_page_bytes { + return Err(limit_error( "property projected pages exceed pre-decode live-byte admission", )); } } largest_group_exposure = largest_group_exposure.max(group_exposure); + if include_codec { + group_values.push(( + u64::try_from(group.num_rows()).map_err(|_| corrupt("negative row-group rows"))?, + group_value_bytes, + )); + } + } + Ok(ParquetDecoderMemory { + decoded_bytes: largest_group_exposure, + // The raw-header audit ends before decoding. Its bounded temporary + // memory does not overlap decoder states or returned Arrow values. + with_codec_bytes: column_memory + .into_iter() + .fold(0, u64::saturating_add) + .saturating_add(contributing_row_group_bytes(&group_values, batch_rows)) + .max(64 * 1024), + }) +} + +/// A batch can start at the final row of any group, then consume up to B-1 +/// rows from following groups. Reserve every touched group's value envelope. +/// The sliding window is linear in the authenticated footer's group count. +fn contributing_row_group_bytes(groups: &[(u64, u64)], batch_rows: usize) -> u64 { + let mut maximum = 0_u64; + let mut end = 0; + let mut rows = 0_u64; + let mut bytes = 0_u64; + for start in 0..groups.len() { + if end == start { + rows = groups[start].0; + bytes = groups[start].1; + end += 1; + } + while end < groups.len() + && rows.saturating_sub(groups[start].0) < batch_rows.saturating_sub(1) as u64 + { + rows = rows.saturating_add(groups[end].0); + bytes = bytes.saturating_add(groups[end].1); + end += 1; + } + maximum = maximum.max(bytes); + // Overflow is a resource rejection, never a wrapped smaller bound. + if rows == u64::MAX || bytes == u64::MAX { + return u64::MAX; + } + rows -= groups[start].0; + bytes -= groups[start].1; } - Ok(largest_group_exposure) + maximum } fn admitted_batch_rows(limits: PropertyOverlayLimits) -> usize { @@ -3504,6 +3838,98 @@ mod tests { use std::collections::{BTreeSet, HashMap}; use tempfile::TempDir; + #[test] + fn replay_decoder_admission_covers_pages_groups_and_repeated_values() { + use arrow::array::{ListBuilder, UInt64Builder}; + let strings = (0..7) + .map(|index| format!("{index}{}", "x".repeat(100 * 1024))) + .collect::>(); + let mut lists = ListBuilder::new(UInt64Builder::new()); + for row in 0..7 { + for value in 0..4097 { + lists.values().append_value(u64::MAX - value - row); + } + lists.append(true); + } + let nested = lists.finish(); + let schema = Arc::new(Schema::new(vec![ + Field::new("value", DataType::Utf8, false), + Field::new("repeated", nested.data_type().clone(), false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(StringArray::from(strings)), Arc::new(nested)], + ) + .unwrap(); + for group_rows in [1, 2, 7] { + let file = tempfile::tempfile().unwrap(); + let properties = crate::permanent_parquet::writer_properties() + .set_dictionary_enabled(false) + .set_write_batch_size(1) + .set_data_page_row_count_limit(1) + .set_max_row_group_row_count(Some(group_rows)) + .build(); + let mut writer = + ArrowWriter::try_new(file.try_clone().unwrap(), schema.clone(), Some(properties)) + .unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + let builder = + ParquetRecordBatchReaderBuilder::try_new(file.try_clone().unwrap()).unwrap(); + let required = + replay_parquet_reader_reservation(builder.metadata(), &file, 64 * 1024 * 1024, 7) + .unwrap(); + assert!( + required + > batch.get_array_memory_size() + + 2 * crate::permanent_parquet::ZSTD_DECODER_WORKSPACE + ); + assert_eq!( + replay_parquet_reader_reservation(builder.metadata(), &file, required, 7).unwrap(), + required + ); + assert_eq!( + replay_parquet_reader_reservation(builder.metadata(), &file, required - 1, 7) + .unwrap_err() + .code(), + "GF_RESOURCE_LIMIT" + ); + let decoded = builder + .with_batch_size(7) + .build() + .unwrap() + .next() + .unwrap() + .unwrap(); + assert_eq!(decoded, batch); + } + } + + #[test] + fn replay_decoder_group_window_covers_every_batch_start() { + let groups = [(3, 100), (1, 700), (4, 200), (2, 900), (1, 300)]; + let rows = groups + .iter() + .enumerate() + .flat_map(|(index, (rows, _))| std::iter::repeat_n(index, *rows as usize)) + .collect::>(); + for batch_rows in 1..=rows.len() + 1 { + let actual = (0..rows.len()) + .map(|start| { + rows[start..rows.len().min(start + batch_rows)] + .iter() + .copied() + .collect::>() + .into_iter() + .map(|index| groups[index].1) + .sum::() + }) + .max() + .unwrap(); + assert_eq!(contributing_row_group_bytes(&groups, batch_rows), actual); + } + } + #[test] fn fragment_identity_is_numeric_canonical_and_total() { let id = PropertyFragmentId { diff --git a/crates/graphforge-storage/src/runtime_entity_labels.rs b/crates/graphforge-storage/src/runtime_entity_labels.rs index 7b4ae0f04..d5a4ae5b5 100644 --- a/crates/graphforge-storage/src/runtime_entity_labels.rs +++ b/crates/graphforge-storage/src/runtime_entity_labels.rs @@ -109,9 +109,12 @@ pub fn persist_runtime_catalog(dir: &Path, catalog: &RuntimeCatalog) -> Result<( let batch = catalog.to_record_batch(); let temporary = tempfile::NamedTempFile::new_in(&topology) .map_err(|error| storage_err(error.to_string()))?; - let mut writer = - parquet::arrow::ArrowWriter::try_new(temporary.as_file(), batch.schema(), None) - .map_err(|error| storage_err(error.to_string()))?; + let mut writer = parquet::arrow::ArrowWriter::try_new( + temporary.as_file(), + batch.schema(), + Some(crate::permanent_parquet::writer_properties().build()), + ) + .map_err(|error| storage_err(error.to_string()))?; writer .write(&batch) .map_err(|error| storage_err(error.to_string()))?; diff --git a/crates/graphforge-storage/src/semantic_bindings.rs b/crates/graphforge-storage/src/semantic_bindings.rs index b4f68b122..e48170e6b 100644 --- a/crates/graphforge-storage/src/semantic_bindings.rs +++ b/crates/graphforge-storage/src/semantic_bindings.rs @@ -1770,7 +1770,7 @@ fn rewrite_legacy_route( let mut writer = parquet::arrow::ArrowWriter::try_new( File::create(new).map_err(|_| corrupt("legacy migration destination cannot open"))?, schema, - None, + Some(crate::permanent_parquet::writer_properties().build()), ) .map_err(|_| corrupt("legacy migration writer cannot be built"))?; for batch in builder @@ -2098,7 +2098,7 @@ pub fn materialize_semantic_migration( let mut writer = parquet::arrow::ArrowWriter::try_new( File::create(&target).map_err(|_| corrupt("migration output cannot be opened"))?, schema.clone(), - None, + Some(crate::permanent_parquet::writer_properties().build()), ) .map_err(|_| corrupt("migration writer cannot be built"))?; for batch in builder diff --git a/crates/graphforge-storage/src/staging.rs b/crates/graphforge-storage/src/staging.rs index cab8b1bc5..891a738bd 100644 --- a/crates/graphforge-storage/src/staging.rs +++ b/crates/graphforge-storage/src/staging.rs @@ -32,7 +32,6 @@ use arrow::array::RecordBatch; use arrow::datatypes::SchemaRef; use parquet::arrow::ArrowWriter; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; -use parquet::file::properties::WriterProperties; use std::collections::BTreeMap; use std::sync::Arc; use tempfile::NamedTempFile; @@ -432,7 +431,7 @@ impl RewriteBatch { .suffix(".tmp") .tempfile_in(parent) .map_err(|error| io_err(&error))?; - let properties = WriterProperties::builder() + let properties = crate::permanent_parquet::writer_properties() .set_max_row_group_row_count(Some(ROW_GROUP_SIZE)) .build(); let mut writer = @@ -902,7 +901,7 @@ fn stage_parquet_temp( .tempfile_in(parent) .map_err(|e| io_err(&e))?; - let props = WriterProperties::builder() + let props = crate::permanent_parquet::writer_properties() .set_max_row_group_row_count(Some(ROW_GROUP_SIZE)) .build(); let mut writer = ArrowWriter::try_new(tmp.as_file(), schema, Some(props)).map_err(pq_err)?; @@ -935,7 +934,7 @@ where .suffix(".tmp") .tempfile_in(parent) .map_err(|error| io_err(&error))?; - let properties = WriterProperties::builder() + let properties = crate::permanent_parquet::writer_properties() .set_max_row_group_row_count(Some(ROW_GROUP_SIZE)) .build(); let mut writer = diff --git a/crates/graphforge-storage/src/vector_store.rs b/crates/graphforge-storage/src/vector_store.rs index b4db4bb68..76a2def40 100644 --- a/crates/graphforge-storage/src/vector_store.rs +++ b/crates/graphforge-storage/src/vector_store.rs @@ -289,8 +289,12 @@ where let batch = rows_to_batch(rows, schema.clone(), dimension, &mut checkpoint)?; let path = build_dir.join(VECTOR_DATA_FILE); let file = File::create(&path).map_err(|source| io("create vector snapshot", &path, source))?; - let mut writer = ArrowWriter::try_new(file, schema, None) - .map_err(|error| build(format!("create vector Parquet writer: {error}")))?; + let mut writer = ArrowWriter::try_new( + file, + schema, + Some(crate::permanent_parquet::writer_properties().build()), + ) + .map_err(|error| build(format!("create vector Parquet writer: {error}")))?; writer .write(&batch) .map_err(|error| build(format!("write vector Parquet batch: {error}")))?; diff --git a/crates/graphforge-storage/src/writer.rs b/crates/graphforge-storage/src/writer.rs index f8234e9ea..fabf54155 100644 --- a/crates/graphforge-storage/src/writer.rs +++ b/crates/graphforge-storage/src/writer.rs @@ -163,15 +163,12 @@ fn replay_resource_limit(message: impl Into) -> GfError { } fn replay_writer_properties(max_batch_rows: usize) -> parquet::file::properties::WriterProperties { - parquet::file::properties::WriterProperties::builder() + crate::permanent_parquet::writer_properties() .set_max_row_group_row_count(Some(max_batch_rows)) .set_dictionary_enabled(false) - .set_compression(parquet::basic::Compression::UNCOMPRESSED) .build() } -const REPLAY_WRITER_FIXED_BYTES: usize = 256 * 1024; -const REPLAY_COLUMN_CHUNK_METADATA_BYTES: usize = 512; const REPLAY_NODE_FIXED_ROW_BYTES: usize = 128; fn parquet_reader_metadata_reservation(path: &Path) -> Result { @@ -197,6 +194,38 @@ fn parquet_reader_metadata_reservation(path: &Path) -> Result { .ok_or_else(|| replay_resource_limit("Parquet reader metadata reservation overflow")) } +fn replay_reader_reservation( + path: &Path, + limits: crate::graph_delta_journal::GraphDeltaJournalLimits, +) -> Result { + let footer = parquet_reader_metadata_reservation(path)?; + if footer > limits.max_replay_memory_bytes { + return Err(replay_resource_limit( + "Parquet reader footer exceeds replay budget", + )); + } + let file = fs::File::open(path).map_err(|error| io_err(&error))?; + let builder = parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder::try_new( + file.try_clone().map_err(|error| io_err(&error))?, + ) + .map_err(pq_err)?; + let metadata_bytes = builder.metadata().memory_size(); + let available = limits + .max_replay_memory_bytes + .checked_sub(metadata_bytes) + .ok_or_else(|| replay_resource_limit("Parquet reader metadata exceeds replay budget"))?; + let decoder = crate::property_overlay::replay_parquet_reader_reservation( + builder.metadata(), + &file, + available, + limits.max_batch_rows, + )?; + metadata_bytes + .checked_add(decoder) + .map(|bytes| bytes.max(footer)) + .ok_or_else(|| replay_resource_limit("Parquet reader reservation overflow")) +} + fn replay_writer_reservation( schema: &Schema, maximum_rows: usize, @@ -204,21 +233,24 @@ fn replay_writer_reservation( max_batch_rows: usize, ) -> Result { let groups = maximum_rows.div_ceil(max_batch_rows); - let fields = schema.fields().len(); - let schema_bytes = schema.fields().iter().try_fold(0_usize, |sum, field| { - sum.checked_add(field.name().len().saturating_add(128)) - .ok_or_else(|| replay_resource_limit("graph delta replay schema memory overflow")) - })?; - let metadata_bytes = groups - .checked_mul(fields) - .and_then(|chunks| chunks.checked_mul(REPLAY_COLUMN_CHUNK_METADATA_BYTES)) - .ok_or_else(|| replay_resource_limit("graph delta replay metadata memory overflow"))?; + let structure_bytes = crate::permanent_parquet::replay_writer_structure_bytes(schema)?; + let metadata_bytes = crate::permanent_parquet::replay_metadata_bytes( + schema, + groups, + max_batch_rows.min(maximum_rows), + maximum_row_bytes, + )?; let active_rows = max_batch_rows.min(maximum_rows); let active_buffer_bytes = maximum_row_bytes .checked_mul(active_rows) .ok_or_else(|| replay_resource_limit("graph delta replay active buffer overflow"))?; - REPLAY_WRITER_FIXED_BYTES - .checked_add(schema_bytes) + let encoder_bytes = if maximum_row_bytes == 0 { + 0 // Property chunks reserve their actual aggregate snapshot bytes below. + } else { + crate::permanent_parquet::replay_encoder_buffers(schema, active_buffer_bytes, active_rows)? + }; + structure_bytes + .checked_add(encoder_bytes) .and_then(|bytes| bytes.checked_add(metadata_bytes)) .and_then(|bytes| bytes.checked_add(active_buffer_bytes)) .ok_or_else(|| replay_resource_limit("graph delta replay writer reservation overflow")) @@ -237,7 +269,7 @@ fn admit_replay_writer( .is_none_or(|bytes| bytes > limit) { return Err(replay_resource_limit(format!( - "graph delta replay {context} writer memory bound exceeded" + "graph delta replay {context} writer memory bound exceeded: overlay={overlay_bytes} authority={authority_bytes} writer={reservation_bytes} limit={limit}" ))); } Ok(()) @@ -291,7 +323,7 @@ pub(crate) fn write_replay_overlay_streaming( target: &Path, overlay: &crate::graph_delta_journal::ReplayOverlay, limits: crate::graph_delta_journal::GraphDeltaJournalLimits, -) -> Result<(), GfError> { +) -> Result { let mut target_routes = crate::route_component::owned::admit_owned_workspace(target)?; let property_inventory = crate::AuthenticatedPropertyInventory::from_inventory_at_root( source, @@ -335,7 +367,216 @@ pub(crate) fn write_replay_overlay_streaming( &target_routes, properties_changed, node_properties_changed, - ) + )?; + Ok(node_scan.spool_evidence) +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct ReplayNodeSpoolEvidence { + pub(crate) bytes: u64, + pub(crate) allocated_bytes: u64, +} + +// Only the low-budget flat-node replay strategy uses this private stream. One +// unlinked file exists at a time; it is never a graph payload or recovery input. +const REPLAY_NODE_SPOOL_LIMIT: u64 = 64 * 1024 * 1024; + +struct ReplayNodeSpoolSink { + file: fs::File, + bytes: u64, + limit: u64, +} + +impl std::io::Write for ReplayNodeSpoolSink { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + if self + .bytes + .checked_add(buffer.len() as u64) + .is_none_or(|end| end > self.limit) + { + return Err(std::io::Error::new( + std::io::ErrorKind::FileTooLarge, + "replay node spool byte ceiling", + )); + } + let written = std::io::Write::write(&mut self.file, buffer)?; + self.bytes += written as u64; + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + std::io::Write::flush(&mut self.file) + } +} + +fn replay_spool_error(error: arrow::error::ArrowError) -> GfError { + if matches!(&error, arrow::error::ArrowError::IoError(_, source) if source.kind() == std::io::ErrorKind::FileTooLarge) + { + replay_resource_limit("replay node spool exceeds temporary-disk byte ceiling") + } else { + pq_err(error) + } +} + +enum ReplayNodeInput { + Direct(parquet::arrow::arrow_reader::ParquetRecordBatchReader), + Spool(arrow::ipc::reader::StreamReader), +} + +impl Iterator for ReplayNodeInput { + type Item = Result; + fn next(&mut self) -> Option { + match self { + Self::Direct(reader) => reader.next(), + Self::Spool(reader) => reader.next(), + } + } +} + +fn spool_replay_nodes( + builder: parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder, + target: &Path, + expected_rows: usize, + batch_rows: usize, + byte_limit: u64, +) -> Result { + use std::io::{Seek, SeekFrom}; + let schema = builder.schema().clone(); + let source = builder + .with_batch_size(batch_rows) + .build() + .map_err(pq_err)?; + let sink = ReplayNodeSpoolSink { + file: tempfile::tempfile_in(target).map_err(|error| io_err(&error))?, + bytes: 0, + limit: byte_limit, + }; + let mut writer = + arrow::ipc::writer::StreamWriter::try_new(sink, &schema).map_err(replay_spool_error)?; + let mut rows = 0_usize; + for batch in source { + let batch = batch.map_err(pq_err)?; + if batch.num_rows() > batch_rows || batch.schema() != schema { + return Err(pq_err( + "replay node spool schema or batch authority differs", + )); + } + rows = rows + .checked_add(batch.num_rows()) + .ok_or_else(|| replay_resource_limit("replay node spool row overflow"))?; + if rows > expected_rows { + return Err(pq_err("replay node spool row authority differs")); + } + writer.write(&batch).map_err(replay_spool_error)?; + } + if rows != expected_rows { + return Err(pq_err("replay node spool row authority differs")); + } + // The Parquet iterator and all decoder contexts have dropped before the + // returned private IPC reader can overlap the permanent Parquet writer. + let mut sink = writer.into_inner().map_err(replay_spool_error)?; + sink.file + .seek(SeekFrom::Start(0)) + .map_err(|error| io_err(&error))?; + let reader = arrow::ipc::reader::StreamReader::try_new(sink.file, None).map_err(pq_err)?; + if reader.schema() != schema { + return Err(pq_err("replay node spool schema differs")); + } + Ok(ReplayNodeInput::Spool(reader)) +} + +fn replay_node_input( + node_path: &Path, + target: &Path, + node_scan: &ReplayNodeAuthority, + overlay_bytes: usize, + limits: crate::graph_delta_journal::GraphDeltaJournalLimits, + node_writer_reservation: usize, +) -> Result<(Option, ReplayNodeSpoolEvidence), GfError> { + use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; + let combined = overlay_bytes + .saturating_add(node_scan.estimated_memory()) + .saturating_add(node_scan.reader_reservation_bytes) + .saturating_add(node_writer_reservation); + let use_spool = node_path.exists() && combined > limits.max_replay_memory_bytes; + let reader = if use_spool { + let builder = ParquetRecordBatchReaderBuilder::try_new( + fs::File::open(node_path).map_err(|error| io_err(&error))?, + ) + .map_err(pq_err)?; + let active_bytes = node_scan + .maximum_row_bytes + .saturating_mul(limits.max_batch_rows.min(node_scan.base_rows)); + let ipc_reservation = 64_usize + .saturating_mul(1024) + .saturating_add(active_bytes.saturating_mul(3)) + .saturating_add(crate::permanent_parquet::replay_schema_bytes( + builder.schema(), + )?); + // Select both admitted phases before allocating the stream. No retry + // after a writer failure, and no uncompressed permanent-output mode. + admit_replay_writer( + overlay_bytes, + node_scan.estimated_memory(), + node_scan + .reader_reservation_bytes + .saturating_add(ipc_reservation), + limits.max_replay_memory_bytes, + "topology node spool decoder", + )?; + admit_replay_writer( + overlay_bytes, + node_scan.estimated_memory(), + node_writer_reservation.saturating_add(ipc_reservation), + limits.max_replay_memory_bytes, + "topology node", + )?; + Some(spool_replay_nodes( + builder, + target, + node_scan.base_rows, + limits.max_batch_rows, + REPLAY_NODE_SPOOL_LIMIT, + )?) + } else { + admit_replay_writer( + overlay_bytes, + node_scan + .estimated_memory() + .saturating_add(node_scan.reader_reservation_bytes), + node_writer_reservation, + limits.max_replay_memory_bytes, + "topology node", + )?; + if node_path.exists() { + Some(ReplayNodeInput::Direct( + ParquetRecordBatchReaderBuilder::try_new( + fs::File::open(node_path).map_err(|error| io_err(&error))?, + ) + .map_err(pq_err)? + .with_batch_size(limits.max_batch_rows) + .build() + .map_err(pq_err)?, + )) + } else { + None + } + }; + let spool_evidence = if let Some(ReplayNodeInput::Spool(reader)) = reader.as_ref() { + ReplayNodeSpoolEvidence { + bytes: reader + .get_ref() + .metadata() + .map_err(|error| io_err(&error))? + .len(), + allocated_bytes: graphforge_filesystem::file_space_usage(reader.get_ref()) + .map_err(|error| io_err(&error))? + .allocated_bytes, + } + } else { + ReplayNodeSpoolEvidence::default() + }; + Ok((reader, spool_evidence)) } fn stream_replay_nodes( @@ -344,10 +585,9 @@ fn stream_replay_nodes( overlay: &crate::graph_delta_journal::ReplayOverlay, limits: crate::graph_delta_journal::GraphDeltaJournalLimits, ) -> Result { - use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; let node_path = source.join("topology/nodes.parquet"); let output_node_path = target.join("topology/nodes.parquet"); - let node_scan = scan_replay_node_authority(&node_path, overlay, limits)?; + let mut node_scan = scan_replay_node_authority(&node_path, overlay, limits)?; fs::create_dir_all(output_node_path.parent().expect("node output has parent")) .map_err(|error| io_err(&error))?; let maximum_node_rows = node_scan.base_rows.saturating_add(overlay.nodes.len()); @@ -357,30 +597,15 @@ fn stream_replay_nodes( node_scan.maximum_row_bytes, limits.max_batch_rows, )?; - admit_replay_writer( + let (reader, spool_evidence) = replay_node_input( + &node_path, + target, + &node_scan, overlay.estimated_memory(), - node_scan.estimated_memory(), + limits, node_writer_reservation, - limits.max_replay_memory_bytes, - "topology node", )?; - let reader = if node_path.exists() { - let node_file = fs::File::open(&node_path).map_err(|error| { - GfError::Storage(format!( - "open canonical replay nodes at {}: {error}", - node_path.display() - )) - })?; - Some( - ParquetRecordBatchReaderBuilder::try_new(node_file) - .map_err(pq_err)? - .with_batch_size(limits.max_batch_rows) - .build() - .map_err(pq_err)?, - ) - } else { - None - }; + node_scan.spool_evidence = spool_evidence; let output = fs::File::create(&output_node_path).map_err(|error| io_err(&error))?; let mut writer = parquet::arrow::ArrowWriter::try_new( output, @@ -445,7 +670,8 @@ struct ReplayNodeAuthority { deleted_nodes: HashSet, base_rows: usize, maximum_row_bytes: usize, - reader_metadata_bytes: usize, + reader_reservation_bytes: usize, + spool_evidence: ReplayNodeSpoolEvidence, } impl ReplayNodeAuthority { @@ -460,7 +686,6 @@ impl ReplayNodeAuthority { .saturating_add(self.endpoint_ids.iter().fold(0_usize, |sum, (uuid, _)| { sum.saturating_add(80).saturating_add(uuid.len()) })) - .saturating_add(self.reader_metadata_bytes) } } @@ -507,9 +732,18 @@ fn scan_replay_node_authority( }) .max() .unwrap_or(REPLAY_NODE_FIXED_ROW_BYTES), - reader_metadata_bytes: 0, + reader_reservation_bytes: 0, + spool_evidence: ReplayNodeSpoolEvidence::default(), }); } + let reader_reservation = replay_reader_reservation(node_path, limits)?; + admit_replay_writer( + overlay.estimated_memory(), + 0, + reader_reservation, + limits.max_replay_memory_bytes, + "topology node decoder", + )?; let input = fs::File::open(node_path).map_err(|error| { GfError::Storage(format!( "scan canonical replay nodes at {}: {error}", @@ -631,7 +865,8 @@ fn scan_replay_node_authority( .max() .unwrap_or(REPLAY_NODE_FIXED_ROW_BYTES), ), - reader_metadata_bytes: parquet_reader_metadata_reservation(node_path)?, + reader_reservation_bytes: reader_reservation, + spool_evidence: ReplayNodeSpoolEvidence::default(), }) } @@ -743,6 +978,21 @@ fn stream_replay_edges( TYPED_EDGE_SCHEMA.clone() }; for (_, source_path) in &source_paths { + let reader_reservation = replay_reader_reservation(source_path, limits)?; + admit_replay_writer( + overlay.estimated_memory(), + nodes + .estimated_memory() + .saturating_add(relation_authority_bytes) + .saturating_add( + existing_overlay + .capacity() + .saturating_mul(std::mem::size_of::<&str>() + 8), + ), + reader_reservation, + limits.max_replay_memory_bytes, + "topology edge decoder", + )?; let input = fs::File::open(source_path).map_err(|error| io_err(&error))?; let builder = ParquetRecordBatchReaderBuilder::try_new(input).map_err(pq_err)?; if builder.schema().fields() != expected_schema.fields() @@ -851,7 +1101,7 @@ fn stream_replay_edges( source_paths .iter() .try_fold(0_usize, |maximum, (_, path)| { - Ok::<_, GfError>(maximum.max(parquet_reader_metadata_reservation(path)?)) + Ok::<_, GfError>(maximum.max(replay_reader_reservation(path, limits)?)) })?, ); admit_replay_writer( @@ -1087,7 +1337,38 @@ fn stream_replay_property_fragments( Ok(()) } +fn replay_property_resource_schema(logical: &Schema) -> Schema { + let mut fields = logical + .fields() + .iter() + .map(|field| field.as_ref().clone()) + .collect::>(); + fields.insert( + 1, + Field::new( + crate::property_overlay::PROPERTY_TOMBSTONE_FIELD, + DataType::Boolean, + false, + ), + ); + Schema::new_with_metadata(fields, logical.metadata().clone()) +} + +fn parse_replay_property_uuids( + names: &[&str], +) -> Result, GfError> { + names + .iter() + .map(|uuid| { + uuid::Uuid::parse_str(uuid) + .map(uuid::Uuid::into_bytes) + .map_err(pq_err) + }) + .collect::, _>>() +} + struct ReplayPropertyFragmentWriter { + extra_metadata_bytes: usize, logical_schema: Schema, physical_schema: SchemaRef, writer: parquet::arrow::ArrowWriter, @@ -1138,7 +1419,11 @@ impl ReplayPropertyRouteContext<'_> { names: &[&str], fragment: &mut Option, ) -> Result<(), GfError> { - let (before, rows) = self.property_rows(names)?; + let retained_writer = fragment.as_ref().map_or(0, |writer| { + self.writer_reservation_bytes + .saturating_add(writer.extra_metadata_bytes) + }); + let (before, rows) = self.property_rows(names, retained_writer)?; drop(before); if rows.is_empty() { return Ok(()); @@ -1149,9 +1434,41 @@ impl ReplayPropertyRouteContext<'_> { sum.checked_add(charge.saturating_mul(3)) .ok_or_else(|| replay_resource_limit("property replay output memory overflow")) })?; + let physical_schema = replay_property_resource_schema(&self.logical_schema); + let encoder_bytes = crate::permanent_parquet::replay_encoder_buffers( + &physical_schema, + output_bytes / 3, + rows.len(), + )?; + let maximum_row_bytes = usize::try_from( + rows.iter() + .map(crate::property_overlay::snapshot_charge) + .max() + .unwrap_or(0), + ) + .map_err(|_| replay_resource_limit("property replay row byte bound overflows"))?; + let chunk_metadata = crate::permanent_parquet::replay_metadata_bytes( + &physical_schema, + 1, + rows.len(), + maximum_row_bytes, + )?; + let base_chunk_metadata = crate::permanent_parquet::replay_metadata_bytes( + &physical_schema, + 1, + self.limits.max_batch_rows, + 0, + )?; + let extra_metadata_bytes = fragment + .as_ref() + .map_or(0, |writer| writer.extra_metadata_bytes) + .checked_add(chunk_metadata.saturating_sub(base_chunk_metadata)) + .ok_or_else(|| replay_resource_limit("property replay metadata overflow"))?; if self .overlay_bytes .checked_add(self.retained_target_bytes) + .and_then(|bytes| bytes.checked_add(encoder_bytes)) + .and_then(|bytes| bytes.checked_add(extra_metadata_bytes)) .and_then(|bytes| bytes.checked_add(output_bytes)) .and_then(|bytes| bytes.checked_add(self.writer_reservation_bytes)) .is_none_or(|bytes| bytes > self.limits.max_replay_memory_bytes) @@ -1161,6 +1478,7 @@ impl ReplayPropertyRouteContext<'_> { )); } let output = self.fragment_writer(fragment)?; + output.extra_metadata_bytes = extra_metadata_bytes; for row in rows { write_replay_property_snapshot( &mut output.writer, @@ -1173,22 +1491,26 @@ impl ReplayPropertyRouteContext<'_> { Ok(()) } - fn property_rows(&self, names: &[&str]) -> Result { - let targets = names - .iter() - .map(|uuid| { - uuid::Uuid::parse_str(uuid) - .map(uuid::Uuid::into_bytes) - .map_err(pq_err) - }) - .collect::, _>>()?; - let (mut baseline, _) = - crate::property_overlay::read_authenticated_property_snapshots_for_inventory( - self.inventory, - self.kind, - self.route, - &targets, - )?; + fn property_rows( + &self, + names: &[&str], + retained_writer_bytes: usize, + ) -> Result { + let targets = parse_replay_property_uuids(names)?; + let (mut baseline, _) = crate::property_overlay::read_replay_property_targets( + self.inventory, + self.kind, + self.route, + &targets, + self.limits + .max_replay_memory_bytes + .checked_sub(self.overlay_bytes) + .and_then(|bytes| bytes.checked_sub(self.retained_target_bytes)) + .and_then(|bytes| bytes.checked_sub(retained_writer_bytes)) + .ok_or_else(|| { + replay_resource_limit("property replay decoder has no available budget") + })?, + )?; let mut before = BTreeMap::new(); let baseline_bytes = baseline.values().fold(0_usize, |sum, row| { sum.saturating_add( @@ -1199,6 +1521,7 @@ impl ReplayPropertyRouteContext<'_> { let resident_before_output = self .overlay_bytes .checked_add(self.retained_target_bytes) + .and_then(|bytes| bytes.checked_add(retained_writer_bytes)) .and_then(|bytes| bytes.checked_add(baseline_bytes)) .ok_or_else(|| replay_resource_limit("property replay decoded memory overflow"))?; if resident_before_output > self.limits.max_replay_memory_bytes { @@ -1337,6 +1660,7 @@ fn open_replay_property_fragment( ) .map_err(pq_err)?; Ok(ReplayPropertyFragmentWriter { + extra_metadata_bytes: 0, logical_schema: logical_schema.as_ref().clone(), physical_schema, writer, @@ -1420,7 +1744,7 @@ fn stream_replay_property_route_with_table( let inferred = Arc::clone(&context.logical_schema); let mut authority = authority; for names in target_names.chunks(limits.max_batch_rows) { - let (before, after) = context.property_rows(names)?; + let (before, after) = context.property_rows(names, 0)?; authority = Some(crate::property_overlay::update_live_route_schema( kind, route, @@ -1433,7 +1757,7 @@ fn stream_replay_property_route_with_table( context.logical_schema = authority.ok_or_else(|| pq_err("property replay route has no touched schema authority"))?; context.writer_reservation_bytes = replay_writer_reservation( - context.logical_schema.as_ref(), + &replay_property_resource_schema(&context.logical_schema), target_names.len(), 0, limits.max_batch_rows, @@ -6693,6 +7017,63 @@ mod tests { ); } + #[test] + fn replay_node_spool_has_exact_values_byte_boundary_and_private_cleanup() { + use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; + let directory = tempfile::tempdir().unwrap(); + let source = directory.path().join("source.parquet"); + let rows = (0..7) + .map(|index| crate::graph_delta_journal::ReplayNodeRow { + node_uuid: new_v7().to_string(), + node_id: u64::MAX - 7 + index, + primary_type: PrimaryEntityTypeId::decode(1).unwrap(), + type_ids: (1..=index + 1) + .map(|id| EntityTypeId::decode(id as u32).unwrap()) + .collect(), + created_at_micros: 1_700_000_000_000_000 + index as i64, + updated_at_micros: 1_700_000_000_000_010 + index as i64, + }) + .collect::>(); + let expected = replay_node_batch(&rows.iter().collect::>()).unwrap(); + let mut writer = parquet::arrow::ArrowWriter::try_new( + fs::File::create(&source).unwrap(), + expected.schema(), + Some(crate::permanent_parquet::writer_properties().build()), + ) + .unwrap(); + writer.write(&expected).unwrap(); + writer.close().unwrap(); + let original = fs::read(&source).unwrap(); + let open = + || ParquetRecordBatchReaderBuilder::try_new(fs::File::open(&source).unwrap()).unwrap(); + let mut stream = + spool_replay_nodes(open(), directory.path(), 7, 2, REPLAY_NODE_SPOOL_LIMIT).unwrap(); + let ReplayNodeInput::Spool(reader) = &stream else { + panic!("private stream expected") + }; + let required = reader.get_ref().metadata().unwrap().len(); + assert!(required < 16 * 1024); + #[cfg(unix)] + assert_eq!( + directory.path().read_dir().unwrap().count(), + 1, + "Unix spool must have no pathname" + ); + let batches = stream.by_ref().collect::, _>>().unwrap(); + assert!(batches.iter().all(|batch| batch.num_rows() <= 2)); + assert_eq!( + arrow::compute::concat_batches(&expected.schema(), &batches).unwrap(), + expected + ); + drop(stream); + drop(spool_replay_nodes(open(), directory.path(), 7, 2, required).unwrap()); + let rejected = spool_replay_nodes(open(), directory.path(), 7, 2, required - 1); + assert!(matches!(rejected, Err(ref error) if error.code() == "GF_RESOURCE_LIMIT")); + assert!(spool_replay_nodes(open(), directory.path(), 6, 2, required).is_err()); + assert_eq!(directory.path().read_dir().unwrap().count(), 1); + assert_eq!(fs::read(&source).unwrap(), original); + } + #[test] fn replay_writer_reservation_scales_with_columns_and_row_groups() { let narrow = Schema::new(vec![Field::new("id", DataType::UInt64, false)]); @@ -6707,9 +7088,9 @@ mod tests { assert!(four_groups > one_group); assert!(four_groups > narrow_four_groups); - assert_eq!( - four_groups - one_group, - 3 * 128 * REPLAY_COLUMN_CHUNK_METADATA_BYTES + assert!( + four_groups - one_group + >= 3 * 128 * size_of::() ); } @@ -6725,7 +7106,8 @@ mod tests { deleted_nodes: (0..count).map(|index| format!("deleted-{index}")).collect(), base_rows: count, maximum_row_bytes: REPLAY_NODE_FIXED_ROW_BYTES, - reader_metadata_bytes: 0, + reader_reservation_bytes: 0, + spool_evidence: ReplayNodeSpoolEvidence::default(), }; let n = authority(64).estimated_memory(); let two_n = authority(128).estimated_memory(); @@ -9693,7 +10075,10 @@ mod tests { .decompress(&mut decoded, &compressed) .unwrap(); assert_eq!(decoded, input); - assert!(active_decoder.sizeof() <= 128 * 1024); + assert!( + active_decoder.sizeof() + empty_compressor_bytes + <= crate::permanent_parquet::ZSTD_DECODER_WORKSPACE + ); let reused_output = reused.compress(&input).unwrap(); assert_eq!(compressed, reused_output); // Pinned Zstd1 fast strategy: fixed contexts/workspace, at most @@ -9701,9 +10086,7 @@ mod tests { // Retained state is bounded by the lifetime maximum source size. // The dormant DCtx is part of this encoder only, not an input reader. maximum_source = maximum_source.max(size); - let block = maximum_source.min(128 * 1024); - let hash = maximum_source.max(64).next_power_of_two().min(16384) * 8; - let envelope = 128 * 1024 + hash + block + 11 * (block / 4); + let envelope = crate::permanent_parquet::zstd_encoder_workspace(maximum_source); assert!(fresh.context_mut().sizeof() + decompressor.sizeof() <= envelope); assert!(reused.context_mut().sizeof() + decompressor.sizeof() <= envelope); diff --git a/crates/graphforge-storage/tests/graph_delta_journal.rs b/crates/graphforge-storage/tests/graph_delta_journal.rs index f8d5e0643..5d84e36f0 100644 --- a/crates/graphforge-storage/tests/graph_delta_journal.rs +++ b/crates/graphforge-storage/tests/graph_delta_journal.rs @@ -271,6 +271,36 @@ fn streaming_resource_ladder_is_independent_of_base_rows() { limits, ) .unwrap(); + assert!(replay.temporary_decode_stream_bytes <= 256 * 1024); + assert!(replay.temporary_decode_stream_allocated_bytes <= 256 * 1024); + if extra_nodes == 1024 { + assert!( + replay.temporary_decode_stream_bytes > 0, + "the 2 MiB phase-separated strategy must be exercised" + ); + let direct = tempfile::tempdir().unwrap(); + let (_, direct_evidence) = materialize_replayed_graph_tree( + &resolved.graph_tree_root(), + &inventory, + direct.path(), + GraphDeltaJournalLimits { + max_batch_rows: 7, + ..GraphDeltaJournalLimits::default() + }, + ) + .unwrap(); + assert_eq!(direct_evidence.temporary_decode_stream_bytes, 0); + assert_eq!( + fs::read(target.path().join("topology/nodes.parquet")).unwrap(), + fs::read(direct.path().join("topology/nodes.parquet")).unwrap() + ); + } + println!( + "REPLAY_STREAM_BUDGET nodes={} stream_bytes={} stream_allocated={}", + extra_nodes + 4, + replay.temporary_decode_stream_bytes, + replay.temporary_decode_stream_allocated_bytes + ); evidence.push(( replay.estimated_replay_memory_bytes, replay.materialization_batch_row_bound, @@ -332,7 +362,9 @@ fn topology_replay_admission_is_path_specific_and_never_creates_rejected_output( let mut observed_node_rejection = false; let mut observed_edge_rejection = false; - for kibibytes in (256..=768).step_by(16) { + // Codec admission moves both thresholds; keep the same 2 MiB admitted + // ceiling and exercise each path below it without loosening that ceiling. + for kibibytes in (256..=2048).step_by(16) { let target = tempfile::tempdir().unwrap(); let node_output = target.path().join("topology/nodes.parquet"); let edge_output = target.path().join(&edge_relative); diff --git a/docs/book/architecture/permanent-storage-assessment.md b/docs/book/architecture/permanent-storage-assessment.md index e696d9a0f..cc1753fd2 100644 --- a/docs/book/architecture/permanent-storage-assessment.md +++ b/docs/book/architecture/permanent-storage-assessment.md @@ -376,3 +376,121 @@ formatting and gate-registry checks pass. Independent review verifies the retained #1224 property-generation repair and real mixed GFDR coverage. #1221 still owns unsupported topology-journal authority and constructed-parent composite topology mutation; this repair does not broaden that support. + +## Permanent Parquet publishing policy (#1213) + +The production audit follows publication ownership, not temporary filenames. +There are fourteen storage constructor sites and two API constructor sites. +`permanent_parquet::writer_properties` selects Zstd level 1, Parquet V1, +page statistics and offset indexes, 1 MiB page/dictionary targets, 20,000 page +rows, 1,024-value write batches and 64-byte statistics/index truncation requests. +These are explicit pinned Parquet 58 defaults, with the codec changed where +necessary. Statistics truncation is a request: an unincrementable maximum can +retain its original value. A footer identifies Zstd but does not encode its +compression level; the shared builder establishes level 1. + +| Permanent producer | Publishing path | Retained resource/lifecycle choices | +| --- | --- | --- | +| `graph_construction_encoding::write_parquet` | Resumable topology, properties and controls | Existing bounded construction windows, cache advice and publication leases | +| `graph_construction::write_parquet_with_properties` | Privately shaped runtime catalog, then published unchanged | Permanent caller explicitly supplies policy; accepted construction batches remain private | +| `writer::stream_replay_nodes` | Canonical replay and compaction nodes | Dictionaries off, row groups at most `max_batch_rows`; bounded decoder strategy below | +| `writer::stream_replay_edges` | Canonical replay and compaction edge routes | Dictionaries off, same row-group bound, authenticated physical route/schema | +| `writer::open_replay_property_fragment` | Immutable property/tombstone fragments | Dictionaries off, chunk flushes, same row-group bound, route/generation metadata | +| `staging::restage_append` | Mutation replacement later owned by `RewriteBatch` | 65,536-row groups; private file ownership and atomic replacement | +| `staging::stage_parquet_temp` | Mutation and catalog replacement | Same row-group bound and publication lifecycle | +| `staging::stage_parquet_batches_temp` | Streaming mutation replacement | Same row-group bound; separate reader/writer lifetime | +| `graph_projection::write_parquet` | Belief and portable projected graph payloads/catalog | Existing collection/sort behavior; no new whole-process memory claim | +| `semantic_bindings::rewrite_legacy_route` | Supported current-format route composition | Existing 8,192-row reader batches and semantic metadata | +| `semantic_bindings::materialize_semantic_migration` | Supported current-format multi-ontology composition | Existing file/row/input-byte limits and cancellation checkpoints | +| `vector_store::write_vector_snapshot` | Vector-search participant | Dimension, vector, cell and encoded-file limits; complete Arrow batch remains | +| `project_checkpoints` restoration encoder | Restoration-transition participant | `graphforge-restoration-transition/1` creator marker and restoration lifecycle | +| `runtime_entity_labels::persist_runtime_catalog` | Bulk/composite runtime catalog publication | Existing private staging and catalog authority | +| API `knowledge::write_parquet` | Assertion/evidence/confidence/reasoning participants | Existing serialized-Vec ownership and atomic participant publication | +| API `provenance::write_parquet` | Provenance participant | Existing serialized-Vec ownership and atomic participant publication | + +Separate writer implementations remain. The policy does not own files, +leases, authentication, durability, recovery, cancellation or commit authority. +No permanent writer has an uncompressed-policy exception. Current semantic +composition remains included despite old function names containing “legacy” or +“migration”; this change adds no compatibility or migration machinery. + +Private accepted construction chunks, shape/merge streams, the bounded replay +IPC stream and test fixtures are excluded. The user-selected Parquet result +sink in `graphforge-io` is an external result artifact, not a graph generation. +Standalone ontology persistence has eight Parquet tables but no verified graph +publication caller; it remains a separate public API, not a test fixture. The +source audit found no production `SerializedFileWriter` constructor. + +### Baseline and paired experiments + +The baseline source is `60ffdca983ed1ad4b2acd4edfe2605f3e85d6e7e`. +[`permanent-parquet-1213-baseline.json`](../../development/evidence/permanent-parquet-1213-baseline.json) +records actual public construction, mutation, compaction, reopen, query, export, +full verification and clean import. At 1,025 nodes and 4,097 random edges with +heterogeneous properties, construction published 353,548 Parquet bytes; mutation +published 377,487; compaction increased that to 537,467. The respective column +codec counts were 99/0, 92/10 and 52/21 Zstd/uncompressed. This demonstrates the +production regression rather than inferring it from a writer helper. + +That baseline process took 12.48 s elapsed, 11.89 s user and 0.84 s system, +with 156,284 KiB peak RSS. Its separate syscall run read 179,612,114 bytes and +wrote 42,717,623 bytes, including file-copy syscalls, startup, queries and portable +operations. A third run sampled a peak of 8,749,056 allocated bytes across the +workspace, deduplicating hard links. Sampling excludes unlinked open files and +can miss short-lived peaks; it includes retained generations and portable +artifacts, so it is neither an exact temporary-only peak nor an admission limit. + +Paired encodes retain each path's schema, dictionary and row-group settings, +changing only the codec. Cases include a materialized one-row fragment, 257 +wide/nullable/heterogeneous rows with a 256 KiB string, and 65,537 random UUID / +full-width integer rows crossing both replay and staging row-group boundaries. +The independent pair-process run took 1.31 s elapsed, 1.27 s user and 0.04 s +system, with 58,604 KiB peak RSS; its syscall run read 22,263,859 and wrote +12,610,549 bytes. These are codec experiments, not whole-publication admission. +Small fragments can grow with Zstd; the one-row dictionaries-on case grew from +10,017 to 10,722 bytes. Wide replay-profile output fell from 502,549 to 26,794 +bytes; the narrow random replay profile fell from 1,576,490 to 1,119,732 bytes. + +Deterministic codec-test ceilings are 2 MiB per output, 8 MiB combined temporary +allocation per pair, and 8 MiB of `ArrowWriter::memory_size()`. The latter omits +native codec allocation and completed metadata, so it is deliberately not used +as a replay memory proof. Timing/RSS/kernel-I/O observations are host-dependent; +row, page, byte and explicit reservation ceilings provide repeatable regressions. + +### Replay resource composition + +The old estimate omitted native codecs, physical nested leaves, decoder page +buffers and retained page indexes. Pinned Parquet 58/Zstd 1.5.7 source inspection +and safe context-size measurements establish separate reservations for schemas, +writer structures, completed metadata/indexes, active encoded chunks and native +contexts. Decoder admission inspects authenticated raw page headers before +allocating the decoder, includes compressed and decompressed buffers, and bounds +returned values across pages and row-group boundaries. Variable/repeated leaves +conservatively reserve the contributing row groups; this can refuse small reads +from very large groups. These are allocation-component/logical estimates, not a +process-RSS or allocator-fragmentation guarantee. + +Dictionaries remain disabled for replay. Below both page thresholds, no page is +compressed during writes; row-group close consumes leaf writers sequentially. +Only one compressor becomes active alongside the other dormant codec pairs. +Larger groups reserve all active codec contexts. Completed compressed chunks +remain charged until flush. Property decoder and encoder phases are separate, +while metadata from previously flushed property chunks remains resident. + +The 2 MiB, seven-row topology replay fixture remains supported. When direct +node decoding plus encoding does not fit but each separate phase does, replay +selects a private uncompressed Arrow IPC stream **before** encoding. One +self-deleting stream is capped at 64 MiB per replay invocation; cumulative writes +are checked before reaching disk. Both phases are admitted independently. +The normal direct strategy incurs no stream I/O. This is private staging, not +an uncompressed permanent-output fallback. + +At 260 and 516 nodes the fixture uses direct replay. At 1,028 nodes the private +stream is 207,496 bytes / 208,896 allocated bytes, below its deterministic +256 KiB fixture ceiling. Successful replay writes and reads that stream once; +its permanent node Parquet file is byte-for-byte identical to the direct +strategy with the same seven-row bound. Separate tests cover exact byte-limit +acceptance, one-byte-below rejection, schema/full-width values, row order, +row-count authority, cleanup and unchanged source bytes. Reader admission also +covers seven one-row large-string pages, repeated values and batches spanning +several tiny row groups, with rejection immediately below the computed bound. diff --git a/docs/development/evidence/permanent-parquet-1213-baseline.json b/docs/development/evidence/permanent-parquet-1213-baseline.json new file mode 100644 index 000000000..847c12b75 --- /dev/null +++ b/docs/development/evidence/permanent-parquet-1213-baseline.json @@ -0,0 +1,735 @@ +{ + "issue": 1213, + "phase": "current defaults before policy change", + "source_commit": "60ffdca983ed1ad4b2acd4edfe2605f3e85d6e7e", + "production_fixture": { + "nodes": 1025, + "edges": 4097, + "routes": 2, + "random_endpoints": true, + "heterogeneous_properties": true, + "actual_published_parquet": [ + { + "stage": "construction", + "logical_bytes": 353548, + "allocated_bytes": 389120, + "zstd_columns": 99, + "uncompressed_columns": 0 + }, + { + "stage": "mutation", + "logical_bytes": 377487, + "allocated_bytes": 413696, + "zstd_columns": 92, + "uncompressed_columns": 10 + }, + { + "stage": "compaction", + "logical_bytes": 537467, + "allocated_bytes": 569344, + "zstd_columns": 52, + "uncompressed_columns": 21 + } + ] + }, + "production_runtime": { + "elapsed_seconds": 12.48, + "user_seconds": 11.89, + "system_seconds": 0.84, + "peak_rss_kib": 156284, + "filesystem_input_512_byte_blocks": 50640, + "filesystem_output_512_byte_blocks": 90752, + "exit_status": 0 + }, + "production_syscall_io": { + "syscalls": { + "read": { + "calls": 27353, + "bytes": 155949734 + }, + "pread64": { + "calls": 22578, + "bytes": 14200336 + }, + "write": { + "calls": 4287, + "bytes": 33255579 + }, + "fsync": { + "calls": 3109, + "bytes": 0 + }, + "copy_file_range": { + "calls": 980, + "bytes": 9462044 + } + }, + "read_bytes": 179612114, + "write_bytes": 42717623, + "errors": [], + "elapsed_seconds": 23.17 + }, + "production_workspace_observation": { + "command": [ + "/home/ubuntu/code/graphforge-target-1195/debug/deps/permanent_storage_budgets-c514423ca6fb68a2", + "permanent_publishing_policy_construction_mutation_and_compaction", + "--exact", + "--nocapture", + "--test-threads=1" + ], + "sampled_unique_inode_allocated_peak_bytes": 8749056, + "sampled_path_logical_peak_bytes": 7397803, + "sampled_file_count_peak": 514, + "samples": 1207, + "requested_poll_interval_seconds": 0.005, + "maximum_observed_sample_interval_seconds": 0.01462249900214374, + "vanished_files_during_scan": 28, + "exit_status": 0, + "limitations": [ + "Sampling can miss shorter-lived allocation peaks and unlinked open files.", + "Includes retained project generations, private hydration/staging and portable artifacts, not only temporary files.", + "Hardlinks are deduplicated for allocated bytes, not path logical bytes.", + "Separate run from CPU/RSS and syscall traces; scanner adds filesystem activity." + ] + }, + "paired_codec_runtime": { + "elapsed_seconds": 1.31, + "user_seconds": 1.27, + "system_seconds": 0.04, + "peak_rss_kib": 58604, + "filesystem_input_512_byte_blocks": 0, + "filesystem_output_512_byte_blocks": 24696, + "exit_status": 0 + }, + "paired_codec_syscall_io": { + "syscalls": { + "read": { + "calls": 2468, + "bytes": 22262179 + }, + "pread64": { + "calls": 2, + "bytes": 1680 + }, + "write": { + "calls": 1642, + "bytes": 12610549 + } + }, + "read_bytes": 22263859, + "write_bytes": 12610549, + "errors": [], + "elapsed_seconds": 1.52 + }, + "limitations": [ + "CPU/RSS, syscall traces and workspace sampling are separate runs. OS I/O depends on cache state.", + "Production observations include public query, reopen, export, verify and clean import, startup and test output.", + "Workspace sampling misses unlinked open files and short-lived peaks; it is not a deterministic temporary-disk ceiling.", + "Paired profiles hold schema, dictionary and row-group settings fixed; they normalize actual production batches, not the mixed writer settings of the original files.", + "ArrowWriter.memory_size excludes native codec allocations and is not process RSS. Codec tests do not establish whole-publication admission." + ], + "codec_pair_cases": [ + { + "arrow_input_bytes": 4609, + "profiles": [ + { + "dictionary": true, + "outputs_current_candidate": [ + { + "allocated_bytes": 12288, + "arrow_writer_peak_bytes": 2664704, + "bytes": 10722, + "codec": "ZSTD(ZstdLevel(1))", + "decode_ns": 2280982, + "encode_ns": 6178583 + }, + { + "allocated_bytes": 12288, + "arrow_writer_peak_bytes": 2664704, + "bytes": 10722, + "codec": "ZSTD(ZstdLevel(1))", + "decode_ns": 1819293, + "encode_ns": 5250027 + } + ], + "pair_temporary_allocated_peak_bytes": 24576, + "profile": "construction", + "row_group_rows": 1048576, + "write_batch_rows": 127 + }, + { + "dictionary": true, + "outputs_current_candidate": [ + { + "allocated_bytes": 12288, + "arrow_writer_peak_bytes": 2664704, + "bytes": 10017, + "codec": "UNCOMPRESSED", + "decode_ns": 1574589, + "encode_ns": 3996974 + }, + { + "allocated_bytes": 12288, + "arrow_writer_peak_bytes": 2664704, + "bytes": 10722, + "codec": "ZSTD(ZstdLevel(1))", + "decode_ns": 1756112, + "encode_ns": 5126494 + } + ], + "pair_temporary_allocated_peak_bytes": 24576, + "profile": "canonical_staging", + "row_group_rows": 65536, + "write_batch_rows": 127 + }, + { + "dictionary": false, + "outputs_current_candidate": [ + { + "allocated_bytes": 12288, + "arrow_writer_peak_bytes": 9768, + "bytes": 9025, + "codec": "UNCOMPRESSED", + "decode_ns": 1243793, + "encode_ns": 2849253 + }, + { + "allocated_bytes": 12288, + "arrow_writer_peak_bytes": 9768, + "bytes": 9387, + "codec": "ZSTD(ZstdLevel(1))", + "decode_ns": 1364455, + "encode_ns": 3004195 + } + ], + "pair_temporary_allocated_peak_bytes": 24576, + "profile": "replay", + "row_group_rows": 8192, + "write_batch_rows": 127 + }, + { + "dictionary": true, + "outputs_current_candidate": [ + { + "allocated_bytes": 12288, + "arrow_writer_peak_bytes": 2664704, + "bytes": 10017, + "codec": "UNCOMPRESSED", + "decode_ns": 1534758, + "encode_ns": 3882031 + }, + { + "allocated_bytes": 12288, + "arrow_writer_peak_bytes": 2664704, + "bytes": 10722, + "codec": "ZSTD(ZstdLevel(1))", + "decode_ns": 1724422, + "encode_ns": 5045112 + } + ], + "pair_temporary_allocated_peak_bytes": 24576, + "profile": "other_permanent", + "row_group_rows": 1048576, + "write_batch_rows": 127 + } + ], + "rows": 1 + }, + { + "arrow_input_bytes": 610185, + "profiles": [ + { + "dictionary": true, + "outputs_current_candidate": [ + { + "allocated_bytes": 36864, + "arrow_writer_peak_bytes": 3371970, + "bytes": 34251, + "codec": "ZSTD(ZstdLevel(1))", + "decode_ns": 4599685, + "encode_ns": 17340049 + }, + { + "allocated_bytes": 36864, + "arrow_writer_peak_bytes": 3371970, + "bytes": 34251, + "codec": "ZSTD(ZstdLevel(1))", + "decode_ns": 4102225, + "encode_ns": 16906531 + } + ], + "pair_temporary_allocated_peak_bytes": 73728, + "profile": "construction", + "row_group_rows": 1048576, + "write_batch_rows": 127 + }, + { + "dictionary": true, + "outputs_current_candidate": [ + { + "allocated_bytes": 397312, + "arrow_writer_peak_bytes": 3371970, + "bytes": 395232, + "codec": "UNCOMPRESSED", + "decode_ns": 3408113, + "encode_ns": 13512168 + }, + { + "allocated_bytes": 36864, + "arrow_writer_peak_bytes": 3371970, + "bytes": 34251, + "codec": "ZSTD(ZstdLevel(1))", + "decode_ns": 4075585, + "encode_ns": 16803889 + } + ], + "pair_temporary_allocated_peak_bytes": 434176, + "profile": "canonical_staging", + "row_group_rows": 65536, + "write_batch_rows": 127 + }, + { + "dictionary": false, + "outputs_current_candidate": [ + { + "allocated_bytes": 503808, + "arrow_writer_peak_bytes": 551688, + "bytes": 502549, + "codec": "UNCOMPRESSED", + "decode_ns": 2271641, + "encode_ns": 4710967 + }, + { + "allocated_bytes": 28672, + "arrow_writer_peak_bytes": 551688, + "bytes": 26794, + "codec": "ZSTD(ZstdLevel(1))", + "decode_ns": 2889213, + "encode_ns": 6419048 + } + ], + "pair_temporary_allocated_peak_bytes": 532480, + "profile": "replay", + "row_group_rows": 8192, + "write_batch_rows": 127 + }, + { + "dictionary": true, + "outputs_current_candidate": [ + { + "allocated_bytes": 397312, + "arrow_writer_peak_bytes": 3371970, + "bytes": 395232, + "codec": "UNCOMPRESSED", + "decode_ns": 3280041, + "encode_ns": 13411216 + }, + { + "allocated_bytes": 36864, + "arrow_writer_peak_bytes": 3371970, + "bytes": 34251, + "codec": "ZSTD(ZstdLevel(1))", + "decode_ns": 4026594, + "encode_ns": 16644416 + } + ], + "pair_temporary_allocated_peak_bytes": 434176, + "profile": "other_permanent", + "row_group_rows": 1048576, + "write_batch_rows": 127 + } + ], + "rows": 257 + }, + { + "arrow_input_bytes": 1573144, + "profiles": [ + { + "dictionary": true, + "outputs_current_candidate": [ + { + "allocated_bytes": 1241088, + "arrow_writer_peak_bytes": 3583211, + "bytes": 1241030, + "codec": "ZSTD(ZstdLevel(1))", + "decode_ns": 12056712, + "encode_ns": 116289629 + }, + { + "allocated_bytes": 1241088, + "arrow_writer_peak_bytes": 3583211, + "bytes": 1241030, + "codec": "ZSTD(ZstdLevel(1))", + "decode_ns": 11965740, + "encode_ns": 116038864 + } + ], + "pair_temporary_allocated_peak_bytes": 2482176, + "profile": "construction", + "row_group_rows": 1048576, + "write_batch_rows": 127 + }, + { + "dictionary": true, + "outputs_current_candidate": [ + { + "allocated_bytes": 1703936, + "arrow_writer_peak_bytes": 3131006, + "bytes": 1703147, + "codec": "UNCOMPRESSED", + "decode_ns": 8156090, + "encode_ns": 101252562 + }, + { + "allocated_bytes": 1241088, + "arrow_writer_peak_bytes": 3131042, + "bytes": 1240628, + "codec": "ZSTD(ZstdLevel(1))", + "decode_ns": 11279937, + "encode_ns": 106788475 + } + ], + "pair_temporary_allocated_peak_bytes": 2945024, + "profile": "canonical_staging", + "row_group_rows": 65536, + "write_batch_rows": 127 + }, + { + "dictionary": false, + "outputs_current_candidate": [ + { + "allocated_bytes": 1576960, + "arrow_writer_peak_bytes": 261120, + "bytes": 1576490, + "codec": "UNCOMPRESSED", + "decode_ns": 3197669, + "encode_ns": 36107324 + }, + { + "allocated_bytes": 1122304, + "arrow_writer_peak_bytes": 261120, + "bytes": 1119732, + "codec": "ZSTD(ZstdLevel(1))", + "decode_ns": 6285506, + "encode_ns": 44280675 + } + ], + "pair_temporary_allocated_peak_bytes": 2699264, + "profile": "replay", + "row_group_rows": 8192, + "write_batch_rows": 127 + }, + { + "dictionary": true, + "outputs_current_candidate": [ + { + "allocated_bytes": 1703936, + "arrow_writer_peak_bytes": 3583157, + "bytes": 1703451, + "codec": "UNCOMPRESSED", + "decode_ns": 7955966, + "encode_ns": 97091446 + }, + { + "allocated_bytes": 1241088, + "arrow_writer_peak_bytes": 3583211, + "bytes": 1241030, + "codec": "ZSTD(ZstdLevel(1))", + "decode_ns": 11292287, + "encode_ns": 106851136 + } + ], + "pair_temporary_allocated_peak_bytes": 2945024, + "profile": "other_permanent", + "row_group_rows": 1048576, + "write_batch_rows": 127 + } + ], + "rows": 65537 + } + ], + "normalized_production_codec_pair_totals": { + "construction": { + "construction": { + "files": 20, + "dictionary": true, + "row_group_rows": 1048576, + "outputs_current_candidate": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "bytes": 353548, + "allocated_bytes": 389120, + "encode_ns": 47968445, + "decode_ns": 13818745, + "arrow_writer_peak_bytes": 476208 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "bytes": 353548, + "allocated_bytes": 389120, + "encode_ns": 47932369, + "decode_ns": 13664503, + "arrow_writer_peak_bytes": 476208 + } + ] + }, + "canonical_staging": { + "files": 20, + "dictionary": true, + "row_group_rows": 65536, + "outputs_current_candidate": [ + { + "codec": "UNCOMPRESSED", + "bytes": 514282, + "allocated_bytes": 561152, + "encode_ns": 40166099, + "decode_ns": 11972048, + "arrow_writer_peak_bytes": 476208 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "bytes": 353548, + "allocated_bytes": 389120, + "encode_ns": 47764367, + "decode_ns": 13679176, + "arrow_writer_peak_bytes": 476208 + } + ] + }, + "replay": { + "files": 20, + "dictionary": false, + "row_group_rows": 8192, + "outputs_current_candidate": [ + { + "codec": "UNCOMPRESSED", + "bytes": 610614, + "allocated_bytes": 655360, + "encode_ns": 23595804, + "decode_ns": 9456465, + "arrow_writer_peak_bytes": 124168 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "bytes": 335122, + "allocated_bytes": 368640, + "encode_ns": 31759745, + "decode_ns": 11581560, + "arrow_writer_peak_bytes": 124168 + } + ] + }, + "other_permanent": { + "files": 20, + "dictionary": true, + "row_group_rows": 1048576, + "outputs_current_candidate": [ + { + "codec": "UNCOMPRESSED", + "bytes": 514282, + "allocated_bytes": 561152, + "encode_ns": 40105989, + "decode_ns": 11931720, + "arrow_writer_peak_bytes": 476208 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "bytes": 353548, + "allocated_bytes": 389120, + "encode_ns": 47621635, + "decode_ns": 13670812, + "arrow_writer_peak_bytes": 476208 + } + ] + } + }, + "mutation": { + "construction": { + "files": 21, + "dictionary": true, + "row_group_rows": 1048576, + "outputs_current_candidate": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "bytes": 371901, + "allocated_bytes": 409600, + "encode_ns": 49242728, + "decode_ns": 14326423, + "arrow_writer_peak_bytes": 476208 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "bytes": 371901, + "allocated_bytes": 409600, + "encode_ns": 49000800, + "decode_ns": 14206189, + "arrow_writer_peak_bytes": 476208 + } + ] + }, + "canonical_staging": { + "files": 21, + "dictionary": true, + "row_group_rows": 65536, + "outputs_current_candidate": [ + { + "codec": "UNCOMPRESSED", + "bytes": 538310, + "allocated_bytes": 585728, + "encode_ns": 41576448, + "decode_ns": 12534720, + "arrow_writer_peak_bytes": 476208 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "bytes": 371901, + "allocated_bytes": 409600, + "encode_ns": 48955879, + "decode_ns": 14215943, + "arrow_writer_peak_bytes": 476208 + } + ] + }, + "replay": { + "files": 21, + "dictionary": false, + "row_group_rows": 8192, + "outputs_current_candidate": [ + { + "codec": "UNCOMPRESSED", + "bytes": 633594, + "allocated_bytes": 679936, + "encode_ns": 24594050, + "decode_ns": 9804565, + "arrow_writer_peak_bytes": 124168 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "bytes": 352306, + "allocated_bytes": 389120, + "encode_ns": 32789601, + "decode_ns": 11951291, + "arrow_writer_peak_bytes": 124168 + } + ] + }, + "other_permanent": { + "files": 21, + "dictionary": true, + "row_group_rows": 1048576, + "outputs_current_candidate": [ + { + "codec": "UNCOMPRESSED", + "bytes": 538310, + "allocated_bytes": 585728, + "encode_ns": 41221889, + "decode_ns": 12439258, + "arrow_writer_peak_bytes": 476208 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "bytes": 371901, + "allocated_bytes": 409600, + "encode_ns": 49001192, + "decode_ns": 14234441, + "arrow_writer_peak_bytes": 476208 + } + ] + } + }, + "compaction": { + "construction": { + "files": 19, + "dictionary": true, + "row_group_rows": 1048576, + "outputs_current_candidate": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "bytes": 311481, + "allocated_bytes": 344064, + "encode_ns": 45045593, + "decode_ns": 13003200, + "arrow_writer_peak_bytes": 1113580 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "bytes": 311481, + "allocated_bytes": 344064, + "encode_ns": 44736552, + "decode_ns": 12743021, + "arrow_writer_peak_bytes": 1113580 + } + ] + }, + "canonical_staging": { + "files": 19, + "dictionary": true, + "row_group_rows": 65536, + "outputs_current_candidate": [ + { + "codec": "UNCOMPRESSED", + "bytes": 504206, + "allocated_bytes": 548864, + "encode_ns": 38426038, + "decode_ns": 11035052, + "arrow_writer_peak_bytes": 1113580 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "bytes": 311481, + "allocated_bytes": 344064, + "encode_ns": 44247174, + "decode_ns": 12712363, + "arrow_writer_peak_bytes": 1113580 + } + ] + }, + "replay": { + "files": 19, + "dictionary": false, + "row_group_rows": 8192, + "outputs_current_candidate": [ + { + "codec": "UNCOMPRESSED", + "bytes": 625761, + "allocated_bytes": 671744, + "encode_ns": 23033593, + "decode_ns": 8655021, + "arrow_writer_peak_bytes": 687888 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "bytes": 295758, + "allocated_bytes": 331776, + "encode_ns": 29864270, + "decode_ns": 10800197, + "arrow_writer_peak_bytes": 687888 + } + ] + }, + "other_permanent": { + "files": 19, + "dictionary": true, + "row_group_rows": 1048576, + "outputs_current_candidate": [ + { + "codec": "UNCOMPRESSED", + "bytes": 504206, + "allocated_bytes": 548864, + "encode_ns": 38272145, + "decode_ns": 11154565, + "arrow_writer_peak_bytes": 1113580 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "bytes": 311481, + "allocated_bytes": 344064, + "encode_ns": 44200006, + "decode_ns": 12698836, + "arrow_writer_peak_bytes": 1113580 + } + ] + } + } + } +} diff --git a/tools/bazel/drift/cargo_feature_fingerprint.json b/tools/bazel/drift/cargo_feature_fingerprint.json index a806748f7..f16ddc73f 100644 --- a/tools/bazel/drift/cargo_feature_fingerprint.json +++ b/tools/bazel/drift/cargo_feature_fingerprint.json @@ -1,6 +1,6 @@ { "schema": "graphforge.cargo-feature-fingerprint.v1", - "sha256": "1f73c7e20543f5cdb8657e4e3a8d0a9cace73beff21fb280429fd4077c4bcf58", + "sha256": "3a90505cee03086fdc15ea2999389015255f7e47e1f03ecc5fd3eba978318b08", "entries": [ { "name": "graphforge-api", @@ -2339,6 +2339,15 @@ "uses_default_features": true, "kind": "dev", "target": null + }, + { + "name": "zstd", + "req": "=0.13.3", + "features": [], + "optional": false, + "uses_default_features": false, + "kind": "dev", + "target": null } ] }, From 928c82e5594e168b00593b81994c45f83345f1eb Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Thu, 10 Sep 2026 07:57:25 +0000 Subject: [PATCH 4/4] docs(storage): record integrated permanent encoding resource tradeoffs --- .../permanent-storage-assessment.md | 52 + .../evidence/permanent-parquet-1213.json | 1751 +++++++++++++++++ 2 files changed, 1803 insertions(+) create mode 100644 docs/development/evidence/permanent-parquet-1213.json diff --git a/docs/book/architecture/permanent-storage-assessment.md b/docs/book/architecture/permanent-storage-assessment.md index cc1753fd2..488ad3ab3 100644 --- a/docs/book/architecture/permanent-storage-assessment.md +++ b/docs/book/architecture/permanent-storage-assessment.md @@ -451,6 +451,14 @@ Small fragments can grow with Zstd; the one-row dictionaries-on case grew from 10,017 to 10,722 bytes. Wide replay-profile output fell from 502,549 to 26,794 bytes; the narrow random replay profile fell from 1,576,490 to 1,119,732 bytes. +The codec tradeoff is measurable even where storage improves. In the paired +wide replay case, encoding elapsed time rose from 4.711 to 6.419 ms and decoding +from 2.272 to 2.889 ms. For 65,537 narrow random rows, encoding rose from 36.107 +to 44.281 ms and decoding from 3.198 to 6.286 ms. These are single-run elapsed +measurements, not CPU guarantees or benchmark distributions. The raw evidence +retains per-profile results and whole-process user/system CPU observations; +Zstd is selected for durable storage despite these measured codec costs. + Deterministic codec-test ceilings are 2 MiB per output, 8 MiB combined temporary allocation per pair, and 8 MiB of `ArrowWriter::memory_size()`. The latter omits native codec allocation and completed metadata, so it is deliberately not used @@ -494,3 +502,47 @@ acceptance, one-byte-below rejection, schema/full-width values, row order, row-count authority, cleanup and unchanged source bytes. Reader admission also covers seven one-row large-string pages, repeated values and batches spanning several tiny row groups, with rejection immediately below the computed bound. + +### Integrated result and regression gates + +Implementation source `c16397d2cd7aeb01411f5cbb51c433154d3c3ef9` was measured +with the same prebuilt public fixture after other builds/tests finished. +[`permanent-parquet-1213.json`](../../development/evidence/permanent-parquet-1213.json) +retains actual column-codec/encoding counts, output inventories and process observations. + +| Published stage | Baseline Parquet bytes / allocated bytes | Shared policy bytes / allocated bytes | +| --- | ---: | ---: | +| Construction | 353,548 / 389,120 | 353,548 / 389,120 | +| Mutation | 377,487 / 413,696 | 371,901 / 409,600 | +| Compaction | 537,467 / 569,344 | 303,718 / 335,872 | + +Every published column uses Zstd (99 construction, 102 mutation and 73 +compaction columns). Changed compaction files retain dictionaries-off encoding +and at most 8,192 rows per group. The public fixture enforces logical/allocated +ceilings of 400/448 KiB for construction, 416/480 KiB for mutation and 352/400 KiB +for compaction, in addition to exact values after reopen and portable round trips. + +The final process took 12.99 s elapsed, 12.41 s user and 0.83 s system, with +159,492 KiB peak RSS, versus baseline 12.48/11.89/0.84 s and 156,284 KiB. Separate +syscall observations fell from 179,612,114 to 163,149,583 read bytes and from +42,717,623 to 40,187,215 write bytes; both had 3,109 successful fsync calls and +no traced syscall errors. OS filesystem output fell from 90,752 to 85,904 +512-byte blocks; input remained 50,640 blocks. Sampled unique-inode workspace +allocation fell from 8,749,056 to 7,798,784 bytes. These single-host observations +include startup, queries, authentication, portable operations and test output; +the candidate additionally inspects pre-compaction inventory and prints file +digests. They are not CPU, RSS or temporary-disk guarantees. The separately +bounded low-memory IPC stream is measured directly, since pathname sampling +cannot see its unlinked file on Unix. + +Validation covers all 20 public publishing fixtures; 734 API unit tests; three +public retained-data semantic-composition certification tests; ten compaction +and 16 journal integration tests; and 20 replay-focused unit tests. The API +fixtures inspect published knowledge, epistemic, provenance, vector, projection +and restoration output, including the restoration `created_by` marker. Existing +recovery, cancellation, active-snapshot, authentication and exact retry tests +remain active. Targeted native Bazel certification/journal/compaction tests, +workspace Clippy, formatting, fast pre-push and gate-registry checks passed. +The full local storage aggregate passed 1,081 tests with two existing ignores; +one unchanged test hardcodes `/tmp`, where this host's tmpfs fails filesystem +admission. Required native Bazel CI remains the merge gate. diff --git a/docs/development/evidence/permanent-parquet-1213.json b/docs/development/evidence/permanent-parquet-1213.json new file mode 100644 index 000000000..2da5248df --- /dev/null +++ b/docs/development/evidence/permanent-parquet-1213.json @@ -0,0 +1,1751 @@ +{ + "issue": 1213, + "phase": "shared permanent encoding policy", + "source_commit": "c16397d2cd7aeb01411f5cbb51c433154d3c3ef9", + "baseline_source_commit": "60ffdca983ed1ad4b2acd4edfe2605f3e85d6e7e", + "production_fixture": { + "compaction": { + "files": [ + { + "allocated_bytes": 12288, + "bytes": 11438, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000000.parquet", + "row_groups": [ + { + "rows": 513, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "6426d5960374171a1ae07adcd54696a064ac5fcd8cab147da056e70a1ca41a2e" + }, + { + "allocated_bytes": 12288, + "bytes": 11430, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000001.parquet", + "row_groups": [ + { + "rows": 511, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "54c2076e9ce5bf82791a4888d548542400945da301d08f484aac534197967186" + }, + { + "allocated_bytes": 12288, + "bytes": 11453, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000002.parquet", + "row_groups": [ + { + "rows": 511, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "c7c54c037f5316d1b24d75f2acb73296cfe83072276993d82d76c2588e45af56" + }, + { + "allocated_bytes": 12288, + "bytes": 11393, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000003.parquet", + "row_groups": [ + { + "rows": 513, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "8f6af7bdf0d304d7528070f9c4ec2b759bf67d37bc6ccee7a668090914ba582e" + }, + { + "allocated_bytes": 12288, + "bytes": 11521, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000004.parquet", + "row_groups": [ + { + "rows": 496, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "9ff8805a86f110c945e8d6e50c60218f01b66f3140a49836224067b1fd553aa7" + }, + { + "allocated_bytes": 16384, + "bytes": 13847, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000005.parquet", + "row_groups": [ + { + "rows": 528, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "378d9e433031bbf86163b4985013cea81733caaad5d76aa3c6ec18df9c2d4ce4" + }, + { + "allocated_bytes": 12288, + "bytes": 12171, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000006.parquet", + "row_groups": [ + { + "rows": 528, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "9ac4796b869fef81c40909d46d5512f1029500640060abf9d9a7b097cebe761a" + }, + { + "allocated_bytes": 16384, + "bytes": 13081, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000007.parquet", + "row_groups": [ + { + "rows": 496, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "8c6e9847c29e81718d741368d4f07c8a9f4355026151121e5fdeb04ae245f587" + }, + { + "allocated_bytes": 4096, + "bytes": 1906, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000008.parquet", + "row_groups": [ + { + "rows": 1, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "ebe7d7b6e22e677493b0155923e6c6344155080c24c0909779f37ccababadbbb" + }, + { + "allocated_bytes": 12288, + "bytes": 11133, + "codec_pairs": null, + "edge_id_order": null, + "path": "properties/r-caf7594a72aef78670c886a56687e31c55908c136dc05830aca6ace2614f9395/00000000000000000001-00000000000000000000.parquet", + "row_groups": [ + { + "rows": 512, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "32fd469f57dd6bfa9f226a40d390d3e2b4462125c9947a61b629e5b8f5678860" + }, + { + "allocated_bytes": 12288, + "bytes": 11140, + "codec_pairs": null, + "edge_id_order": null, + "path": "properties/r-caf7594a72aef78670c886a56687e31c55908c136dc05830aca6ace2614f9395/00000000000000000001-00000000000000000001.parquet", + "row_groups": [ + { + "rows": 512, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "c24458d0f57d778bf1730b68bf8fdf9a20d69747c1b786e80225b8e78120316c" + }, + { + "allocated_bytes": 20480, + "bytes": 18233, + "codec_pairs": null, + "edge_id_order": null, + "path": "properties/r-caf7594a72aef78670c886a56687e31c55908c136dc05830aca6ace2614f9395/00000000000000000002-00000000000000000000.parquet", + "row_groups": [ + { + "rows": 877, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "b107f70d13ccbacb91f4d4094f899f61bdb9929e97040d013190c0319d5a291b" + }, + { + "allocated_bytes": 4096, + "bytes": 1571, + "codec_pairs": null, + "edge_id_order": null, + "path": "properties/r-caf7594a72aef78670c886a56687e31c55908c136dc05830aca6ace2614f9395/00000000000000000003-00000000000000000000.parquet", + "row_groups": [ + { + "rows": 1, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 3 + } + ] + } + ], + "sha256": "092ac7eb1328c45ef8c733e26820105a83ffa40f7fc61a2ce7d1460bf2cd7b00" + }, + { + "allocated_bytes": 135168, + "bytes": 134981, + "codec_pairs": null, + "edge_id_order": { + "first": 1, + "first_inversion": null, + "last": 4097, + "prefix": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12 + ], + "rows": 4097 + }, + "path": "topology/edges/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22.parquet", + "row_groups": [ + { + "rows": 4097, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 8 + } + ] + } + ], + "sha256": "aea395166ad4b663296aebfc9dfc297fdb9f4891bf7ab770111fd2581e2534e7" + }, + { + "allocated_bytes": 4096, + "bytes": 1029, + "codec_pairs": null, + "edge_id_order": null, + "path": "topology/nodes.parquet", + "row_groups": [], + "sha256": "4946d3f301537616c0b9c7a1491f04f08ef6ef4de6f38d91621e3efa0d47c310" + }, + { + "allocated_bytes": 24576, + "bytes": 21659, + "codec_pairs": null, + "edge_id_order": null, + "path": "topology/nodes/00000000000000000001-00000000000000001024.parquet", + "row_groups": [ + { + "rows": 1024, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 5 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 1 + } + ] + } + ], + "sha256": "6a855197ee6494f49fd352521a707636938812f844af407bd7046fb715df902c" + }, + { + "allocated_bytes": 4096, + "bytes": 2229, + "codec_pairs": null, + "edge_id_order": null, + "path": "topology/nodes/00000000000000001025-00000000000000001025.parquet", + "row_groups": [ + { + "rows": 1, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 5 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 1 + } + ] + } + ], + "sha256": "460fb043cfaeaed1939a3e82c608c1bfe3b7760312254db67ae0249bc3e67b74" + }, + { + "allocated_bytes": 4096, + "bytes": 2691, + "codec_pairs": null, + "edge_id_order": null, + "path": "topology/runtime_catalog.parquet", + "row_groups": [ + { + "rows": 15, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 7 + } + ] + } + ], + "sha256": "f5982012ffdf0e0fc6c00f231ce6001fd248d06db6661e7dd4d5e8465993a2d9" + }, + { + "allocated_bytes": 4096, + "bytes": 812, + "codec_pairs": null, + "edge_id_order": null, + "path": "topology/surrogate_tails.parquet", + "row_groups": [ + { + "rows": 1, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + } + ] + } + ], + "sha256": "06c3d7d0ae09ea20dae5f1c76e257d42063068612841d1936fa4794b1a0834c6" + } + ], + "ownership": "generation_graph_tree", + "parquet_allocated_bytes": 335872, + "parquet_bytes": 303718, + "zstd_columns": 73, + "uncompressed_columns": 0 + }, + "compaction_ns": 504391277, + "compaction_report": "GraphDeltaCompactionReport { dry_run: false, input_generation_uuid: 9ca0e18c-69ad-8b6a-a5d6-6532bbb0fd79, output_generation_uuid: Some(00000000-0000-0000-0000-00000001d9da), input_runs: 1, compacted_runs: 1, retained_suffix_runs: 0, input_rows: 1, output_rows: 5122, input_bytes: 340, output_bytes: 467530, spill_bytes: 0, peak_memory_bytes: 898, elapsed_ms: 503, state_fingerprint: [177, 71, 160, 157, 78, 40, 213, 248, 180, 126, 157, 177, 105, 254, 1, 53, 138, 117, 115, 177, 52, 124, 86, 28, 111, 157, 46, 168, 123, 209, 24, 196], publication: Some(ProjectPublicationReceipt { transaction_uuid: 00000000-0000-0000-0000-00000001d9d9, generation_uuid: 00000000-0000-0000-0000-00000001d9da, generation_manifest_sha256: [78, 67, 223, 220, 44, 210, 105, 57, 37, 202, 9, 232, 226, 168, 45, 51, 59, 22, 132, 212, 71, 77, 3, 223, 13, 63, 170, 248, 167, 112, 100, 179], idempotent_replay: false }), cleanup: None }", + "construction": { + "files": [ + { + "allocated_bytes": 12288, + "bytes": 11438, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000000.parquet", + "row_groups": [ + { + "rows": 513, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "6426d5960374171a1ae07adcd54696a064ac5fcd8cab147da056e70a1ca41a2e" + }, + { + "allocated_bytes": 12288, + "bytes": 11430, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000001.parquet", + "row_groups": [ + { + "rows": 511, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "54c2076e9ce5bf82791a4888d548542400945da301d08f484aac534197967186" + }, + { + "allocated_bytes": 12288, + "bytes": 11453, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000002.parquet", + "row_groups": [ + { + "rows": 511, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "c7c54c037f5316d1b24d75f2acb73296cfe83072276993d82d76c2588e45af56" + }, + { + "allocated_bytes": 12288, + "bytes": 11393, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000003.parquet", + "row_groups": [ + { + "rows": 513, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "8f6af7bdf0d304d7528070f9c4ec2b759bf67d37bc6ccee7a668090914ba582e" + }, + { + "allocated_bytes": 12288, + "bytes": 11521, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000004.parquet", + "row_groups": [ + { + "rows": 496, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "9ff8805a86f110c945e8d6e50c60218f01b66f3140a49836224067b1fd553aa7" + }, + { + "allocated_bytes": 16384, + "bytes": 13847, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000005.parquet", + "row_groups": [ + { + "rows": 528, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "378d9e433031bbf86163b4985013cea81733caaad5d76aa3c6ec18df9c2d4ce4" + }, + { + "allocated_bytes": 12288, + "bytes": 12171, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000006.parquet", + "row_groups": [ + { + "rows": 528, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "9ac4796b869fef81c40909d46d5512f1029500640060abf9d9a7b097cebe761a" + }, + { + "allocated_bytes": 16384, + "bytes": 13081, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000007.parquet", + "row_groups": [ + { + "rows": 496, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "8c6e9847c29e81718d741368d4f07c8a9f4355026151121e5fdeb04ae245f587" + }, + { + "allocated_bytes": 4096, + "bytes": 1906, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000008.parquet", + "row_groups": [ + { + "rows": 1, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "ebe7d7b6e22e677493b0155923e6c6344155080c24c0909779f37ccababadbbb" + }, + { + "allocated_bytes": 12288, + "bytes": 11133, + "codec_pairs": null, + "edge_id_order": null, + "path": "properties/r-caf7594a72aef78670c886a56687e31c55908c136dc05830aca6ace2614f9395/00000000000000000001-00000000000000000000.parquet", + "row_groups": [ + { + "rows": 512, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "32fd469f57dd6bfa9f226a40d390d3e2b4462125c9947a61b629e5b8f5678860" + }, + { + "allocated_bytes": 12288, + "bytes": 11140, + "codec_pairs": null, + "edge_id_order": null, + "path": "properties/r-caf7594a72aef78670c886a56687e31c55908c136dc05830aca6ace2614f9395/00000000000000000001-00000000000000000001.parquet", + "row_groups": [ + { + "rows": 512, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "c24458d0f57d778bf1730b68bf8fdf9a20d69747c1b786e80225b8e78120316c" + }, + { + "allocated_bytes": 53248, + "bytes": 51136, + "codec_pairs": null, + "edge_id_order": { + "first": 1, + "first_inversion": null, + "last": 1024, + "prefix": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12 + ], + "rows": 1024 + }, + "path": "topology/edges/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000001024.parquet", + "row_groups": [ + { + "rows": 1024, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 5 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 3 + } + ] + } + ], + "sha256": "36e122d4d9068ca6831ee98bd7d0cf4e92aac4fd0e5a40ae3a013873d7cbe923" + }, + { + "allocated_bytes": 53248, + "bytes": 50442, + "codec_pairs": null, + "edge_id_order": { + "first": 1025, + "first_inversion": null, + "last": 2048, + "prefix": [ + 1025, + 1026, + 1027, + 1028, + 1029, + 1030, + 1031, + 1032, + 1033, + 1034, + 1035, + 1036 + ], + "rows": 1024 + }, + "path": "topology/edges/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000001025-00000000000000002048.parquet", + "row_groups": [ + { + "rows": 1024, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 5 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 3 + } + ] + } + ], + "sha256": "7bcba5232b94533f1a835e5ae30b3622d9ba12d21a5798f6c99a6fa843e26e96" + }, + { + "allocated_bytes": 53248, + "bytes": 50734, + "codec_pairs": null, + "edge_id_order": { + "first": 2049, + "first_inversion": null, + "last": 3072, + "prefix": [ + 2049, + 2050, + 2051, + 2052, + 2053, + 2054, + 2055, + 2056, + 2057, + 2058, + 2059, + 2060 + ], + "rows": 1024 + }, + "path": "topology/edges/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000002049-00000000000000003072.parquet", + "row_groups": [ + { + "rows": 1024, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 5 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 3 + } + ] + } + ], + "sha256": "afcafebdd82cca1ff4aae9e1d6d64e48a9848a590c424a7698e3c1b7d0676c0b" + }, + { + "allocated_bytes": 53248, + "bytes": 50611, + "codec_pairs": null, + "edge_id_order": { + "first": 3073, + "first_inversion": null, + "last": 4096, + "prefix": [ + 3073, + 3074, + 3075, + 3076, + 3077, + 3078, + 3079, + 3080, + 3081, + 3082, + 3083, + 3084 + ], + "rows": 1024 + }, + "path": "topology/edges/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000003073-00000000000000004096.parquet", + "row_groups": [ + { + "rows": 1024, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 5 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 3 + } + ] + } + ], + "sha256": "c3c96692b607746c2fa996a7e9296ee98db22f75a00bc2de758ade73d8c99642" + }, + { + "allocated_bytes": 4096, + "bytes": 2841, + "codec_pairs": null, + "edge_id_order": { + "first": 4097, + "first_inversion": null, + "last": 4097, + "prefix": [ + 4097 + ], + "rows": 1 + }, + "path": "topology/edges/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000004097-00000000000000004097.parquet", + "row_groups": [ + { + "rows": 1, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 5 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 3 + } + ] + } + ], + "sha256": "4b9842c66aed74336888fb776722aa8e846c892b4a0492956d3f41688a23dc0c" + }, + { + "allocated_bytes": 24576, + "bytes": 21659, + "codec_pairs": null, + "edge_id_order": null, + "path": "topology/nodes/00000000000000000001-00000000000000001024.parquet", + "row_groups": [ + { + "rows": 1024, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 5 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 1 + } + ] + } + ], + "sha256": "6a855197ee6494f49fd352521a707636938812f844af407bd7046fb715df902c" + }, + { + "allocated_bytes": 4096, + "bytes": 2229, + "codec_pairs": null, + "edge_id_order": null, + "path": "topology/nodes/00000000000000001025-00000000000000001025.parquet", + "row_groups": [ + { + "rows": 1, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 5 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 1 + } + ] + } + ], + "sha256": "460fb043cfaeaed1939a3e82c608c1bfe3b7760312254db67ae0249bc3e67b74" + }, + { + "allocated_bytes": 4096, + "bytes": 2571, + "codec_pairs": null, + "edge_id_order": null, + "path": "topology/runtime_catalog.parquet", + "row_groups": [ + { + "rows": 10, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 7 + } + ] + } + ], + "sha256": "9ecfe0159015a8ff31fea6843f44adc38ffffe41ec3a5631264c138832a25b5b" + }, + { + "allocated_bytes": 4096, + "bytes": 812, + "codec_pairs": null, + "edge_id_order": null, + "path": "topology/surrogate_tails.parquet", + "row_groups": [ + { + "rows": 1, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + } + ] + } + ], + "sha256": "06c3d7d0ae09ea20dae5f1c76e257d42063068612841d1936fa4794b1a0834c6" + } + ], + "ownership": "cas", + "parquet_allocated_bytes": 389120, + "parquet_bytes": 353548, + "zstd_columns": 99, + "uncompressed_columns": 0 + }, + "mutation": { + "files": [ + { + "allocated_bytes": 12288, + "bytes": 11438, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000000.parquet", + "row_groups": [ + { + "rows": 513, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "6426d5960374171a1ae07adcd54696a064ac5fcd8cab147da056e70a1ca41a2e" + }, + { + "allocated_bytes": 12288, + "bytes": 11430, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000001.parquet", + "row_groups": [ + { + "rows": 511, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "54c2076e9ce5bf82791a4888d548542400945da301d08f484aac534197967186" + }, + { + "allocated_bytes": 12288, + "bytes": 11453, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000002.parquet", + "row_groups": [ + { + "rows": 511, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "c7c54c037f5316d1b24d75f2acb73296cfe83072276993d82d76c2588e45af56" + }, + { + "allocated_bytes": 12288, + "bytes": 11393, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000003.parquet", + "row_groups": [ + { + "rows": 513, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "8f6af7bdf0d304d7528070f9c4ec2b759bf67d37bc6ccee7a668090914ba582e" + }, + { + "allocated_bytes": 12288, + "bytes": 11521, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000004.parquet", + "row_groups": [ + { + "rows": 496, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "9ff8805a86f110c945e8d6e50c60218f01b66f3140a49836224067b1fd553aa7" + }, + { + "allocated_bytes": 16384, + "bytes": 13847, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000005.parquet", + "row_groups": [ + { + "rows": 528, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "378d9e433031bbf86163b4985013cea81733caaad5d76aa3c6ec18df9c2d4ce4" + }, + { + "allocated_bytes": 12288, + "bytes": 12171, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000006.parquet", + "row_groups": [ + { + "rows": 528, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "9ac4796b869fef81c40909d46d5512f1029500640060abf9d9a7b097cebe761a" + }, + { + "allocated_bytes": 16384, + "bytes": 13081, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000007.parquet", + "row_groups": [ + { + "rows": 496, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "8c6e9847c29e81718d741368d4f07c8a9f4355026151121e5fdeb04ae245f587" + }, + { + "allocated_bytes": 4096, + "bytes": 1906, + "codec_pairs": null, + "edge_id_order": null, + "path": "edge_properties/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000008.parquet", + "row_groups": [ + { + "rows": 1, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "ebe7d7b6e22e677493b0155923e6c6344155080c24c0909779f37ccababadbbb" + }, + { + "allocated_bytes": 12288, + "bytes": 11133, + "codec_pairs": null, + "edge_id_order": null, + "path": "properties/r-caf7594a72aef78670c886a56687e31c55908c136dc05830aca6ace2614f9395/00000000000000000001-00000000000000000000.parquet", + "row_groups": [ + { + "rows": 512, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "32fd469f57dd6bfa9f226a40d390d3e2b4462125c9947a61b629e5b8f5678860" + }, + { + "allocated_bytes": 12288, + "bytes": 11140, + "codec_pairs": null, + "edge_id_order": null, + "path": "properties/r-caf7594a72aef78670c886a56687e31c55908c136dc05830aca6ace2614f9395/00000000000000000001-00000000000000000001.parquet", + "row_groups": [ + { + "rows": 512, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "c24458d0f57d778bf1730b68bf8fdf9a20d69747c1b786e80225b8e78120316c" + }, + { + "allocated_bytes": 20480, + "bytes": 18233, + "codec_pairs": null, + "edge_id_order": null, + "path": "properties/r-caf7594a72aef78670c886a56687e31c55908c136dc05830aca6ace2614f9395/00000000000000000002-00000000000000000000.parquet", + "row_groups": [ + { + "rows": 877, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 1 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 2 + } + ] + } + ], + "sha256": "b107f70d13ccbacb91f4d4094f899f61bdb9929e97040d013190c0319d5a291b" + }, + { + "allocated_bytes": 53248, + "bytes": 51136, + "codec_pairs": null, + "edge_id_order": { + "first": 1, + "first_inversion": null, + "last": 1024, + "prefix": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12 + ], + "rows": 1024 + }, + "path": "topology/edges/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000001024.parquet", + "row_groups": [ + { + "rows": 1024, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 5 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 3 + } + ] + } + ], + "sha256": "36e122d4d9068ca6831ee98bd7d0cf4e92aac4fd0e5a40ae3a013873d7cbe923" + }, + { + "allocated_bytes": 53248, + "bytes": 50442, + "codec_pairs": null, + "edge_id_order": { + "first": 1025, + "first_inversion": null, + "last": 2048, + "prefix": [ + 1025, + 1026, + 1027, + 1028, + 1029, + 1030, + 1031, + 1032, + 1033, + 1034, + 1035, + 1036 + ], + "rows": 1024 + }, + "path": "topology/edges/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000001025-00000000000000002048.parquet", + "row_groups": [ + { + "rows": 1024, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 5 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 3 + } + ] + } + ], + "sha256": "7bcba5232b94533f1a835e5ae30b3622d9ba12d21a5798f6c99a6fa843e26e96" + }, + { + "allocated_bytes": 53248, + "bytes": 50734, + "codec_pairs": null, + "edge_id_order": { + "first": 2049, + "first_inversion": null, + "last": 3072, + "prefix": [ + 2049, + 2050, + 2051, + 2052, + 2053, + 2054, + 2055, + 2056, + 2057, + 2058, + 2059, + 2060 + ], + "rows": 1024 + }, + "path": "topology/edges/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000002049-00000000000000003072.parquet", + "row_groups": [ + { + "rows": 1024, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 5 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 3 + } + ] + } + ], + "sha256": "afcafebdd82cca1ff4aae9e1d6d64e48a9848a590c424a7698e3c1b7d0676c0b" + }, + { + "allocated_bytes": 53248, + "bytes": 50611, + "codec_pairs": null, + "edge_id_order": { + "first": 3073, + "first_inversion": null, + "last": 4096, + "prefix": [ + 3073, + 3074, + 3075, + 3076, + 3077, + 3078, + 3079, + 3080, + 3081, + 3082, + 3083, + 3084 + ], + "rows": 1024 + }, + "path": "topology/edges/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000003073-00000000000000004096.parquet", + "row_groups": [ + { + "rows": 1024, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 5 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 3 + } + ] + } + ], + "sha256": "c3c96692b607746c2fa996a7e9296ee98db22f75a00bc2de758ade73d8c99642" + }, + { + "allocated_bytes": 4096, + "bytes": 2841, + "codec_pairs": null, + "edge_id_order": { + "first": 4097, + "first_inversion": null, + "last": 4097, + "prefix": [ + 4097 + ], + "rows": 1 + }, + "path": "topology/edges/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000004097-00000000000000004097.parquet", + "row_groups": [ + { + "rows": 1, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 5 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 3 + } + ] + } + ], + "sha256": "4b9842c66aed74336888fb776722aa8e846c892b4a0492956d3f41688a23dc0c" + }, + { + "allocated_bytes": 24576, + "bytes": 21659, + "codec_pairs": null, + "edge_id_order": null, + "path": "topology/nodes/00000000000000000001-00000000000000001024.parquet", + "row_groups": [ + { + "rows": 1024, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 5 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 1 + } + ] + } + ], + "sha256": "6a855197ee6494f49fd352521a707636938812f844af407bd7046fb715df902c" + }, + { + "allocated_bytes": 4096, + "bytes": 2229, + "codec_pairs": null, + "edge_id_order": null, + "path": "topology/nodes/00000000000000001025-00000000000000001025.parquet", + "row_groups": [ + { + "rows": 1, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 5 + }, + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE]", + "columns": 1 + } + ] + } + ], + "sha256": "460fb043cfaeaed1939a3e82c608c1bfe3b7760312254db67ae0249bc3e67b74" + }, + { + "allocated_bytes": 4096, + "bytes": 2691, + "codec_pairs": null, + "edge_id_order": null, + "path": "topology/runtime_catalog.parquet", + "row_groups": [ + { + "rows": 15, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 7 + } + ] + } + ], + "sha256": "49ac1b20c20b6963cebfe9222760734bbc71839a12109927382d799ffa9a00cf" + }, + { + "allocated_bytes": 4096, + "bytes": 812, + "codec_pairs": null, + "edge_id_order": null, + "path": "topology/surrogate_tails.parquet", + "row_groups": [ + { + "rows": 1, + "column_encoding_counts": [ + { + "codec": "ZSTD(ZstdLevel(1))", + "encodings": "[PLAIN, RLE, RLE_DICTIONARY]", + "columns": 2 + } + ] + } + ], + "sha256": "06c3d7d0ae09ea20dae5f1c76e257d42063068612841d1936fa4794b1a0834c6" + } + ], + "ownership": "generation_graph_tree", + "parquet_allocated_bytes": 409600, + "parquet_bytes": 371901, + "zstd_columns": 102, + "uncompressed_columns": 0 + }, + "mutation_ns": 1320885812 + }, + "production_runtime": { + "elapsed_seconds": 12.99, + "user_seconds": 12.41, + "system_seconds": 0.83, + "peak_rss_kib": 159492, + "filesystem_input_512_byte_blocks": 50640, + "filesystem_output_512_byte_blocks": 85904, + "exit_status": 0 + }, + "production_syscall_io": { + "syscalls": { + "read": { + "calls": 27329, + "bytes": 141332523 + }, + "pread64": { + "calls": 29663, + "bytes": 13585043 + }, + "write": { + "calls": 4192, + "bytes": 31955198 + }, + "fsync": { + "calls": 3109, + "bytes": 0 + }, + "copy_file_range": { + "calls": 980, + "bytes": 8232017 + } + }, + "read_bytes": 163149583, + "write_bytes": 40187215, + "errors": [], + "elapsed_seconds": 24.26 + }, + "production_workspace_observation": { + "command": [ + "/home/ubuntu/code/graphforge-target-1195/debug/deps/permanent_storage_budgets-c514423ca6fb68a2", + "permanent_publishing_policy_construction_mutation_and_compaction", + "--exact", + "--nocapture", + "--test-threads=1" + ], + "sampled_unique_inode_allocated_peak_bytes": 7798784, + "sampled_path_logical_peak_bytes": 6440309, + "sampled_file_count_peak": 514, + "samples": 1274, + "requested_poll_interval_seconds": 0.005, + "maximum_observed_sample_interval_seconds": 0.016670843004249036, + "vanished_files_during_scan": 29, + "exit_status": 0, + "limitations": [ + "Sampling can miss shorter-lived allocation peaks and unlinked open files.", + "Includes retained project generations, private hydration/staging and portable artifacts, not only temporary files.", + "Hardlinks are deduplicated for allocated bytes, not path logical bytes.", + "Separate run from CPU/RSS and syscall traces; scanner adds filesystem activity." + ] + }, + "deterministic_budgets": { + "construction_parquet_logical_bytes": 409600, + "construction_parquet_allocated_bytes": 458752, + "mutation_parquet_logical_bytes": 425984, + "mutation_parquet_allocated_bytes": 491520, + "compaction_parquet_logical_bytes": 360448, + "compaction_parquet_allocated_bytes": 409600, + "topology_ladder_replay_memory_bytes": 2097152, + "topology_ladder_max_batch_rows": 7, + "topology_ladder_private_stream_bytes": 262144, + "private_stream_per_invocation_maximum_bytes": 67108864 + }, + "limitations": [ + "CPU/RSS, syscall traces and workspace sampling are separate runs. OS I/O depends on cache state.", + "Production observations include public query, reopen, export, verify and clean import, startup and test output.", + "Workspace sampling misses unlinked open files and short-lived peaks; it is not a deterministic temporary-disk ceiling.", + "Paired profiles hold schema, dictionary and row-group settings fixed; they normalize actual production batches, not the mixed writer settings of the original files.", + "ArrowWriter.memory_size excludes native codec allocations and is not process RSS. Codec tests do not establish whole-publication admission.", + "Candidate fixture additionally checks pre-compaction authenticated inventory, output encodings and deterministic payload ceilings; printed output includes file digests. These small instrumentation differences are included in process/syscall observations.", + "No process-RSS or CPU-time admission guarantee is claimed. Native codec and decoder/encoder component reservations are tested independently.", + "Per-row-group column metadata is aggregated by codec and encoding list; exact decoded values and all individual column codecs are asserted by the public test." + ] +}