feat: add --unordered to forest-cli snapshot export

.github/workflows/forest.yml

@@ -306,7 +306,29 @@
       - name: Snapshot export check v2
         run: ./scripts/tests/calibnet_export_f3_check.sh
         timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}
+  calibnet-unordered-export-check:
+    needs:
+      - build-ubuntu
+    name: Snapshot unordered export checks
+    runs-on: ubuntu-24.04
+    steps:
+      - run: lscpu
+      - uses: actions/cache@v4
+        with:
+          path: "${{ env.FIL_PROOFS_PARAMETER_CACHE }}"
+          key: proof-params-keys
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: "forest-${{ runner.os }}"
+          path: ~/.cargo/bin
+      - name: Set permissions
+        run: |
+          chmod +x ~/.cargo/bin/forest*
+      - name: Snapshot unordered export check
+        run: ./scripts/tests/calibnet_export_unordered_check.sh
+        timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}
   calibnet-no-discovery-checks:
     needs:
       - build-ubuntu
     name: Calibnet no discovery checks
@@ -577,6 +599,7 @@
       - calibnet-wallet-check
       - calibnet-export-check
       - calibnet-export-check-v2
+      - calibnet-unordered-export-check
       - calibnet-no-discovery-checks
       - calibnet-kademlia-checks
       - calibnet-eth-mapping-check

CHANGELOG.md

@@ -35,6 +35,8 @@
 
 - [#5859](https://github.com/ChainSafe/forest/pull/5859) Added size metrics for zstd frame cache and made max size configurable via `FOREST_ZSTD_FRAME_CACHE_DEFAULT_MAX_SIZE` environment variable.
 
+- [#5867](https://github.com/ChainSafe/forest/pull/5867) Added `--unordered` to `forest-cli snapshot export` for exporting `CAR` blocks in non-deterministic order for better performance with more parallelization.
+
 - [#5886](https://github.com/ChainSafe/forest/issues/5886) Add `forest-tool archive merge-f3` subcommand for merging a v1 Filecoin snapshot and an F3 snapshot into a v2 Filecoin snapshot.
 
 - [#4976](https://github.com/ChainSafe/forest/issues/4976) Add support for the `Filecoin.EthSubscribe` and `Filecoin.EthUnsubscribe` API methods to enable subscriptions to Ethereum event types: `heads` and `logs`.

scripts/tests/calibnet_export_unordered_check.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# This script is checking the correctness of 
+# the snapshot export feature.
+# It requires both the `forest` and `forest-cli` binaries to be in the PATH.
+
+set -eu
+
+source "$(dirname "$0")/harness.sh"
+
+forest_init "$@"
+
+echo "Cleaning up the initial snapshot"
+rm --force --verbose ./*.{car,car.zst,sha256sum}
+
+echo "Exporting zstd compressed snapshot with unordred graph traversal"
+$FOREST_CLI_PATH snapshot export --unordered -o unordered.forest.car.zst
+
+$FOREST_CLI_PATH shutdown --force
+
+for f in *.car.zst; do
+  echo "Inspecting archive info $f"
+  $FOREST_TOOL_PATH archive info "$f"
+  echo "Inspecting archive metadata $f"
+  $FOREST_TOOL_PATH archive metadata "$f"
+done
+
+echo "Cleanup calibnet db"
+$FOREST_TOOL_PATH db destroy --chain calibnet --force
+
+echo "Import the unordered snapshot"
+$FOREST_PATH --chain calibnet --encrypt-keystore false --halt-after-import --height=-100 --import-snapshot unordered.forest.car.zst
+
+echo "Check if Forest is able to sync"
+forest_run_node_detached
+forest_wait_api
+forest_wait_for_sync
+forest_wait_for_healthcheck_ready

src/chain/mod.rs

@@ -13,7 +13,7 @@ use crate::blocks::{Tipset, TipsetKey};
 use crate::cid_collections::CidHashSet;
 use crate::db::car::forest::{self, ForestCarFrame, finalize_frame};
 use crate::db::{SettingsStore, SettingsStoreExt};
-use crate::ipld::stream_chain;
+use crate::ipld::{stream_chain, unordered_stream_chain};
 use crate::utils::db::car_stream::{CarBlock, CarBlockWrite};
 use crate::utils::io::{AsyncWriterWithChecksum, Checksum};
 use crate::utils::multihash::MultihashCode;
@@ -31,17 +31,23 @@ use std::io::{Seek as _, SeekFrom};
 use std::sync::Arc;
 use tokio::io::{AsyncWrite, AsyncWriteExt, BufWriter};
 
+#[derive(Debug, Clone, Default)]
+pub struct ExportOptions {
+    pub skip_checksum: bool,
+    pub unordered: bool,
+    pub seen: CidHashSet,
+}
+
 pub async fn export_from_head<D: Digest>(
     db: &Arc<impl Blockstore + SettingsStore + Send + Sync + 'static>,
     lookup_depth: ChainEpochDelta,
     writer: impl AsyncWrite + Unpin,
-    seen: CidHashSet,
-    skip_checksum: bool,
+    options: Option<ExportOptions>,
 ) -> anyhow::Result<(Tipset, Option<digest::Output<D>>)> {
     let head_key = SettingsStoreExt::read_obj::<TipsetKey>(db, crate::db::setting_keys::HEAD_KEY)?
         .context("chain head key not found")?;
     let head_ts = Tipset::load_required(&db, &head_key)?;
-    let digest = export::<D>(db, &head_ts, lookup_depth, writer, seen, skip_checksum).await?;
+    let digest = export::<D>(db, &head_ts, lookup_depth, writer, options).await?;
     Ok((head_ts, digest))
 }
 
@@ -52,21 +58,10 @@ pub async fn export<D: Digest>(
     tipset: &Tipset,
     lookup_depth: ChainEpochDelta,
     writer: impl AsyncWrite + Unpin,
-    seen: CidHashSet,
-    skip_checksum: bool,
+    options: Option<ExportOptions>,
 ) -> anyhow::Result<Option<digest::Output<D>>> {
     let roots = tipset.key().to_cids();
-    export_to_forest_car::<D>(
-        roots,
-        None,
-        db,
-        tipset,
-        lookup_depth,
-        writer,
-        seen,
-        skip_checksum,
-    )
-    .await
+    export_to_forest_car::<D>(roots, None, db, tipset, lookup_depth, writer, options).await
 }
 
 /// Exports a Filecoin snapshot in v2 format
@@ -77,8 +72,7 @@ pub async fn export_v2<D: Digest>(
     tipset: &Tipset,
     lookup_depth: ChainEpochDelta,
     writer: impl AsyncWrite + Unpin,
-    seen: CidHashSet,
-    skip_checksum: bool,
+    options: Option<ExportOptions>,
 ) -> anyhow::Result<Option<digest::Output<D>>> {
     // validate f3 data
     if let Some((f3_cid, f3_data)) = &mut f3 {
@@ -131,8 +125,7 @@ pub async fn export_v2<D: Digest>(
         tipset,
         lookup_depth,
         writer,
-        seen,
-        skip_checksum,
+        options,
     )
     .await
 }
@@ -145,9 +138,14 @@ async fn export_to_forest_car<D: Digest>(
     tipset: &Tipset,
     lookup_depth: ChainEpochDelta,
     writer: impl AsyncWrite + Unpin,
-    seen: CidHashSet,
-    skip_checksum: bool,
+    options: Option<ExportOptions>,
 ) -> anyhow::Result<Option<digest::Output<D>>> {
+    let ExportOptions {
+        skip_checksum,
+        unordered,
+        seen,
+    } = options.unwrap_or_default();
+
     let stateroot_lookup_limit = tipset.epoch() - lookup_depth;
 
     // Wrap writer in optional checksum calculator
@@ -160,12 +158,25 @@ async fn export_to_forest_car<D: Digest>(
         // are small enough that keeping 1k in memory isn't a problem. Average
         // block size is between 1kb and 2kb.
         1024,
-        stream_chain(
-            Arc::clone(db),
-            tipset.clone().chain_owned(Arc::clone(db)),
-            stateroot_lookup_limit,
-        )
-        .with_seen(seen),
+        if unordered {
+            futures::future::Either::Left(
+                unordered_stream_chain(
+                    Arc::clone(db),
+                    tipset.clone().chain_owned(Arc::clone(db)),
+                    stateroot_lookup_limit,
+                )
+                .with_seen(seen),
+            )
+        } else {
+            futures::future::Either::Right(
+                stream_chain(
+                    Arc::clone(db),
+                    tipset.clone().chain_owned(Arc::clone(db)),
+                    stateroot_lookup_limit,
+                )
+                .with_seen(seen),
+            )
+        },
     );
 
     // Encode Ipld key-value pairs in zstd frames

src/chain/tests.rs

@@ -59,19 +59,10 @@ async fn test_export_inner(version: FilecoinSnapshotVersion) -> anyhow::Result<(
 
     let checksum = match version {
         FilecoinSnapshotVersion::V1 => {
-            export::<Sha256>(&db, &head, 0, &mut car_bytes, Default::default(), false).await?
+            export::<Sha256>(&db, &head, 0, &mut car_bytes, None).await?
         }
         FilecoinSnapshotVersion::V2 => {
-            export_v2::<Sha256>(
-                &db,
-                None,
-                &head,
-                0,
-                &mut car_bytes,
-                Default::default(),
-                false,
-            )
-            .await?
+            export_v2::<Sha256>(&db, None, &head, 0, &mut car_bytes, None).await?
         }
     };
 

src/cli/subcommands/snapshot_cmd.rs

@@ -36,6 +36,9 @@ pub enum SnapshotCommands {
         /// How many state-roots to include. Lower limit is 900 for `calibnet` and `mainnet`.
         #[arg(short, long)]
         depth: Option<crate::chain::ChainEpochDelta>,
+        /// Traverse chain in non-deterministic order for better performance with more parallelization.
+        #[arg(long)]
+        unordered: bool,
         /// Export snapshot in the experimental v2 format(FRC-0108).
         #[arg(long, value_enum, default_value_t = FilecoinSnapshotVersion::V1)]
         format: FilecoinSnapshotVersion,
@@ -51,6 +54,7 @@ impl SnapshotCommands {
                 dry_run,
                 tipset,
                 depth,
+                unordered,
                 format,
             } => {
                 let chain_head = ChainHead::call(&client, ()).await?;
@@ -93,6 +97,7 @@ impl SnapshotCommands {
                     recent_roots: depth.unwrap_or(SyncConfig::default().recent_state_roots),
                     output_path: temp_path.to_path_buf(),
                     tipset_keys: ApiTipsetKey(Some(chain_head.key().clone())),
+                    unordered,
                     skip_checksum,
                     dry_run,
                 };
@@ -128,10 +133,12 @@ impl SnapshotCommands {
                 pb.finish();
                 _ = handle.await;
 
-                if let Some(hash) = hash_result {
-                    save_checksum(&output_path, hash).await?;
+                if !dry_run {
+                    if let Some(hash) = hash_result {
+                        save_checksum(&output_path, hash).await?;
+                    }
+                    temp_path.persist(output_path)?;
                 }
-                temp_path.persist(output_path)?;
 
                 println!("Export completed.");
                 Ok(())

src/db/gc/snapshot.rs

@@ -43,7 +43,7 @@
 //!
 
 use crate::blocks::{Tipset, TipsetKey};
-use crate::cid_collections::CidHashSet;
+use crate::chain::ExportOptions;
 use crate::cli_shared::chain_path;
 use crate::db::car::forest::new_forest_car_temp_path_in;
 use crate::db::{
@@ -223,8 +223,10 @@ where
             &db,
             self.recent_state_roots,
             file,
-            CidHashSet::default(),
-            true,
+            Some(ExportOptions {
+                skip_checksum: true,
+                ..Default::default()
+            }),
         )
         .await?;
         let target_path = self.car_db_dir.join(format!(