diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 28b8ec29e33..ce69f8e233d 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -34,257 +34,258 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'pull_request' || github.event_name == 'merge_group' steps: - - name: Check that the pull request is not targeting the stable branch - run: test ${{ github.base_ref }} != "stable" + - name: Check that the pull request is not targeting the stable branch + run: test ${{ github.base_ref }} != "stable" release-tests-ubuntu: name: release-tests-ubuntu # Use self-hosted runners only on the sigp repo. runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "large"]') || 'ubuntu-latest' }} steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - if: env.SELF_HOSTED_RUNNERS == 'false' - uses: moonrepo/setup-rust@v1 - with: - channel: stable - cache-target: release - bins: cargo-nextest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Install Foundry (anvil) - if: env.SELF_HOSTED_RUNNERS == 'false' - uses: foundry-rs/foundry-toolchain@v1 - with: - version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d - - name: Run tests in release - run: make nextest-release - - name: Show cache stats - if: env.SELF_HOSTED_RUNNERS == 'true' - run: sccache --show-stats - release-tests-windows: - name: release-tests-windows - runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows", "CI"]') || 'windows-2019' }} - steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - if: env.SELF_HOSTED_RUNNERS == 'false' - uses: moonrepo/setup-rust@v1 - with: + - uses: actions/checkout@v3 + - name: Get latest version of stable Rust + if: env.SELF_HOSTED_RUNNERS == 'false' + uses: moonrepo/setup-rust@v1 + with: channel: stable cache-target: release bins: cargo-nextest - env: + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Install Foundry (anvil) - if: env.SELF_HOSTED_RUNNERS == 'false' - uses: foundry-rs/foundry-toolchain@v1 - with: - version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d - - name: Install make - if: env.SELF_HOSTED_RUNNERS == 'false' - run: choco install -y make -# - uses: KyleMayes/install-llvm-action@v1 -# if: env.SELF_HOSTED_RUNNERS == 'false' -# with: -# version: "16.0" -# directory: ${{ runner.temp }}/llvm - - name: Set LIBCLANG_PATH - run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV - - name: Run tests in release - run: make nextest-release - - name: Show cache stats - if: env.SELF_HOSTED_RUNNERS == 'true' - run: sccache --show-stats + - name: Install Foundry (anvil) + if: env.SELF_HOSTED_RUNNERS == 'false' + uses: foundry-rs/foundry-toolchain@v1 + with: + version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d + - name: Run tests in release + run: make nextest-release + - name: Show cache stats + if: env.SELF_HOSTED_RUNNERS == 'true' + run: sccache --show-stats + # FIXME(das): disabled for now as the c-kzg-4844 `das` branch doesn't build on windows. + # release-tests-windows: + # name: release-tests-windows + # runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows", "CI"]') || 'windows-2019' }} + # steps: + # - uses: actions/checkout@v3 + # - name: Get latest version of stable Rust + # if: env.SELF_HOSTED_RUNNERS == 'false' + # uses: moonrepo/setup-rust@v1 + # with: + # channel: stable + # cache-target: release + # bins: cargo-nextest + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # - name: Install Foundry (anvil) + # if: env.SELF_HOSTED_RUNNERS == 'false' + # uses: foundry-rs/foundry-toolchain@v1 + # with: + # version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d + # - name: Install make + # if: env.SELF_HOSTED_RUNNERS == 'false' + # run: choco install -y make + ## - uses: KyleMayes/install-llvm-action@v1 + ## if: env.SELF_HOSTED_RUNNERS == 'false' + ## with: + ## version: "16.0" + ## directory: ${{ runner.temp }}/llvm + # - name: Set LIBCLANG_PATH + # run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV + # - name: Run tests in release + # run: make nextest-release + # - name: Show cache stats + # if: env.SELF_HOSTED_RUNNERS == 'true' + # run: sccache --show-stats beacon-chain-tests: name: beacon-chain-tests # Use self-hosted runners only on the sigp repo. runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "large"]') || 'ubuntu-latest' }} env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - if: env.SELF_HOSTED_RUNNERS == 'false' - uses: moonrepo/setup-rust@v1 - with: + - uses: actions/checkout@v3 + - name: Get latest version of stable Rust + if: env.SELF_HOSTED_RUNNERS == 'false' + uses: moonrepo/setup-rust@v1 + with: channel: stable cache-target: release bins: cargo-nextest - - name: Run beacon_chain tests for all known forks - run: make test-beacon-chain - - name: Show cache stats - if: env.SELF_HOSTED_RUNNERS == 'true' - run: sccache --show-stats + - name: Run beacon_chain tests for all known forks + run: make test-beacon-chain + - name: Show cache stats + if: env.SELF_HOSTED_RUNNERS == 'true' + run: sccache --show-stats op-pool-tests: name: op-pool-tests runs-on: ubuntu-latest env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - uses: moonrepo/setup-rust@v1 - with: + - uses: actions/checkout@v3 + - name: Get latest version of stable Rust + uses: moonrepo/setup-rust@v1 + with: channel: stable cache-target: release bins: cargo-nextest - - name: Run operation_pool tests for all known forks - run: make test-op-pool + - name: Run operation_pool tests for all known forks + run: make test-op-pool network-tests: name: network-tests runs-on: ubuntu-latest env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - uses: moonrepo/setup-rust@v1 - with: - channel: stable - cache-target: release - bins: cargo-nextest - - name: Run network tests for all known forks - run: make test-network + - uses: actions/checkout@v3 + - name: Get latest version of stable Rust + uses: moonrepo/setup-rust@v1 + with: + channel: stable + cache-target: release + bins: cargo-nextest + - name: Run network tests for all known forks + run: make test-network slasher-tests: name: slasher-tests runs-on: ubuntu-latest env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - uses: moonrepo/setup-rust@v1 - with: + - uses: actions/checkout@v3 + - name: Get latest version of stable Rust + uses: moonrepo/setup-rust@v1 + with: channel: stable cache-target: release bins: cargo-nextest - - name: Run slasher tests for all supported backends - run: make test-slasher + - name: Run slasher tests for all supported backends + run: make test-slasher debug-tests-ubuntu: name: debug-tests-ubuntu # Use self-hosted runners only on the sigp repo. runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "large"]') || 'ubuntu-latest' }} env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - if: env.SELF_HOSTED_RUNNERS == 'false' - uses: moonrepo/setup-rust@v1 - with: + - uses: actions/checkout@v3 + - name: Get latest version of stable Rust + if: env.SELF_HOSTED_RUNNERS == 'false' + uses: moonrepo/setup-rust@v1 + with: channel: stable bins: cargo-nextest - - name: Install Foundry (anvil) - if: env.SELF_HOSTED_RUNNERS == 'false' - uses: foundry-rs/foundry-toolchain@v1 - with: - version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d - - name: Run tests in debug - run: make nextest-debug - - name: Show cache stats - if: env.SELF_HOSTED_RUNNERS == 'true' - run: sccache --show-stats + - name: Install Foundry (anvil) + if: env.SELF_HOSTED_RUNNERS == 'false' + uses: foundry-rs/foundry-toolchain@v1 + with: + version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d + - name: Run tests in debug + run: make nextest-debug + - name: Show cache stats + if: env.SELF_HOSTED_RUNNERS == 'true' + run: sccache --show-stats state-transition-vectors-ubuntu: name: state-transition-vectors-ubuntu runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - uses: moonrepo/setup-rust@v1 - with: + - uses: actions/checkout@v3 + - name: Get latest version of stable Rust + uses: moonrepo/setup-rust@v1 + with: channel: stable cache-target: release - - name: Run state_transition_vectors in release. - run: make run-state-transition-tests + - name: Run state_transition_vectors in release. + run: make run-state-transition-tests ef-tests-ubuntu: name: ef-tests-ubuntu # Use self-hosted runners only on the sigp repo. runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "small"]') || 'ubuntu-latest' }} env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - if: env.SELF_HOSTED_RUNNERS == 'false' - uses: moonrepo/setup-rust@v1 - with: + - uses: actions/checkout@v3 + - name: Get latest version of stable Rust + if: env.SELF_HOSTED_RUNNERS == 'false' + uses: moonrepo/setup-rust@v1 + with: channel: stable cache-target: release bins: cargo-nextest - - name: Run consensus-spec-tests with blst, milagro and fake_crypto - run: make nextest-ef - - name: Show cache stats - if: env.SELF_HOSTED_RUNNERS == 'true' - run: sccache --show-stats + - name: Run consensus-spec-tests with blst, milagro and fake_crypto + run: make nextest-ef + - name: Show cache stats + if: env.SELF_HOSTED_RUNNERS == 'true' + run: sccache --show-stats dockerfile-ubuntu: name: dockerfile-ubuntu runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Build the root Dockerfile - run: docker build --build-arg FEATURES=portable -t lighthouse:local . - - name: Test the built image - run: docker run -t lighthouse:local lighthouse --version + - uses: actions/checkout@v3 + - name: Build the root Dockerfile + run: docker build --build-arg FEATURES=portable -t lighthouse:local . + - name: Test the built image + run: docker run -t lighthouse:local lighthouse --version eth1-simulator-ubuntu: name: eth1-simulator-ubuntu runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - uses: moonrepo/setup-rust@v1 - with: + - uses: actions/checkout@v3 + - name: Get latest version of stable Rust + uses: moonrepo/setup-rust@v1 + with: channel: stable cache-target: release - - name: Install Foundry (anvil) - uses: foundry-rs/foundry-toolchain@v1 - with: - version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d - - name: Run the beacon chain sim that starts from an eth1 contract - run: cargo run --release --bin simulator eth1-sim + - name: Install Foundry (anvil) + uses: foundry-rs/foundry-toolchain@v1 + with: + version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d + - name: Run the beacon chain sim that starts from an eth1 contract + run: cargo run --release --bin simulator eth1-sim merge-transition-ubuntu: name: merge-transition-ubuntu runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - uses: moonrepo/setup-rust@v1 - with: + - uses: actions/checkout@v3 + - name: Get latest version of stable Rust + uses: moonrepo/setup-rust@v1 + with: channel: stable cache-target: release - - name: Install Foundry (anvil) - uses: foundry-rs/foundry-toolchain@v1 - with: - version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d - - name: Run the beacon chain sim and go through the merge transition - run: cargo run --release --bin simulator eth1-sim --post-merge + - name: Install Foundry (anvil) + uses: foundry-rs/foundry-toolchain@v1 + with: + version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d + - name: Run the beacon chain sim and go through the merge transition + run: cargo run --release --bin simulator eth1-sim --post-merge no-eth1-simulator-ubuntu: name: no-eth1-simulator-ubuntu runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - uses: moonrepo/setup-rust@v1 - with: + - uses: actions/checkout@v3 + - name: Get latest version of stable Rust + uses: moonrepo/setup-rust@v1 + with: channel: stable cache-target: release - - name: Run the beacon chain sim without an eth1 connection - run: cargo run --release --bin simulator no-eth1-sim + - name: Run the beacon chain sim without an eth1 connection + run: cargo run --release --bin simulator no-eth1-sim syncing-simulator-ubuntu: name: syncing-simulator-ubuntu runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - uses: moonrepo/setup-rust@v1 - with: + - uses: actions/checkout@v3 + - name: Get latest version of stable Rust + uses: moonrepo/setup-rust@v1 + with: channel: stable cache-target: release - - name: Install Foundry (anvil) - uses: foundry-rs/foundry-toolchain@v1 - with: - version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d - - name: Run the syncing simulator - run: cargo run --release --bin simulator syncing-sim + - name: Install Foundry (anvil) + uses: foundry-rs/foundry-toolchain@v1 + with: + version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d + - name: Run the syncing simulator + run: cargo run --release --bin simulator syncing-sim doppelganger-protection-test: name: doppelganger-protection-test runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "small"]') || 'ubuntu-latest' }} @@ -292,113 +293,113 @@ jobs: # Enable portable to prevent issues with caching `blst` for the wrong CPU type FEATURES: jemalloc,portable steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - if: env.SELF_HOSTED_RUNNERS == 'false' - uses: moonrepo/setup-rust@v1 - with: + - uses: actions/checkout@v3 + - name: Get latest version of stable Rust + if: env.SELF_HOSTED_RUNNERS == 'false' + uses: moonrepo/setup-rust@v1 + with: channel: stable cache-target: release - - name: Install geth - if: env.SELF_HOSTED_RUNNERS == 'false' - run: | + - name: Install geth + if: env.SELF_HOSTED_RUNNERS == 'false' + run: | sudo add-apt-repository -y ppa:ethereum/ethereum sudo apt-get update sudo apt-get install ethereum - - name: Install lighthouse - run: | + - name: Install lighthouse + run: | make - - name: Install lcli - # TODO: uncomment after the version of lcli in https://github.com/sigp/lighthouse/pull/5137 - # is installed on the runners - # if: env.SELF_HOSTED_RUNNERS == 'false' - run: make install-lcli - - name: Run the doppelganger protection failure test script - run: | + - name: Install lcli + # TODO: uncomment after the version of lcli in https://github.com/sigp/lighthouse/pull/5137 + # is installed on the runners + # if: env.SELF_HOSTED_RUNNERS == 'false' + run: make install-lcli + - name: Run the doppelganger protection failure test script + run: | cd scripts/tests ./doppelganger_protection.sh failure genesis.json - - name: Run the doppelganger protection success test script - run: | + - name: Run the doppelganger protection success test script + run: | cd scripts/tests ./doppelganger_protection.sh success genesis.json execution-engine-integration-ubuntu: name: execution-engine-integration-ubuntu runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "small"]') || 'ubuntu-latest' }} steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - if: env.SELF_HOSTED_RUNNERS == 'false' - uses: moonrepo/setup-rust@v1 - with: + - uses: actions/checkout@v3 + - name: Get latest version of stable Rust + if: env.SELF_HOSTED_RUNNERS == 'false' + uses: moonrepo/setup-rust@v1 + with: channel: stable cache-target: release cache: false - env: + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Add go compiler to $PATH - if: env.SELF_HOSTED_RUNNERS == 'true' - run: echo "/usr/local/go/bin" >> $GITHUB_PATH - - name: Run exec engine integration tests in release - run: make test-exec-engine + - name: Add go compiler to $PATH + if: env.SELF_HOSTED_RUNNERS == 'true' + run: echo "/usr/local/go/bin" >> $GITHUB_PATH + - name: Run exec engine integration tests in release + run: make test-exec-engine check-code: name: check-code runs-on: ubuntu-latest env: CARGO_INCREMENTAL: 1 steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - uses: moonrepo/setup-rust@v1 - with: + - uses: actions/checkout@v3 + - name: Get latest version of stable Rust + uses: moonrepo/setup-rust@v1 + with: channel: stable cache-target: release components: rustfmt,clippy bins: cargo-audit - - name: Check formatting with cargo fmt - run: make cargo-fmt - - name: Lint code for quality and style with Clippy - run: make lint - - name: Certify Cargo.lock freshness - run: git diff --exit-code Cargo.lock - - name: Typecheck benchmark code without running it - run: make check-benches - - name: Validate state_processing feature arbitrary-fuzz - run: make arbitrary-fuzz - - name: Run cargo audit - run: make audit-CI - - name: Run cargo vendor to make sure dependencies can be vendored for packaging, reproducibility and archival purpose - run: CARGO_HOME=$(readlink -f $HOME) make vendor + - name: Check formatting with cargo fmt + run: make cargo-fmt + - name: Lint code for quality and style with Clippy + run: make lint + - name: Certify Cargo.lock freshness + run: git diff --exit-code Cargo.lock + - name: Typecheck benchmark code without running it + run: make check-benches + - name: Validate state_processing feature arbitrary-fuzz + run: make arbitrary-fuzz + - name: Run cargo audit + run: make audit-CI + - name: Run cargo vendor to make sure dependencies can be vendored for packaging, reproducibility and archival purpose + run: CARGO_HOME=$(readlink -f $HOME) make vendor check-msrv: name: check-msrv runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Install Rust at Minimum Supported Rust Version (MSRV) - run: | - metadata=$(cargo metadata --no-deps --format-version 1) - msrv=$(echo $metadata | jq -r '.packages | map(select(.name == "lighthouse")) | .[0].rust_version') - rustup override set $msrv - - name: Run cargo check - run: cargo check --workspace + - uses: actions/checkout@v3 + - name: Install Rust at Minimum Supported Rust Version (MSRV) + run: | + metadata=$(cargo metadata --no-deps --format-version 1) + msrv=$(echo $metadata | jq -r '.packages | map(select(.name == "lighthouse")) | .[0].rust_version') + rustup override set $msrv + - name: Run cargo check + run: cargo check --workspace cargo-udeps: name: cargo-udeps runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Get latest version of nightly Rust - uses: moonrepo/setup-rust@v1 - with: + - uses: actions/checkout@v3 + - name: Get latest version of nightly Rust + uses: moonrepo/setup-rust@v1 + with: channel: nightly bins: cargo-udeps cache: false - env: + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Create Cargo config dir - run: mkdir -p .cargo - - name: Install custom Cargo config - run: cp -f .github/custom/config.toml .cargo/config.toml - - name: Run cargo udeps to identify unused crates in the dependency graph - run: make udeps + - name: Create Cargo config dir + run: mkdir -p .cargo + - name: Install custom Cargo config + run: cp -f .github/custom/config.toml .cargo/config.toml + - name: Run cargo udeps to identify unused crates in the dependency graph + run: make udeps env: # Allow warnings on Nightly RUSTFLAGS: "" @@ -406,25 +407,25 @@ jobs: name: compile-with-beta-compiler runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Install dependencies - run: sudo apt install -y git gcc g++ make cmake pkg-config llvm-dev libclang-dev clang - - name: Use Rust beta - run: rustup override set beta - - name: Run make - run: make + - uses: actions/checkout@v3 + - name: Install dependencies + run: sudo apt install -y git gcc g++ make cmake pkg-config llvm-dev libclang-dev clang + - name: Use Rust beta + run: rustup override set beta + - name: Run make + run: make cli-check: name: cli-check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - uses: moonrepo/setup-rust@v1 - with: - channel: stable - cache-target: release - - name: Run Makefile to trigger the bash script - run: make cli + - uses: actions/checkout@v3 + - name: Get latest version of stable Rust + uses: moonrepo/setup-rust@v1 + with: + channel: stable + cache-target: release + - name: Run Makefile to trigger the bash script + run: make cli # This job succeeds ONLY IF all others succeed. It is used by the merge queue to determine whether # a PR is safe to merge. New jobs should be added here. test-suite-success: @@ -433,7 +434,8 @@ jobs: needs: [ 'target-branch-check', 'release-tests-ubuntu', - 'release-tests-windows', + # FIXME(das): disabled for now as the c-kzg-4844 `das` branch doesn't build on windows. + # 'release-tests-windows', 'beacon-chain-tests', 'op-pool-tests', 'network-tests', diff --git a/Cargo.lock b/Cargo.lock index 46fd981aa77..be905910acd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1295,7 +1295,7 @@ dependencies = [ [[package]] name = "c-kzg" version = "0.1.0" -source = "git+https://github.com/ethereum/c-kzg-4844?rev=748283cced543c486145d5f3f38684becdfe3e1b#748283cced543c486145d5f3f38684becdfe3e1b" +source = "git+https://github.com/ethereum/c-kzg-4844?branch=das#5a2656d9409713fdb5112ed5c1c9ec808093de8f" dependencies = [ "bindgen 0.66.1", "blst", @@ -8985,6 +8985,7 @@ dependencies = [ "criterion", "derivative", "eth2_interop_keypairs", + "eth2_network_config", "ethereum-types 0.14.1", "ethereum_hashing", "ethereum_serde_utils", diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index b896327e06f..c0308e0b808 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -53,7 +53,9 @@ use crate::block_verification_types::{ AsBlock, BlockContentsError, BlockImportData, GossipVerifiedBlockContents, RpcBlock, }; use crate::data_availability_checker::{AvailabilityCheckError, MaybeAvailableBlock}; -use crate::data_column_verification::GossipDataColumnError; +use crate::data_column_verification::{ + GossipDataColumnError, GossipVerifiedDataColumn, GossipVerifiedDataColumnList, +}; use crate::eth1_finalization_cache::Eth1FinalizationData; use crate::execution_payload::{ is_optimistic_candidate_block, validate_execution_payload_for_gossip, validate_merge_block, @@ -99,10 +101,12 @@ use std::time::Duration; use store::{Error as DBError, HotStateSummary, KeyValueStore, StoreOp}; use task_executor::JoinHandle; use tree_hash::TreeHash; +use types::data_column_sidecar::DataColumnSidecarError; use types::{ - BeaconBlockRef, BeaconState, BeaconStateError, ChainSpec, CloneConfig, Epoch, EthSpec, - ExecutionBlockHash, Hash256, InconsistentFork, PublicKey, PublicKeyBytes, RelativeEpoch, - SignedBeaconBlock, SignedBeaconBlockHeader, Slot, + BeaconBlockRef, BeaconState, BeaconStateError, BlobSidecarList, ChainSpec, CloneConfig, + DataColumnSidecar, DataColumnSubnetId, Epoch, EthSpec, ExecutionBlockHash, Hash256, + InconsistentFork, PublicKey, PublicKeyBytes, RelativeEpoch, SignedBeaconBlock, + SignedBeaconBlockHeader, Slot, }; use types::{BlobSidecar, ExecPayload}; @@ -727,7 +731,7 @@ impl IntoGossipVerifiedBlockContents for PublishBlockReq let _timer = metrics::start_timer(&metrics::BLOB_SIDECAR_INCLUSION_PROOF_COMPUTATION); let blob = BlobSidecar::new(i, blob, &block, *kzg_proof) - .map_err(BlockContentsError::SidecarError)?; + .map_err(BlockContentsError::BlobSidecarError)?; drop(_timer); let gossip_verified_blob = GossipVerifiedBlob::new(Arc::new(blob), i as u64, chain)?; @@ -737,9 +741,52 @@ impl IntoGossipVerifiedBlockContents for PublishBlockReq Ok::<_, BlockContentsError>(gossip_verified_blobs) }) .transpose()?; + + let gossip_verified_data_columns = gossip_verified_blobs + .as_ref() + .map(|blobs| { + // NOTE: we expect KZG to be initialized if the blobs are present + let kzg = chain + .kzg + .as_ref() + .ok_or(BlockContentsError::DataColumnError( + GossipDataColumnError::::KzgNotInitialized, + ))?; + + let blob_sidecar_list: Vec<_> = + blobs.iter().map(|blob| blob.clone_blob()).collect(); + let blob_sidecar_list = BlobSidecarList::new(blob_sidecar_list) + .map_err(DataColumnSidecarError::SszError)?; + let timer = metrics::start_timer(&metrics::DATA_COLUMN_SIDECAR_COMPUTATION); + let sidecars = DataColumnSidecar::build_sidecars(&blob_sidecar_list, &block, kzg)?; + drop(timer); + let mut gossip_verified_data_columns = vec![]; + for sidecar in sidecars { + let subnet = DataColumnSubnetId::try_from_column_index::( + sidecar.index as usize, + ) + .map_err(|_| { + BlockContentsError::::DataColumnSidecarError( + DataColumnSidecarError::DataColumnIndexOutOfBounds, + ) + })?; + let column = GossipVerifiedDataColumn::new(sidecar, subnet.into(), chain)?; + gossip_verified_data_columns.push(column); + } + let gossip_verified_data_columns = + GossipVerifiedDataColumnList::new(gossip_verified_data_columns) + .map_err(DataColumnSidecarError::SszError)?; + Ok::<_, BlockContentsError>(gossip_verified_data_columns) + }) + .transpose()?; + let gossip_verified_block = GossipVerifiedBlock::new(block, chain)?; - Ok((gossip_verified_block, gossip_verified_blobs)) + Ok(( + gossip_verified_block, + gossip_verified_blobs, + gossip_verified_data_columns, + )) } fn inner_block(&self) -> &SignedBeaconBlock { diff --git a/beacon_node/beacon_chain/src/block_verification_types.rs b/beacon_node/beacon_chain/src/block_verification_types.rs index 263a6eab074..d509ba565a8 100644 --- a/beacon_node/beacon_chain/src/block_verification_types.rs +++ b/beacon_node/beacon_chain/src/block_verification_types.rs @@ -2,14 +2,15 @@ use crate::blob_verification::{GossipBlobError, GossipVerifiedBlobList}; use crate::block_verification::BlockError; use crate::data_availability_checker::AvailabilityCheckError; pub use crate::data_availability_checker::{AvailableBlock, MaybeAvailableBlock}; +use crate::data_column_verification::{GossipDataColumnError, GossipVerifiedDataColumnList}; use crate::eth1_finalization_cache::Eth1FinalizationData; use crate::{get_block_root, GossipVerifiedBlock, PayloadVerificationOutcome}; use derivative::Derivative; use ssz_types::VariableList; use state_processing::ConsensusContext; use std::sync::Arc; -use types::blob_sidecar::{BlobIdentifier, BlobSidecarError, FixedBlobSidecarList}; -use types::data_column_sidecar::DataColumnSidecarList; +use types::blob_sidecar::{self, BlobIdentifier, FixedBlobSidecarList}; +use types::data_column_sidecar::{self, DataColumnSidecarList}; use types::{ BeaconBlockRef, BeaconState, BlindedPayload, BlobSidecarList, Epoch, EthSpec, Hash256, SignedBeaconBlock, SignedBeaconBlockHeader, Slot, @@ -320,14 +321,19 @@ pub struct BlockImportData { pub consensus_context: ConsensusContext, } -pub type GossipVerifiedBlockContents = - (GossipVerifiedBlock, Option>); +pub type GossipVerifiedBlockContents = ( + GossipVerifiedBlock, + Option>, + Option>, +); #[derive(Debug)] pub enum BlockContentsError { BlockError(BlockError), BlobError(GossipBlobError), - SidecarError(BlobSidecarError), + BlobSidecarError(blob_sidecar::BlobSidecarError), + DataColumnError(GossipDataColumnError), + DataColumnSidecarError(data_column_sidecar::DataColumnSidecarError), } impl From> for BlockContentsError { @@ -342,6 +348,18 @@ impl From> for BlockContentsError { } } +impl From> for BlockContentsError { + fn from(value: GossipDataColumnError) -> Self { + Self::DataColumnError(value) + } +} + +impl From for BlockContentsError { + fn from(value: data_column_sidecar::DataColumnSidecarError) -> Self { + Self::DataColumnSidecarError(value) + } +} + impl std::fmt::Display for BlockContentsError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -351,8 +369,14 @@ impl std::fmt::Display for BlockContentsError { BlockContentsError::BlobError(err) => { write!(f, "BlobError({})", err) } - BlockContentsError::SidecarError(err) => { - write!(f, "SidecarError({:?})", err) + BlockContentsError::BlobSidecarError(err) => { + write!(f, "BlobSidecarError({:?})", err) + } + BlockContentsError::DataColumnError(err) => { + write!(f, "DataColumnError({:?})", err) + } + BlockContentsError::DataColumnSidecarError(err) => { + write!(f, "DataColumnSidecarError({:?})", err) } } } diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 2ea8fe1ae4f..8e045b7f29f 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -5,7 +5,10 @@ use kzg::{Error as KzgError, Kzg}; use ssz_derive::{Decode, Encode}; use std::sync::Arc; use types::data_column_sidecar::{ColumnIndex, DataColumnIdentifier}; -use types::{BeaconStateError, DataColumnSidecar, EthSpec, Hash256, SignedBeaconBlockHeader, Slot}; +use types::{ + BeaconStateError, DataColumnSidecar, EthSpec, Hash256, SignedBeaconBlockHeader, Slot, + VariableList, +}; /// An error occurred while validating a gossip data column. #[derive(Debug)] @@ -77,6 +80,11 @@ impl From for GossipDataColumnError { } } +pub type GossipVerifiedDataColumnList = VariableList< + GossipVerifiedDataColumn, + <::EthSpec as EthSpec>::DataColumnCount, +>; + /// A wrapper around a `DataColumnSidecar` that indicates it has been approved for re-gossiping on /// the p2p network. #[derive(Debug)] @@ -113,6 +121,11 @@ impl GossipVerifiedDataColumn { self.data_column.as_data_column() } + /// This is cheap as we're calling clone on an Arc + pub fn clone_data_column(&self) -> Arc> { + self.data_column.clone_data_column() + } + pub fn block_root(&self) -> Hash256 { self.block_root } diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 9ab2a0d395e..ffc13bd7cae 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1042,6 +1042,10 @@ lazy_static! { "blob_sidecar_inclusion_proof_computation_seconds", "Time taken to compute blob sidecar inclusion proof" ); + pub static ref DATA_COLUMN_SIDECAR_COMPUTATION: Result = try_create_histogram( + "data_column_sidecar_computation_seconds", + "Time taken to compute data column sidecar, including cells, proofs and inclusion proof" + ); } // Fifth lazy-static block is used to account for macro recursion limit. diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 67e5d00f8c6..7a5c384a4c8 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -18,10 +18,11 @@ use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; use tree_hash::TreeHash; +use types::data_column_sidecar::DataColumnSidecarList; use types::{ - AbstractExecPayload, BeaconBlockRef, BlobSidecarList, DataColumnSidecar, DataColumnSubnetId, - EthSpec, ExecPayload, ExecutionBlockHash, ForkName, FullPayload, FullPayloadMerge, Hash256, - SignedBeaconBlock, SignedBlindedBeaconBlock, VariableList, + AbstractExecPayload, BeaconBlockRef, BlobSidecarList, DataColumnSubnetId, EthSpec, ExecPayload, + ExecutionBlockHash, ForkName, FullPayload, FullPayloadMerge, Hash256, SignedBeaconBlock, + SignedBlindedBeaconBlock, VariableList, }; use warp::http::StatusCode; use warp::{reply::Response, Rejection, Reply}; @@ -67,6 +68,7 @@ pub async fn publish_block>, blobs_opt: Option>, + data_cols_opt: Option>, sender, log, seen_timestamp| { @@ -88,23 +90,6 @@ pub async fn publish_block { let mut pubsub_messages = vec![PubsubMessage::BeaconBlock(block.clone())]; if let Some(blob_sidecars) = blobs_opt { - // Build and publish column sidecars - let col_sidecars = DataColumnSidecar::random_from_blob_sidecars(&blob_sidecars) - .map_err(|e| { - BeaconChainError::UnableToBuildColumnSidecar(format!("{e:?}")) - })?; - - for (col_index, col_sidecar) in col_sidecars.into_iter().enumerate() { - let subnet_id = - DataColumnSubnetId::try_from_column_index::(col_index) - .map_err(|e| { - BeaconChainError::UnableToBuildColumnSidecar(format!("{e:?}")) - })?; - pubsub_messages.push(PubsubMessage::DataColumnSidecar(Box::new(( - subnet_id, - Arc::new(col_sidecar), - )))); - } // Publish blob sidecars for (blob_index, blob) in blob_sidecars.into_iter().enumerate() { pubsub_messages.push(PubsubMessage::BlobSidecar(Box::new(( @@ -113,6 +98,19 @@ pub async fn publish_block( + data_col.index as usize, + ) + .map_err(|e| { + BeaconChainError::UnableToBuildColumnSidecar(format!("{e:?}")) + })?; + pubsub_messages.push(PubsubMessage::DataColumnSidecar(Box::new(( + subnet, data_col, + )))); + } + } crate::publish_pubsub_messages(&sender, pubsub_messages) .map_err(|_| BlockError::BeaconChainError(BeaconChainError::UnableToPublish))?; } @@ -128,7 +126,7 @@ pub async fn publish_block b, Err(BlockContentsError::BlockError(BlockError::BlockIsAlreadyKnown)) @@ -167,6 +165,15 @@ pub async fn publish_block>(); VariableList::from(blobs) }); + let data_cols_opt = gossip_verified_data_columns + .as_ref() + .map(|gossip_verified_data_columns| { + let data_columns = gossip_verified_data_columns + .into_iter() + .map(|col| col.clone_data_column()) + .collect::>(); + VariableList::from(data_columns) + }); let block_root = block_root.unwrap_or(gossip_verified_block.block_root); @@ -174,6 +181,7 @@ pub async fn publish_block publish_block( block_clone, blobs_opt, + data_cols_opt, sender_clone, log_clone, seen_timestamp, @@ -203,6 +212,7 @@ pub async fn publish_block> BeaconBlockBodyRef<'a, T, } } + /// Produces the proof of inclusion for `self.blob_kzg_commitments`. + pub fn kzg_commitments_merkle_proof( + &self, + ) -> Result, Error> { + match self { + Self::Base(_) | Self::Altair(_) | Self::Merge(_) | Self::Capella(_) => { + Err(Error::IncorrectStateVariant) + } + Self::Deneb(body) => { + let leaves = [ + body.randao_reveal.tree_hash_root(), + body.eth1_data.tree_hash_root(), + body.graffiti.tree_hash_root(), + body.proposer_slashings.tree_hash_root(), + body.attester_slashings.tree_hash_root(), + body.attestations.tree_hash_root(), + body.deposits.tree_hash_root(), + body.voluntary_exits.tree_hash_root(), + body.sync_aggregate.tree_hash_root(), + body.execution_payload.tree_hash_root(), + body.bls_to_execution_changes.tree_hash_root(), + body.blob_kzg_commitments.tree_hash_root(), + ]; + let beacon_block_body_depth = leaves.len().next_power_of_two().ilog2() as usize; + let tree = MerkleTree::create(&leaves, beacon_block_body_depth); + let (_, proof) = tree + .generate_proof(BLOB_KZG_COMMITMENTS_INDEX, beacon_block_body_depth) + .map_err(Error::MerkleTreeError)?; + Ok(proof.into()) + } + } + } + /// Return `true` if this block body has a non-zero number of blobs. pub fn has_blobs(self) -> bool { self.blob_kzg_commitments() diff --git a/consensus/types/src/data_column_sidecar.rs b/consensus/types/src/data_column_sidecar.rs index a6fc4c56745..7ff371cee93 100644 --- a/consensus/types/src/data_column_sidecar.rs +++ b/consensus/types/src/data_column_sidecar.rs @@ -1,13 +1,14 @@ use crate::beacon_block_body::KzgCommitments; use crate::test_utils::TestRandom; +use crate::BeaconStateError; use crate::{ - BeaconBlockHeader, BlobSidecarList, EthSpec, Hash256, KzgProofs, SignedBeaconBlockHeader, Slot, + BeaconBlockHeader, BlobSidecarList, EthSpec, Hash256, KzgProofs, SignedBeaconBlock, + SignedBeaconBlockHeader, Slot, }; use bls::Signature; use derivative::Derivative; +use kzg::{Blob as KzgBlob, Error as KzgError, Kzg}; use kzg::{KzgCommitment, KzgProof}; -use rand::rngs::StdRng; -use rand::{Rng, SeedableRng}; use safe_arith::ArithError; use serde::{Deserialize, Serialize}; use ssz::Encode; @@ -62,94 +63,96 @@ pub struct DataColumnSidecar { } impl DataColumnSidecar { - pub fn random_from_blob_sidecars( - blob_sidecars: &BlobSidecarList, - ) -> Result>, DataColumnSidecarError> { - if blob_sidecars.is_empty() { - return Ok(vec![]); - } - - let first_blob_sidecar = blob_sidecars - .first() - .ok_or(DataColumnSidecarError::MissingBlobSidecars)?; - let slot = first_blob_sidecar.slot(); - - // Proof for kzg commitments in `BeaconBlockBody` - let body_proof_start = first_blob_sidecar - .kzg_commitment_inclusion_proof - .len() - .saturating_sub(T::kzg_commitments_inclusion_proof_depth()); - let kzg_commitments_inclusion_proof: FixedVector< - Hash256, - T::KzgCommitmentsInclusionProofDepth, - > = first_blob_sidecar - .kzg_commitment_inclusion_proof - .get(body_proof_start..) - .ok_or(DataColumnSidecarError::KzgCommitmentInclusionProofOutOfBounds)? - .to_vec() - .into(); - - let mut rng = StdRng::seed_from_u64(slot.as_u64()); - let num_of_blobs = blob_sidecars.len(); - - (0..T::number_of_columns()) - .map(|col_index| { - Ok(DataColumnSidecar { - index: col_index as u64, - column: Self::generate_column_data(&mut rng, num_of_blobs, col_index)?, - kzg_commitments: blob_sidecars - .iter() - .map(|b| b.kzg_commitment) - .collect::>() - .into(), - kzg_proofs: blob_sidecars - .iter() - .map(|b| b.kzg_proof) - .collect::>() - .into(), - signed_block_header: first_blob_sidecar.signed_block_header.clone(), - kzg_commitments_inclusion_proof: kzg_commitments_inclusion_proof.clone(), - }) - }) - .collect::, _>>() + pub fn slot(&self) -> Slot { + self.signed_block_header.message.slot } - fn generate_column_data( - rng: &mut StdRng, - num_of_blobs: usize, - index: usize, - ) -> Result, DataColumnSidecarError> { - let mut dummy_cell_data = Cell::::default(); - // Prefix with column index - let prefix = index.to_le_bytes(); - dummy_cell_data - .get_mut(..prefix.len()) - .ok_or(DataColumnSidecarError::DataColumnIndexOutOfBounds)? - .copy_from_slice(&prefix); - // Fill the rest of the vec with random values - rng.fill( - dummy_cell_data - .get_mut(prefix.len()..) - .ok_or(DataColumnSidecarError::DataColumnIndexOutOfBounds)?, - ); - - let column = DataColumn::::new(vec![dummy_cell_data; num_of_blobs])?; - Ok(column) + pub fn block_root(&self) -> Hash256 { + self.signed_block_header.message.tree_hash_root() } - pub fn id(&self) -> DataColumnIdentifier { - DataColumnIdentifier { - block_root: self.block_root(), - index: self.index, + pub fn build_sidecars( + blobs: &BlobSidecarList, + block: &SignedBeaconBlock, + kzg: &Kzg, + ) -> Result, DataColumnSidecarError> { + let kzg_commitments = block + .message() + .body() + .blob_kzg_commitments() + .map_err(|_err| DataColumnSidecarError::PreDeneb)?; + let kzg_commitments_inclusion_proof = + block.message().body().kzg_commitments_merkle_proof()?; + let signed_block_header = block.signed_block_header(); + + let mut columns = + vec![Vec::with_capacity(T::max_blobs_per_block()); T::number_of_columns()]; + let mut column_kzg_proofs = + vec![Vec::with_capacity(T::max_blobs_per_block()); T::number_of_columns()]; + + // NOTE: assumes blob sidecars are ordered by index + for blob in blobs { + let blob = KzgBlob::from_bytes(&blob.blob).map_err(KzgError::from)?; + let (blob_cells, blob_cell_proofs) = kzg.compute_cells_and_proofs(&blob)?; + + // we iterate over each column, and we construct the column from "top to bottom", + // pushing on the cell and the corresponding proof at each column index. we do this for + // each blob (i.e. the outer loop). + for col in 0..T::number_of_columns() { + let cell = + blob_cells + .get(col) + .ok_or(DataColumnSidecarError::InconsistentArrayLength(format!( + "Missing blob cell at index {col}" + )))?; + let cell: Vec = cell + .into_inner() + .into_iter() + .flat_map(|data| (*data).into_iter()) + .collect(); + let cell = Cell::::from(cell); + + let proof = blob_cell_proofs.get(col).ok_or( + DataColumnSidecarError::InconsistentArrayLength(format!( + "Missing blob cell KZG proof at index {col}" + )), + )?; + + let column = + columns + .get_mut(col) + .ok_or(DataColumnSidecarError::InconsistentArrayLength(format!( + "Missing data column at index {col}" + )))?; + let column_proofs = column_kzg_proofs.get_mut(col).ok_or( + DataColumnSidecarError::InconsistentArrayLength(format!( + "Missing data column proofs at index {col}" + )), + )?; + + column.push(cell); + column_proofs.push(*proof); + } } - } - pub fn slot(&self) -> Slot { - self.signed_block_header.message.slot - } + let sidecars: Vec>> = columns + .into_iter() + .zip(column_kzg_proofs) + .enumerate() + .map(|(index, (col, proofs))| { + Arc::new(DataColumnSidecar { + index: index as u64, + column: DataColumn::::from(col), + kzg_commitments: kzg_commitments.clone(), + kzg_proofs: KzgProofs::::from(proofs), + signed_block_header: signed_block_header.clone(), + kzg_commitments_inclusion_proof: kzg_commitments_inclusion_proof.clone(), + }) + }) + .collect(); + let sidecars = DataColumnSidecarList::from(sidecars); - pub fn block_root(&self) -> Hash256 { - self.signed_block_header.message.tree_hash_root() + Ok(sidecars) } pub fn min_size() -> usize { @@ -195,10 +198,14 @@ impl DataColumnSidecar { #[derive(Debug)] pub enum DataColumnSidecarError { ArithError(ArithError), - MissingBlobSidecars, - KzgCommitmentInclusionProofOutOfBounds, + BeaconStateError(BeaconStateError), DataColumnIndexOutOfBounds, + KzgCommitmentInclusionProofOutOfBounds, + KzgError(KzgError), + MissingBlobSidecars, + PreDeneb, SszError(SszError), + InconsistentArrayLength(String), } impl From for DataColumnSidecarError { @@ -207,6 +214,18 @@ impl From for DataColumnSidecarError { } } +impl From for DataColumnSidecarError { + fn from(e: BeaconStateError) -> Self { + Self::BeaconStateError(e) + } +} + +impl From for DataColumnSidecarError { + fn from(e: KzgError) -> Self { + Self::KzgError(e) + } +} + impl From for DataColumnSidecarError { fn from(e: SszError) -> Self { Self::SszError(e) @@ -228,35 +247,56 @@ mod test { DataColumnSidecar, MainnetEthSpec, SignedBeaconBlock, }; use bls::Signature; - use kzg::{KzgCommitment, KzgProof}; + use eth2_network_config::TRUSTED_SETUP_BYTES; + use kzg::{Kzg, KzgCommitment, KzgProof, TrustedSetup}; use std::sync::Arc; #[test] - fn test_random_from_blob_sidecars() { + fn test_build_sidecars() { type E = MainnetEthSpec; let num_of_blobs = 6; let spec = E::default_spec(); - let blob_sidecars: BlobSidecarList = create_test_blob_sidecars(num_of_blobs, &spec); + let (signed_block, blob_sidecars) = + create_test_block_and_blob_sidecars::(num_of_blobs, &spec); - let column_sidecars = DataColumnSidecar::random_from_blob_sidecars(&blob_sidecars).unwrap(); + let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP_BYTES).unwrap(); + let kzg = Arc::new(Kzg::new_from_trusted_setup(trusted_setup).unwrap()); - assert_eq!(column_sidecars.len(), E::number_of_columns()); + let column_sidecars = + DataColumnSidecar::build_sidecars(&blob_sidecars, &signed_block, &kzg).unwrap(); + + let block_kzg_commitments = signed_block + .message() + .body() + .blob_kzg_commitments() + .unwrap() + .clone(); + let block_kzg_commitments_inclusion_proof = signed_block + .message() + .body() + .kzg_commitments_merkle_proof() + .unwrap(); + assert_eq!(column_sidecars.len(), E::number_of_columns()); for (idx, col_sidecar) in column_sidecars.iter().enumerate() { assert_eq!(col_sidecar.index, idx as u64); + assert_eq!(col_sidecar.kzg_commitments.len(), num_of_blobs); - // ensure column sidecars are prefixed with column index (for verification purpose in prototype only) - let prefix_len = 8; // column index (u64) is stored as the first 8 bytes - let cell = col_sidecar.column.first().unwrap(); - let col_index_prefix = u64::from_le_bytes(cell[0..prefix_len].try_into().unwrap()); - assert_eq!(col_index_prefix, idx as u64) + assert_eq!(col_sidecar.column.len(), num_of_blobs); + assert_eq!(col_sidecar.kzg_proofs.len(), num_of_blobs); + + assert_eq!(col_sidecar.kzg_commitments, block_kzg_commitments); + assert_eq!( + col_sidecar.kzg_commitments_inclusion_proof, + block_kzg_commitments_inclusion_proof + ); } } - fn create_test_blob_sidecars( + fn create_test_block_and_blob_sidecars( num_of_blobs: usize, spec: &ChainSpec, - ) -> BlobSidecarList { + ) -> (SignedBeaconBlock, BlobSidecarList) { let mut block = BeaconBlock::Deneb(BeaconBlockDeneb::empty(spec)); let mut body = block.body_mut(); let blob_kzg_commitments = body.blob_kzg_commitments_mut().unwrap(); @@ -266,7 +306,7 @@ mod test { let signed_block = SignedBeaconBlock::from_block(block, Signature::empty()); - (0..num_of_blobs) + let sidecars = (0..num_of_blobs) .map(|index| { BlobSidecar::new( index, @@ -278,6 +318,8 @@ mod test { }) .collect::, _>>() .unwrap() - .into() + .into(); + + (signed_block, sidecars) } } diff --git a/crypto/kzg/Cargo.toml b/crypto/kzg/Cargo.toml index 7b70166f921..34ec99bbd34 100644 --- a/crypto/kzg/Cargo.toml +++ b/crypto/kzg/Cargo.toml @@ -16,4 +16,4 @@ serde = { workspace = true } ethereum_serde_utils = { workspace = true } hex = { workspace = true } ethereum_hashing = { workspace = true } -c-kzg = { git = "https://github.com/ethereum/c-kzg-4844", rev = "748283cced543c486145d5f3f38684becdfe3e1b"} \ No newline at end of file +c-kzg = { git = "https://github.com/ethereum/c-kzg-4844", branch = "das" } \ No newline at end of file diff --git a/crypto/kzg/src/lib.rs b/crypto/kzg/src/lib.rs index 0e096ba55c2..cd87514943f 100644 --- a/crypto/kzg/src/lib.rs +++ b/crypto/kzg/src/lib.rs @@ -13,6 +13,7 @@ pub use c_kzg::{ Blob, Bytes32, Bytes48, KzgSettings, BYTES_PER_BLOB, BYTES_PER_COMMITMENT, BYTES_PER_FIELD_ELEMENT, BYTES_PER_PROOF, FIELD_ELEMENTS_PER_BLOB, }; +use c_kzg::{Cell, CELLS_PER_BLOB}; #[derive(Debug)] pub enum Error { /// An error from the underlying kzg library. @@ -141,4 +142,51 @@ impl Kzg { ) .map_err(Into::into) } + + /// Computes the cells and associated proofs for a given `blob` at index `index`. + #[allow(clippy::type_complexity)] + pub fn compute_cells_and_proofs( + &self, + blob: &Blob, + ) -> Result<(Box<[Cell; CELLS_PER_BLOB]>, Box<[KzgProof; CELLS_PER_BLOB]>), Error> { + let (cells, proofs) = c_kzg::Cell::compute_cells_and_proofs(blob, &self.trusted_setup) + .map_err(Into::::into)?; + let proofs = Box::new(proofs.map(|proof| KzgProof::from(proof.to_bytes().into_inner()))); + Ok((cells, proofs)) + } + + /// Verifies a batch of cell-proof-commitment triplets. + /// + /// Here, `coordinates` correspond to the (row, col) coordinate of the cell in the extended + /// blob "matrix". In the 1D extension, row corresponds to the blob index, and col corresponds + /// to the data column index. + pub fn verify_cell_proof_batch( + &self, + cells: &[Cell], + kzg_proofs: &[KzgProof], + coordinates: &[(u64, u64)], + kzg_commitments: &[KzgCommitment], + ) -> Result<(), Error> { + let commitments_bytes: Vec = kzg_commitments + .iter() + .map(|comm| Bytes48::from(*comm)) + .collect(); + let proofs_bytes: Vec = kzg_proofs + .iter() + .map(|proof| Bytes48::from(*proof)) + .collect(); + let (rows, columns): (Vec, Vec) = coordinates.iter().cloned().unzip(); + if !c_kzg::KzgProof::verify_cell_proof_batch( + &commitments_bytes, + &rows, + &columns, + cells, + &proofs_bytes, + &self.trusted_setup, + )? { + Err(Error::KzgVerificationFailed) + } else { + Ok(()) + } + } }