From be50e4a8a4fcb5aed70b0b011dcd2117b980d0ed Mon Sep 17 00:00:00 2001 From: James MacMahon Date: Fri, 7 Jul 2023 12:09:17 -0400 Subject: [PATCH] Self assembling zones should set their own MTU Bump crucible rev to pick up: - Self assembling zones must set their own MTU - fix build break caused by merging oxidecomputer/crucible#801 - Issue extent flushes in parallel - Add Logger Option to Volume construct method - Update Rust crate libc to 0.2.147 - Update Rust crate percent-encoding to 2.3 - Retry jobs until they succeed - Reorder select arms so pings can't be starved out - Treat a skipped IO like an error IO for ACK results. - Retry pantry requests - Remove panics and asserts in dummy tests - Update Rust crate csv to 1.2.2 - Update Rust crate reedline to 0.21.0 - Set open file resource limit to the max - Update Rust crate ringbuffer to 0.14 - DTrace meet cmon - Widen assert values to u128 to deal with u64::MAX - Change size_to_validate from usize to u64 - Turn on live-repair test in CI - Increase flush_timeout for some tests, collect cores - Update to latest dropshot Bump propolis rev to pick up: - Self assembling zones must set their own MTU - Bump crucible rev to latest - Make the propolis zone self-assembling - Flesh out more PIIX3-PM to suppress log gripes - Bump crucible rev to latest - Restructure PM-timer under PIIX3 device - Fix inventory handling for nested child entities - Centralize vmm-data interface into bhyve_api - Clean up PCI device classes - Update openssl dep to 0.10.55 - Allow propolis-standalone to use VMM reservoir - only allow one request to reboot to be enqueued at a time Nexus is currently setting the MTU inside self-assembling zones, but this goes against the idea that self-assembling zones perform their own configuration. Remove the code in `RunningZone::boot` that performs commands on self-assembling zones, and set the MTU of $DATALINK in the Clickhouse and CockroachDB method scripts. Fixes #3512 --- Cargo.lock | 44 ++++++++----- Cargo.toml | 14 ++-- illumos-utils/src/running_zone.rs | 106 +++++++++++++----------------- package-manifest.toml | 12 ++-- smf/clickhouse/method_script.sh | 7 ++ smf/cockroachdb/method_script.sh | 7 ++ 6 files changed, 99 insertions(+), 91 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 83f69338c91..459115008fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -458,7 +458,7 @@ dependencies = [ [[package]] name = "bhyve_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=04a275736e9f3316de6bf2a4077d03acfa4e2cdf#04a275736e9f3316de6bf2a4077d03acfa4e2cdf" +source = "git+https://github.com/oxidecomputer/propolis?rev=9e12522341048d7b8c38394f0ddeb18083685c2f#9e12522341048d7b8c38394f0ddeb18083685c2f" dependencies = [ "bhyve_api_sys", "libc", @@ -468,7 +468,7 @@ dependencies = [ [[package]] name = "bhyve_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=04a275736e9f3316de6bf2a4077d03acfa4e2cdf#04a275736e9f3316de6bf2a4077d03acfa4e2cdf" +source = "git+https://github.com/oxidecomputer/propolis?rev=9e12522341048d7b8c38394f0ddeb18083685c2f#9e12522341048d7b8c38394f0ddeb18083685c2f" dependencies = [ "libc", "num_enum", @@ -1343,7 +1343,7 @@ dependencies = [ [[package]] name = "crucible-agent-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=df7e274edf0a1df287770e9567b27676f3ae4cc5#df7e274edf0a1df287770e9567b27676f3ae4cc5" +source = "git+https://github.com/oxidecomputer/crucible?rev=c574ff1232aa66eb60069795fe45f0c47f5da51d#c574ff1232aa66eb60069795fe45f0c47f5da51d" dependencies = [ "anyhow", "chrono", @@ -1358,7 +1358,19 @@ dependencies = [ [[package]] name = "crucible-client-types" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=df7e274edf0a1df287770e9567b27676f3ae4cc5#df7e274edf0a1df287770e9567b27676f3ae4cc5" +source = "git+https://github.com/oxidecomputer/crucible?rev=c574ff1232aa66eb60069795fe45f0c47f5da51d#c574ff1232aa66eb60069795fe45f0c47f5da51d" +dependencies = [ + "base64 0.21.2", + "schemars", + "serde", + "serde_json", + "uuid", +] + +[[package]] +name = "crucible-client-types" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/crucible?rev=f78832dc7ca48bce53f23de00d557bc9320a933f#f78832dc7ca48bce53f23de00d557bc9320a933f" dependencies = [ "base64 0.21.2", "schemars", @@ -1370,7 +1382,7 @@ dependencies = [ [[package]] name = "crucible-pantry-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=df7e274edf0a1df287770e9567b27676f3ae4cc5#df7e274edf0a1df287770e9567b27676f3ae4cc5" +source = "git+https://github.com/oxidecomputer/crucible?rev=c574ff1232aa66eb60069795fe45f0c47f5da51d#c574ff1232aa66eb60069795fe45f0c47f5da51d" dependencies = [ "anyhow", "chrono", @@ -1386,7 +1398,7 @@ dependencies = [ [[package]] name = "crucible-smf" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=df7e274edf0a1df287770e9567b27676f3ae4cc5#df7e274edf0a1df287770e9567b27676f3ae4cc5" +source = "git+https://github.com/oxidecomputer/crucible?rev=c574ff1232aa66eb60069795fe45f0c47f5da51d#c574ff1232aa66eb60069795fe45f0c47f5da51d" dependencies = [ "libc", "num-derive", @@ -1865,7 +1877,7 @@ checksum = "7e1a8646b2c125eeb9a84ef0faa6d2d102ea0d5da60b824ade2743263117b848" [[package]] name = "dladm" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=04a275736e9f3316de6bf2a4077d03acfa4e2cdf#04a275736e9f3316de6bf2a4077d03acfa4e2cdf" +source = "git+https://github.com/oxidecomputer/propolis?rev=9e12522341048d7b8c38394f0ddeb18083685c2f#9e12522341048d7b8c38394f0ddeb18083685c2f" dependencies = [ "libc", "num_enum", @@ -4844,7 +4856,7 @@ dependencies = [ "chrono", "clap 4.3.8", "crucible-agent-client", - "crucible-client-types", + "crucible-client-types 0.1.0 (git+https://github.com/oxidecomputer/crucible?rev=c574ff1232aa66eb60069795fe45f0c47f5da51d)", "ddm-admin-client", "dns-server", "dns-service-client 0.1.0", @@ -6006,7 +6018,7 @@ dependencies = [ [[package]] name = "propolis" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=04a275736e9f3316de6bf2a4077d03acfa4e2cdf#04a275736e9f3316de6bf2a4077d03acfa4e2cdf" +source = "git+https://github.com/oxidecomputer/propolis?rev=9e12522341048d7b8c38394f0ddeb18083685c2f#9e12522341048d7b8c38394f0ddeb18083685c2f" dependencies = [ "anyhow", "bhyve_api", @@ -6036,11 +6048,11 @@ dependencies = [ [[package]] name = "propolis-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=04a275736e9f3316de6bf2a4077d03acfa4e2cdf#04a275736e9f3316de6bf2a4077d03acfa4e2cdf" +source = "git+https://github.com/oxidecomputer/propolis?rev=9e12522341048d7b8c38394f0ddeb18083685c2f#9e12522341048d7b8c38394f0ddeb18083685c2f" dependencies = [ "async-trait", "base64 0.21.2", - "crucible-client-types", + "crucible-client-types 0.1.0 (git+https://github.com/oxidecomputer/crucible?rev=f78832dc7ca48bce53f23de00d557bc9320a933f)", "futures", "progenitor", "propolis_types", @@ -6060,7 +6072,7 @@ dependencies = [ [[package]] name = "propolis-server" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=04a275736e9f3316de6bf2a4077d03acfa4e2cdf#04a275736e9f3316de6bf2a4077d03acfa4e2cdf" +source = "git+https://github.com/oxidecomputer/propolis?rev=9e12522341048d7b8c38394f0ddeb18083685c2f#9e12522341048d7b8c38394f0ddeb18083685c2f" dependencies = [ "anyhow", "async-trait", @@ -6112,7 +6124,7 @@ dependencies = [ [[package]] name = "propolis-server-config" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=04a275736e9f3316de6bf2a4077d03acfa4e2cdf#04a275736e9f3316de6bf2a4077d03acfa4e2cdf" +source = "git+https://github.com/oxidecomputer/propolis?rev=9e12522341048d7b8c38394f0ddeb18083685c2f#9e12522341048d7b8c38394f0ddeb18083685c2f" dependencies = [ "serde", "serde_derive", @@ -6123,7 +6135,7 @@ dependencies = [ [[package]] name = "propolis_types" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=04a275736e9f3316de6bf2a4077d03acfa4e2cdf#04a275736e9f3316de6bf2a4077d03acfa4e2cdf" +source = "git+https://github.com/oxidecomputer/propolis?rev=9e12522341048d7b8c38394f0ddeb18083685c2f#9e12522341048d7b8c38394f0ddeb18083685c2f" dependencies = [ "schemars", "serde", @@ -8976,7 +8988,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "viona_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=04a275736e9f3316de6bf2a4077d03acfa4e2cdf#04a275736e9f3316de6bf2a4077d03acfa4e2cdf" +source = "git+https://github.com/oxidecomputer/propolis?rev=9e12522341048d7b8c38394f0ddeb18083685c2f#9e12522341048d7b8c38394f0ddeb18083685c2f" dependencies = [ "libc", "num_enum", @@ -8986,7 +8998,7 @@ dependencies = [ [[package]] name = "viona_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=04a275736e9f3316de6bf2a4077d03acfa4e2cdf#04a275736e9f3316de6bf2a4077d03acfa4e2cdf" +source = "git+https://github.com/oxidecomputer/propolis?rev=9e12522341048d7b8c38394f0ddeb18083685c2f#9e12522341048d7b8c38394f0ddeb18083685c2f" dependencies = [ "libc", ] diff --git a/Cargo.toml b/Cargo.toml index b550e7cc8c1..ae3631b1398 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -151,10 +151,10 @@ cookie = "0.16" criterion = { version = "0.5.1", features = [ "async_tokio" ] } crossbeam = "0.8" crossterm = { version = "0.26.1", features = ["event-stream"] } -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "df7e274edf0a1df287770e9567b27676f3ae4cc5" } -crucible-client-types = { git = "https://github.com/oxidecomputer/crucible", rev = "df7e274edf0a1df287770e9567b27676f3ae4cc5" } -crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "df7e274edf0a1df287770e9567b27676f3ae4cc5" } -crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "df7e274edf0a1df287770e9567b27676f3ae4cc5" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "c574ff1232aa66eb60069795fe45f0c47f5da51d" } +crucible-client-types = { git = "https://github.com/oxidecomputer/crucible", rev = "c574ff1232aa66eb60069795fe45f0c47f5da51d" } +crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "c574ff1232aa66eb60069795fe45f0c47f5da51d" } +crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "c574ff1232aa66eb60069795fe45f0c47f5da51d" } curve25519-dalek = "3" datatest-stable = "0.1.3" display-error-chain = "0.1.1" @@ -264,9 +264,9 @@ pretty-hex = "0.3.0" proc-macro2 = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } -bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "04a275736e9f3316de6bf2a4077d03acfa4e2cdf" } -propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "04a275736e9f3316de6bf2a4077d03acfa4e2cdf", features = [ "generated-migration" ] } -propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "04a275736e9f3316de6bf2a4077d03acfa4e2cdf", default-features = false, features = ["mock-only"] } +bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "9e12522341048d7b8c38394f0ddeb18083685c2f" } +propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "9e12522341048d7b8c38394f0ddeb18083685c2f", features = [ "generated-migration" ] } +propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "9e12522341048d7b8c38394f0ddeb18083685c2f", default-features = false, features = ["mock-only"] } #propolis-server = { path = "../propolis/bin/propolis-server" } proptest = "1.2.0" quote = "1.0" diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs index 439236c96f8..2ea10ecca85 100644 --- a/illumos-utils/src/running_zone.rs +++ b/illumos-utils/src/running_zone.rs @@ -372,78 +372,60 @@ impl RunningZone { } })?; - // Pull the zone ID. + // If the zone is self-assembling, then SMF service(s) inside the zone + // will be creating the listen address for the zone's service(s), + // setting the appropriate ifprop MTU, and so on. The idea behind + // self-assembling zones is that once they boot there should be *no* + // zlogin required. + + // Use the zone ID in order to check if /var/svc/profile/site.xml + // exists. let id = Zones::id(&zone.name) .await? .ok_or_else(|| BootError::NoZoneId { zone: zone.name.clone() })?; let site_profile_xml_exists = std::path::Path::new(&zone.site_profile_xml_path()).exists(); - let running_zone = RunningZone { id: Some(id), inner: zone }; - // Make sure the control vnic has an IP MTU of 9000 inside the zone - const CONTROL_VNIC_MTU: usize = 9000; - let vnic = running_zone.inner.control_vnic.name().to_string(); - - // If the zone is self-assembling, then SMF service(s) inside the zone - // will be creating the listen address for the zone's service(s). This - // will create IP interfaces, and means that `create-if` here will fail - // due to the interface already existing. Checking the output of - // `show-if` is also problematic due to TOCTOU. Use the check for the - // existence of site.xml, which means the zone is performing this - // self-assembly, and skip create-if if so. + let running_zone = RunningZone { id: Some(id), inner: zone }; if !site_profile_xml_exists { - let args = vec![ - IPADM.to_string(), - "create-if".to_string(), - "-t".to_string(), - vnic.clone(), + // If the zone is not self-assembling, make sure the control vnic + // has an IP MTU of 9000 inside the zone. + const CONTROL_VNIC_MTU: usize = 9000; + let vnic = running_zone.inner.control_vnic.name().to_string(); + + let commands = vec![ + vec![ + IPADM.to_string(), + "create-if".to_string(), + "-t".to_string(), + vnic.clone(), + ], + vec![ + IPADM.to_string(), + "set-ifprop".to_string(), + "-t".to_string(), + "-p".to_string(), + format!("mtu={}", CONTROL_VNIC_MTU), + "-m".to_string(), + "ipv4".to_string(), + vnic.clone(), + ], + vec![ + IPADM.to_string(), + "set-ifprop".to_string(), + "-t".to_string(), + "-p".to_string(), + format!("mtu={}", CONTROL_VNIC_MTU), + "-m".to_string(), + "ipv6".to_string(), + vnic, + ], ]; - running_zone.run_cmd(args)?; - } else { - // If the zone is self-assembling, then it's possible that the IP - // interface does not exist yet because it has not been brought up - // by the software in the zone. Run `create-if` here, but eat the - // error if there is one: this is safe unless the software that's - // part of self-assembly inside the zone is also trying to run - // `create-if` (instead of `create-addr`), and required for the - // `set-ifprop` commands below to pass. - let args = vec![ - IPADM.to_string(), - "create-if".to_string(), - "-t".to_string(), - vnic.clone(), - ]; - - let _result = running_zone.run_cmd(args); - } - - let commands = vec![ - vec![ - IPADM.to_string(), - "set-ifprop".to_string(), - "-t".to_string(), - "-p".to_string(), - format!("mtu={}", CONTROL_VNIC_MTU), - "-m".to_string(), - "ipv4".to_string(), - vnic.clone(), - ], - vec![ - IPADM.to_string(), - "set-ifprop".to_string(), - "-t".to_string(), - "-p".to_string(), - format!("mtu={}", CONTROL_VNIC_MTU), - "-m".to_string(), - "ipv6".to_string(), - vnic, - ], - ]; - - for args in &commands { - running_zone.run_cmd(args)?; + for args in &commands { + running_zone.run_cmd(args)?; + } } Ok(running_zone) diff --git a/package-manifest.toml b/package-manifest.toml index f611ef40274..27ee33acb03 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -241,10 +241,10 @@ only_for_targets.image = "standard" # 3. Use source.type = "manual" instead of "prebuilt" source.type = "prebuilt" source.repo = "crucible" -source.commit = "df7e274edf0a1df287770e9567b27676f3ae4cc5" +source.commit = "c574ff1232aa66eb60069795fe45f0c47f5da51d" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible.sha256.txt -source.sha256 = "c1a7996dfdded9476d5a595f16fcd07e902bb02f2ec0858bb3625f9b2195a919" +source.sha256 = "72363fbbfde15689bdbafee53e415ca994903644a9fdbb33ca0bc72d65927f0f" output.type = "zone" [package.crucible-pantry] @@ -252,10 +252,10 @@ service_name = "crucible_pantry" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "crucible" -source.commit = "df7e274edf0a1df287770e9567b27676f3ae4cc5" +source.commit = "c574ff1232aa66eb60069795fe45f0c47f5da51d" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible-pantry.sha256.txt -source.sha256 = "23611f3770a797b829c340e6dba25a4898678ece79db8534e06f76e438214e4c" +source.sha256 = "4645454e3af72ee68bc3916042f78deaac44a8c07bb6c1a1e5766b83f5326e07" output.type = "zone" # Refer to @@ -266,10 +266,10 @@ service_name = "propolis-server" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "propolis" -source.commit = "21ac8a9f5005f96caa384e3de0bd38283fc0188f" +source.commit = "9e12522341048d7b8c38394f0ddeb18083685c2f" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/image//propolis-server.sha256.txt -source.sha256 = "2a7b4bfa6d2b13714f68ec0095419218257ab584e763faac1d34baf38c8b09de" +source.sha256 = "084df2cf5866a17b5467dccdaddef5088cc021a1ee0eb3bea99472bcf67ea7fa" output.type = "zone" [package.maghemite] diff --git a/smf/clickhouse/method_script.sh b/smf/clickhouse/method_script.sh index 74782bb2f7b..c2784321109 100755 --- a/smf/clickhouse/method_script.sh +++ b/smf/clickhouse/method_script.sh @@ -17,6 +17,13 @@ if [[ $DATALINK == unknown ]] || [[ $GATEWAY == unknown ]]; then exit "$SMF_EXIT_ERR_CONFIG" fi +# TODO remove when https://github.com/oxidecomputer/stlouis/issues/435 is addressed +ipadm delete-if "$DATALINK" || true +ipadm create-if -t "$DATALINK" + +ipadm set-ifprop -t -p mtu=9000 -m ipv4 "$DATALINK" +ipadm set-ifprop -t -p mtu=9000 -m ipv6 "$DATALINK" + ipadm show-addr "$DATALINK/ll" || ipadm create-addr -t -T addrconf "$DATALINK/ll" ipadm show-addr "$DATALINK/omicron6" || ipadm create-addr -t -T static -a "$LISTEN_ADDR" "$DATALINK/omicron6" route get -inet6 default -inet6 "$GATEWAY" || route add -inet6 default -inet6 "$GATEWAY" diff --git a/smf/cockroachdb/method_script.sh b/smf/cockroachdb/method_script.sh index 38c3831702b..ee42ab18917 100755 --- a/smf/cockroachdb/method_script.sh +++ b/smf/cockroachdb/method_script.sh @@ -17,6 +17,13 @@ if [[ $DATALINK == unknown ]] || [[ $GATEWAY == unknown ]]; then exit "$SMF_EXIT_ERR_CONFIG" fi +# TODO remove when https://github.com/oxidecomputer/stlouis/issues/435 is addressed +ipadm delete-if "$DATALINK" || true +ipadm create-if -t "$DATALINK" + +ipadm set-ifprop -t -p mtu=9000 -m ipv4 "$DATALINK" +ipadm set-ifprop -t -p mtu=9000 -m ipv6 "$DATALINK" + ipadm show-addr "$DATALINK/ll" || ipadm create-addr -t -T addrconf "$DATALINK/ll" ipadm show-addr "$DATALINK/omicron6" || ipadm create-addr -t -T static -a "$LISTEN_ADDR" "$DATALINK/omicron6" route get -inet6 default -inet6 "$GATEWAY" || route add -inet6 default -inet6 "$GATEWAY"