From 5ae31bd7435e9869a1c277c314dc4a9faa6cd37d Mon Sep 17 00:00:00 2001 From: Benjamin Naecker Date: Wed, 16 Mar 2022 16:37:44 +0000 Subject: [PATCH] Initial integration with Oxide Packet Transformation Engine - Brings in OPTE via the `opte-ioctl` and `opte` crates. - Modifies the instance-ensure request from Nexus to the sled agent, to carry the actual information required for setting up the guest OPTE port. This includes the actual IP subnet and MAC, rather than things like the VPC Subnet UUID. - Adds a database query and method to extract the above information from both the network interface and VPC subnet tables. - Adds OPTE port for the guests (and currently still a VNIC on top), with the right OPTE settings for traffic to flow between two guests in the same VPC subnet. That's the virtual-to-physical mapping and a router entry for the subnet. - Adds the VNICs over each OPTE port to the running zone. Note that this removes the specification of guest NICs for the zone itself as VNICs. They are passed as OPTE ports, and the VNIC is pulled out internally, so hopefully little will need to change when the VNIC is removed entirely. - Store the main underlay address for the sled agent, currently its dropshot server IP address, in the instance manager, and forward to each instance. It's then used as the underlay address when setting up the OPTE ports for the guest. Addressing review comments Add a unique VNI to each VPC Updating OPTE dependency and package repos Add dummy/mock module for OPTE on non-illumos systems Update error handling to be more self-contained in the callee --- Cargo.lock | 806 ++++++++++++++++++++----- common/src/api/external/mod.rs | 65 ++ common/src/sql/dbinit.sql | 12 + nexus/src/db/datastore.rs | 81 +++ nexus/src/db/model/mod.rs | 2 + nexus/src/db/model/vni.rs | 42 ++ nexus/src/db/model/vpc.rs | 3 + nexus/src/db/schema.rs | 1 + nexus/src/nexus.rs | 20 +- nexus/src/sagas.rs | 51 +- openapi/sled-agent.json | 117 ++-- sled-agent-client/src/lib.rs | 47 +- sled-agent/Cargo.toml | 5 + sled-agent/src/illumos/running_zone.rs | 21 +- sled-agent/src/illumos/vnic.rs | 18 +- sled-agent/src/illumos/zone.rs | 15 +- sled-agent/src/instance.rs | 96 +-- sled-agent/src/instance_manager.rs | 31 +- sled-agent/src/lib.rs | 1 + sled-agent/src/opte/mock_opte.rs | 185 ++++++ sled-agent/src/opte/mod.rs | 11 + sled-agent/src/opte/opte.rs | 298 +++++++++ sled-agent/src/params.rs | 17 +- sled-agent/src/server.rs | 2 +- sled-agent/src/sled_agent.rs | 18 +- smf/sled-agent/config.toml | 7 +- tools/install_opte.sh | 20 +- 27 files changed, 1633 insertions(+), 359 deletions(-) create mode 100644 nexus/src/db/model/vni.rs create mode 100644 sled-agent/src/opte/mock_opte.rs create mode 100644 sled-agent/src/opte/mod.rs create mode 100644 sled-agent/src/opte/opte.rs diff --git a/Cargo.lock b/Cargo.lock index 2db98f030ed..02b367c665e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cipher", "cpufeatures", "opaque-debug 0.3.0", @@ -44,6 +44,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ahash" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29661b60bec623f0586702976ff4d0c9942dcb6723161c2df0eea78455cfedfb" +dependencies = [ + "const-random", +] + [[package]] name = "ahash" version = "0.7.6" @@ -79,6 +88,14 @@ version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" +[[package]] +name = "anymap" +version = "0.12.1" +source = "git+https://github.com/michaelmelanson/anymap?branch=no_std#2957b71eef770e50222c1398d390324697e0c928" +dependencies = [ + "hashbrown 0.6.3", +] + [[package]] name = "api_identity" version = "0.1.0" @@ -153,6 +170,15 @@ dependencies = [ "syn", ] +[[package]] +name = "atomic-polyfill" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14bf7b4f565e5e717d7a7a65b2a05c0b8c96e4db636d6f780f03b15108cdd1b" +dependencies = [ + "critical-section", +] + [[package]] name = "atty" version = "0.2.14" @@ -176,6 +202,15 @@ dependencies = [ "syn", ] +[[package]] +name = "autocfg" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78" +dependencies = [ + "autocfg 1.1.0", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -196,6 +231,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "bare-metal" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5deb64efa5bd81e31fcd1938615a6d98c82eafcbcd787162b6f63b91d6bac5b3" +dependencies = [ + "rustc_version 0.2.3", +] + +[[package]] +name = "bare-metal" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fe8f5a8a398345e52358e18ff07cc17a568fbca5c6f73873d3a62056309603" + [[package]] name = "base64" version = "0.13.0" @@ -239,6 +289,18 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bit_field" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4" + +[[package]] +name = "bitfield" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46afbd2983a5d5a7bd740ccb198caf5b82f45c40c09c0eed36052d91cb92e719" + [[package]] name = "bitflags" version = "1.3.2" @@ -350,6 +412,12 @@ version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" @@ -433,6 +501,17 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "colored" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd" +dependencies = [ + "atty", + "lazy_static", + "winapi", +] + [[package]] name = "console" version = "0.15.0" @@ -454,6 +533,28 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d6f2aa4d0537bcc1c74df8755072bd31c1ef1a3a1b85a68e8404a8c353b7b8b" +[[package]] +name = "const-random" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f590d95d011aa80b063ffe3253422ed5aa462af4e9867d43ce8337562bac77c4" +dependencies = [ + "const-random-macro", + "proc-macro-hack", +] + +[[package]] +name = "const-random-macro" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "615f6e27d000a2bffbc7f2f6a8669179378fa27ee4d0a509e985dfc0a7defb40" +dependencies = [ + "getrandom", + "lazy_static", + "proc-macro-hack", + "tiny-keccak", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -492,6 +593,18 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "cortex-m" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ff967e867ca14eba0c34ac25cd71ea98c678e741e3915d923999bb2fe7c826" +dependencies = [ + "bare-metal 0.2.5", + "bitfield", + "embedded-hal", + "volatile-register", +] + [[package]] name = "cpufeatures" version = "0.2.2" @@ -507,7 +620,7 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -548,13 +661,25 @@ dependencies = [ "itertools", ] +[[package]] +name = "critical-section" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95da181745b56d4bd339530ec393508910c909c784e8962d15d722bacf0bcbcd" +dependencies = [ + "bare-metal 1.0.0", + "cfg-if 1.0.0", + "cortex-m", + "riscv", +] + [[package]] name = "crossbeam" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae5588f6b3c3cb05239e90bd110f257254aecd01e4635400391aeae07497845" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", @@ -568,7 +693,7 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crossbeam-utils", ] @@ -578,7 +703,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crossbeam-epoch", "crossbeam-utils", ] @@ -589,8 +714,8 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" dependencies = [ - "autocfg", - "cfg-if", + "autocfg 1.1.0", + "cfg-if 1.0.0", "crossbeam-utils", "lazy_static", "memoffset", @@ -603,7 +728,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f25d8400f4a7a5778f0e4e52384a48cbd9b5c495d110786187fc750075277a2" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crossbeam-utils", ] @@ -613,7 +738,7 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "lazy_static", ] @@ -750,6 +875,26 @@ dependencies = [ "subtle", ] +[[package]] +name = "cstr-argument" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bd9c8e659a473bce955ae5c35b116af38af11a7acb0b480e01f3ed348aeb40" +dependencies = [ + "cfg-if 1.0.0", + "memchr", +] + +[[package]] +name = "cstr_core" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "644828c273c063ab0d39486ba42a5d1f3a499d35529c759e763a9c6cb8a0fb08" +dependencies = [ + "cty", + "memchr", +] + [[package]] name = "csv" version = "1.1.6" @@ -781,11 +926,17 @@ dependencies = [ "cipher", ] +[[package]] +name = "cty" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" + [[package]] name = "darling" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0d720b8683f8dd83c65155f0530560cba68cd2bf395f6513a483caee57ff7f4" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" dependencies = [ "darling_core", "darling_macro", @@ -793,9 +944,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a340f241d2ceed1deb47ae36c4144b2707ec7dd0b649f894cb39bb595986324" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ "fnv", "ident_case", @@ -807,9 +958,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c41b3b7352feb3211a0d743dc5700a4e3b60f51bd2b368892d1e0f9a95f44b" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core", "quote", @@ -934,7 +1085,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "dirs-sys-next", ] @@ -949,6 +1100,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "dlpi" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/dlpi-sys#0cc34b488374eb7a5c4904759b45c70285c4f049" +dependencies = [ + "libc", + "num_enum", + "pretty-hex 0.2.1", + "thiserror", +] + [[package]] name = "doc-comment" version = "0.3.3" @@ -962,7 +1124,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e6b21a1211455e82b1245d6e1b024f30606afbb734c114515d40d0e0b34ce81" dependencies = [ "thiserror", - "zerocopy", + "zerocopy 0.3.0", ] [[package]] @@ -1073,6 +1235,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "embedded-hal" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35949884794ad573cf46071e41c9b60efb0cb311e3ca01f7af807af1debc66ff" +dependencies = [ + "nb 0.1.3", + "void", +] + [[package]] name = "ena" version = "0.14.0" @@ -1090,11 +1262,11 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.30" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -1191,11 +1363,11 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "975ccf83d8d9d0d84682850a38c8169027be83368805971cc4f238c2b245bc98" +checksum = "c0408e2626025178a6a7f7ffc05a25bc47103229f19c113755de7bf63816290c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "redox_syscall", "winapi", @@ -1215,11 +1387,11 @@ checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" [[package]] name = "flate2" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crc32fast", "libc", "miniz_oxide", @@ -1246,7 +1418,28 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" dependencies = [ - "foreign-types-shared", + "foreign-types-shared 0.1.1", +] + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared 0.3.1", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8469d0d40519bc608ec6863f1cc88f3f1deee15913f2f3b3e573d81ed38cccc" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1255,6 +1448,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -1267,9 +1466,9 @@ dependencies = [ [[package]] name = "fragile" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da1b8f89c5b5a5b7e59405cfcf0bb9588e5ed19f0b57a4cd542bbba3f164a6d" +checksum = "e9d758e60b45e8d749c89c1b389ad8aee550f86aa12e2b9298b546dda7a82ab1" [[package]] name = "fs2" @@ -1486,11 +1685,11 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi 0.10.0+wasi-snapshot-preview1", ] @@ -1528,9 +1727,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" dependencies = [ "bytes", "fnv", @@ -1541,7 +1740,7 @@ dependencies = [ "indexmap", "slab", "tokio", - "tokio-util 0.6.9", + "tokio-util 0.7.1", "tracing", ] @@ -1551,13 +1750,32 @@ version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e6073d0ca812575946eb5f35ff68dbe519907b25c42530389ff946dc84c6ead" +dependencies = [ + "ahash 0.2.19", + "autocfg 0.1.8", +] + [[package]] name = "hashbrown" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" dependencies = [ - "ahash", + "ahash 0.7.6", ] [[package]] @@ -1566,7 +1784,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" dependencies = [ - "hashbrown", + "hashbrown 0.11.2", ] [[package]] @@ -1594,6 +1812,19 @@ dependencies = [ "http", ] +[[package]] +name = "heapless" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d076121838e03f862871315477528debffdb7462fb229216ecef91b1a3eb31eb" +dependencies = [ + "atomic-polyfill", + "hash32", + "serde", + "spin 0.9.3", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.3.3" @@ -1687,9 +1918,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9100414882e15fb7feccb4897e5f0ff0ff1ca7d1a86a23208ada4d7a18e6c6c4" +checksum = "6330e8a36bd8c859f3fa6d9382911fbb7147ec39807f63b923933a247240b9ba" [[package]] name = "httpdate" @@ -1811,6 +2042,19 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "illumos-ddi-dki" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/opte?rev=cb1767c#cb1767c80d4e9d97cb79901eed3c9d08e1fb3826" +dependencies = [ + "illumos-sys-hdrs", +] + +[[package]] +name = "illumos-sys-hdrs" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/opte?rev=cb1767c#cb1767c80d4e9d97cb79901eed3c9d08e1fb3826" + [[package]] name = "impl-trait-for-tuples" version = "0.2.2" @@ -1828,8 +2072,8 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" dependencies = [ - "autocfg", - "hashbrown", + "autocfg 1.1.0", + "hashbrown 0.11.2", "serde", ] @@ -1861,7 +2105,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -1877,7 +2121,7 @@ dependencies = [ "openapi-lint", "openapiv3", "portpicker", - "pretty-hex", + "pretty-hex 0.3.0", "schemars", "serde", "serde_json", @@ -1922,9 +2166,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e70ee094dc02fd9c13fdad4940090f22dbd6ac7c9e7094a46cf0232a50bc7c" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" [[package]] name = "ipnetwork" @@ -1958,9 +2202,9 @@ checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" [[package]] name = "js-sys" -version = "0.3.56" +version = "0.3.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" dependencies = [ "wasm-bindgen", ] @@ -2005,11 +2249,28 @@ version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb691a747a7ab48abc15c5b42066eaafde10dc427e3b6ee2a1cf43db04c763bd" +[[package]] +name = "libnet" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/netadm-sys#435d0936639133a451fe64f7f0d2c568e768f4c7" +dependencies = [ + "anyhow", + "colored", + "dlpi", + "libc", + "nvpair", + "nvpair-sys", + "rusty-doors", + "socket2", + "thiserror", + "tracing", +] + [[package]] name = "libsqlite3-sys" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb644c388dfaefa18035c12614156d285364769e818893da0dda9030c80ad2ba" +checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14" dependencies = [ "pkg-config", "vcpkg", @@ -2036,7 +2297,7 @@ version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -2057,6 +2318,12 @@ dependencies = [ "serde", ] +[[package]] +name = "managed" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" + [[package]] name = "maplit" version = "1.0.2" @@ -2096,7 +2363,7 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" dependencies = [ - "autocfg", + "autocfg 1.1.0", ] [[package]] @@ -2117,12 +2384,11 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.4" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082" dependencies = [ "adler", - "autocfg", ] [[package]] @@ -2154,7 +2420,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d4d70639a72f972725db16350db56da68266ca368b2a1fe26724a903ad3d6b8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "downcast", "fragile", "lazy_static", @@ -2169,7 +2435,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79ef208208a0dea3f72221e26e904cdc6db2e481d9ade89081ddd494f1dbaa6b" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "proc-macro2", "quote", "syn", @@ -2177,9 +2443,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" dependencies = [ "lazy_static", "libc", @@ -2193,6 +2459,21 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nb" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "801d31da0513b6ec5214e9bf433a77966320625a37860f910be265be6e18d06f" +dependencies = [ + "nb 1.0.0", +] + +[[package]] +name = "nb" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "546c37ac5d9e56f55e73b677106873d9d9f5190605e41a856503623648488cae" + [[package]] name = "new_debug_unreachable" version = "1.0.4" @@ -2293,7 +2574,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ - "autocfg", + "autocfg 1.1.0", "num-traits", ] @@ -2303,7 +2584,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ - "autocfg", + "autocfg 1.1.0", ] [[package]] @@ -2316,6 +2597,27 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf5395665662ef45796a4ff5486c5d41d29e0c09640af4c5f17fd94ee2c119c9" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0498641e53dd6ac1a4f22547548caa6864cc4933784319cd1775271c5a46ce" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "num_threads" version = "0.1.5" @@ -2331,6 +2633,21 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "nvpair" +version = "0.5.0" +source = "git+https://github.com/jmesmon/rust-libzfs?branch=master#2d9b97f220926d95d91a187fbce1d61b8c2209c8" +dependencies = [ + "cstr-argument", + "foreign-types 0.5.0", + "nvpair-sys", +] + +[[package]] +name = "nvpair-sys" +version = "0.4.0" +source = "git+https://github.com/jmesmon/rust-libzfs?branch=master#2d9b97f220926d95d91a187fbce1d61b8c2209c8" + [[package]] name = "olpc-cjson" version = "0.1.1" @@ -2540,7 +2857,7 @@ dependencies = [ "async-trait", "bincode", "bytes", - "cfg-if", + "cfg-if 1.0.0", "chrono", "crucible-agent-client", "dropshot", @@ -2548,6 +2865,7 @@ dependencies = [ "futures", "http", "ipnetwork", + "libc", "macaddr", "mockall", "nexus-client", @@ -2555,6 +2873,8 @@ dependencies = [ "omicron-test-utils", "openapi-lint", "openapiv3", + "opte", + "opte-ioctl", "p256", "percent-encoding", "progenitor", @@ -2679,8 +2999,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" dependencies = [ "bitflags", - "cfg-if", - "foreign-types", + "cfg-if 1.0.0", + "foreign-types 0.3.2", "libc", "once_cell", "openssl-sys", @@ -2698,13 +3018,43 @@ version = "0.9.72" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" dependencies = [ - "autocfg", + "autocfg 1.1.0", "cc", "libc", "pkg-config", "vcpkg", ] +[[package]] +name = "opte" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/opte?rev=cb1767c#cb1767c80d4e9d97cb79901eed3c9d08e1fb3826" +dependencies = [ + "anymap", + "cfg-if 0.1.10", + "cstr_core", + "heapless", + "illumos-ddi-dki", + "illumos-sys-hdrs", + "postcard", + "serde", + "smoltcp", + "zerocopy 0.6.1", +] + +[[package]] +name = "opte-ioctl" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/opte?rev=cb1767c#cb1767c80d4e9d97cb79901eed3c9d08e1fb3826" +dependencies = [ + "libc", + "libnet", + "opte", + "postcard", + "serde", + "thiserror", +] + [[package]] name = "os_str_bytes" version = "6.0.0" @@ -2888,7 +3238,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" dependencies = [ "lock_api", - "parking_lot_core 0.9.1", + "parking_lot_core 0.9.2", ] [[package]] @@ -2897,7 +3247,7 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "instant", "libc", "redox_syscall", @@ -2907,11 +3257,11 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "redox_syscall", "smallvec", @@ -2952,18 +3302,18 @@ checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" [[package]] name = "path-absolutize" -version = "3.0.12" +version = "3.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a2a79d7c1c4eab523515c4561459b10516d6e7014aa76edc3ea05680d5c5d2d" +checksum = "d3de4b40bd9736640f14c438304c09538159802388febb02c8abaae0846c1f13" dependencies = [ "path-dedot", ] [[package]] name = "path-dedot" -version = "3.0.16" +version = "3.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f326e2a3331685a5e3d4633bb9836bd92126e08037cb512252f3612f616a0b28" +checksum = "d611d5291372b3738a34ebf0d1f849e58b1dcc1101032f76a346eaa1f8ddbb5b" dependencies = [ "once_cell", ] @@ -3088,9 +3438,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "plotters" @@ -3142,7 +3492,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8419d2b623c7c0896ff2d5d96e2cb4ede590fed28fcc34934f4c33c036e620a1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "opaque-debug 0.3.0", "universal-hash", @@ -3157,6 +3507,23 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "postcard" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a25c0b0ae06fcffe600ad392aabfa535696c8973f2253d9ac83171924c58a858" +dependencies = [ + "heapless", + "postcard-cobs", + "serde", +] + +[[package]] +name = "postcard-cobs" +version = "0.1.5-pre" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c68cb38ed13fd7bc9dd5db8f165b7c8d9c1a315104083a2b10f11354c2af97f" + [[package]] name = "postgres-protocol" version = "0.6.3" @@ -3240,12 +3607,28 @@ dependencies = [ "termtree", ] +[[package]] +name = "pretty-hex" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5c99d529f0d30937f6f4b8a86d988047327bb88d04d2c4afc356de74722131" + [[package]] name = "pretty-hex" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6fa0831dd7cc608c38a5e323422a0077678fa5744aa2be4ad91c4ece8eec8d5" +[[package]] +name = "proc-macro-crate" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17d47ce914bf4de440332250b0edd23ce48c005f59fab39d3335866b114f11a" +dependencies = [ + "thiserror", + "toml", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -3270,6 +3653,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + [[package]] name = "proc-macro2" version = "1.0.37" @@ -3467,7 +3856,7 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd249e82c21598a9a426a4e00dd7adc1d640b22445ec8545feef801d1a74c221" dependencies = [ - "autocfg", + "autocfg 1.1.0", "crossbeam-deque", "either", "rayon-core", @@ -3496,18 +3885,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae183fc1b06c149f0c1793e1eb447c8b04bfe46d48e9e48bfb8d2d7ed64ecf0" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ "bitflags", ] [[package]] name = "redox_users" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7776223e2696f1aa4c6b0170e83212f47296a00424305117d013dfe86fb0fe55" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom", "redox_syscall", @@ -3627,7 +4016,7 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", + "spin 0.5.2", "untrusted", "web-sys", "winapi", @@ -3642,6 +4031,27 @@ dependencies = [ "array-init", ] +[[package]] +name = "riscv" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6907ccdd7a31012b70faf2af85cd9e5ba97657cc3987c4f13f8e4d2c2a088aba" +dependencies = [ + "bare-metal 1.0.0", + "bit_field", + "riscv-target", +] + +[[package]] +name = "riscv-target" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88aa938cda42a0cf62a20cfe8d139ff1af20c2e681212b5b34adb5a58333f222" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "rusqlite" version = "0.27.0" @@ -3666,13 +4076,22 @@ dependencies = [ "semver 0.1.20", ] +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver 0.9.0", +] + [[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.6", + "semver 1.0.7", ] [[package]] @@ -3722,6 +4141,24 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" +[[package]] +name = "rusty-doors" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/rusty-doors#c9c064efbb686af124264ee6995d9db7082b249d" +dependencies = [ + "libc", + "rusty-doors-macros", +] + +[[package]] +name = "rusty-doors-macros" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/rusty-doors#c9c064efbb686af124264ee6995d9db7082b249d" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "ryu" version = "1.0.9" @@ -3828,20 +4265,35 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4f410fedcf71af0345d7607d246e7ad15faaadd49d240ee3b24e5dc21a820ac" +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser 0.7.0", +] + [[package]] name = "semver" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" dependencies = [ - "semver-parser", + "semver-parser 0.10.2", ] [[package]] name = "semver" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d" +checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "semver-parser" @@ -3969,9 +4421,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12e47be9471c72889ebafb5e14d5ff930d89ae7a67bbdb5f8abb564f845a927e" +checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" dependencies = [ "darling", "proc-macro2", @@ -4021,7 +4473,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.10.3", ] @@ -4033,7 +4485,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.9.0", "opaque-debug 0.3.0", @@ -4045,7 +4497,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.10.3", ] @@ -4114,9 +4566,9 @@ checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" [[package]] name = "slab" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" [[package]] name = "sled" @@ -4269,6 +4721,17 @@ dependencies = [ "thiserror", ] +[[package]] +name = "smoltcp" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2308a1657c8db1f5b4993bab4e620bdbe5623bd81f254cf60326767bb243237" +dependencies = [ + "bitflags", + "byteorder", + "managed", +] + [[package]] name = "snafu" version = "0.7.0" @@ -4338,6 +4801,15 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c530c2b0d0bf8b69304b39fe2001993e267461948b890cd037d8ad4293fa1a0d" +dependencies = [ + "lock_api", +] + [[package]] name = "spki" version = "0.4.1" @@ -4347,6 +4819,12 @@ dependencies = [ "der", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -4375,13 +4853,13 @@ dependencies = [ [[package]] name = "string_cache" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33994d0838dc2d152d17a62adf608a869b5e846b65b389af7f3dbc1de45c5b26" +checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08" dependencies = [ - "lazy_static", "new_debug_unreachable", - "parking_lot 0.11.2", + "once_cell", + "parking_lot 0.12.0", "phf_shared", "precomputed-hash", ] @@ -4546,7 +5024,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fastrand", "libc", "redox_syscall", @@ -4891,11 +5369,11 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.32" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f" +checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "log", "pin-project-lite", "tracing-attributes", @@ -4915,9 +5393,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.23" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa31669fa42c09c34d94d8165dd2012e8ff3c66aca50f3bb226b68f216f2706c" +checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" dependencies = [ "lazy_static", "valuable", @@ -4925,9 +5403,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e0ab7bdc962035a87fba73f3acca9b8a8d0034c2e6f60b84aeaaddddc155dce" +checksum = "4bc28f93baff38037f64e6f43d34cfa1605f27a49c34e8a04c5e78b0babf2596" dependencies = [ "sharded-slab", "thread_local", @@ -4936,11 +5414,11 @@ dependencies = [ [[package]] name = "trust-dns-client" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3be5f2ead860f0d3aabc01433bc6fff0fe5e86bfbe2dd16e32b9c79959310ad" +checksum = "a6d9ba1c6079f6f9b4664e482db1700bd53d2ee77b1c9752c1d7a66c0c8bda99" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "data-encoding", "futures-channel", "futures-util", @@ -4961,7 +5439,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c31f240f59877c3d4bb3b3ea0ec5a6a0cff07323580ff8c7a605cd7d08b255d" dependencies = [ "async-trait", - "cfg-if", + "cfg-if 1.0.0", "data-encoding", "enum-as-inner", "futures-channel", @@ -4985,7 +5463,7 @@ version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4ba72c2ea84515690c9fcef4c6c660bb9df3036ed1051686de84605b74fd558" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "futures-util", "ipconfig", "lazy_static", @@ -5007,7 +5485,7 @@ checksum = "a395a2e0fd8aac9b4613767a5b4ba4b2040de1b767fa03ace8c9d6f351d60b2d" dependencies = [ "async-trait", "bytes", - "cfg-if", + "cfg-if 1.0.0", "enum-as-inner", "env_logger", "futures-executor", @@ -5068,7 +5546,7 @@ version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee73e6e4924fe940354b8d4d98cad5231175d615cd855b758adc658c0aac6a0" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "rand 0.8.5", "static_assertions", ] @@ -5286,6 +5764,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcell" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77439c1b53d2303b20d9459b1ade71a83c716e3f9c34f3228c00e6f185d6c002" + [[package]] name = "vcpkg" version = "0.2.15" @@ -5304,6 +5788,21 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "volatile-register" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ee8f19f9d74293faf70901bc20ad067dc1ad390d2cbf1e3f75f721ffee908b6" +dependencies = [ + "vcell", +] + [[package]] name = "vsss-rs" version = "2.0.0-pre0" @@ -5356,19 +5855,19 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" dependencies = [ "bumpalo", "lazy_static", @@ -5381,11 +5880,11 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.29" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb6ec270a31b1d3c7e266b999739109abce8b6c87e4b31fcfcd788b65267395" +checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "wasm-bindgen", "web-sys", @@ -5393,9 +5892,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5403,9 +5902,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" dependencies = [ "proc-macro2", "quote", @@ -5416,15 +5915,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" [[package]] name = "web-sys" -version = "0.3.56" +version = "0.3.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" +checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" dependencies = [ "js-sys", "wasm-bindgen", @@ -5442,9 +5941,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552ceb903e957524388c4d3475725ff2c8b7960922063af6ce53c9a43da07449" +checksum = "44d8de8415c823c8abd270ad483c6feeac771fad964890779f9a8cb24fbbc1bf" dependencies = [ "webpki", ] @@ -5488,9 +5987,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825" dependencies = [ "windows_aarch64_msvc", "windows_i686_gnu", @@ -5501,33 +6000,33 @@ dependencies = [ [[package]] name = "windows_aarch64_msvc" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" +checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" [[package]] name = "windows_i686_gnu" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" +checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" [[package]] name = "windows_i686_msvc" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" +checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" [[package]] name = "windows_x86_64_gnu" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" +checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" [[package]] name = "windows_x86_64_msvc" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" +checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" [[package]] name = "winreg" @@ -5572,7 +6071,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6580539ad917b7c026220c4b3f2c08d52ce54d6ce0dc491e66002e35388fab46" dependencies = [ "byteorder", - "zerocopy-derive", + "zerocopy-derive 0.2.0", +] + +[[package]] +name = "zerocopy" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332f188cc1bcf1fe1064b8c58d150f497e697f49774aa846f2dc949d9a25f236" +dependencies = [ + "byteorder", + "zerocopy-derive 0.3.1", ] [[package]] @@ -5586,6 +6095,17 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy-derive" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0fbc82b82efe24da867ee52e015e58178684bd9dd64c34e66bdf21da2582a9f" +dependencies = [ + "proc-macro2", + "syn", + "synstructure", +] + [[package]] name = "zeroize" version = "1.3.0" diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 29c9824cf55..6ceab0ddcb3 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1170,6 +1170,15 @@ impl FromStr for IpNet { } } +impl From for ipnetwork::IpNetwork { + fn from(net: IpNet) -> ipnetwork::IpNetwork { + match net { + IpNet::V4(net) => ipnetwork::IpNetwork::from(net.0), + IpNet::V6(net) => ipnetwork::IpNetwork::from(net.0), + } + } +} + /// A `RouteTarget` describes the possible locations that traffic matching a /// route destination can be sent. #[derive( @@ -1668,6 +1677,62 @@ impl JsonSchema for MacAddr { } } +/// A Geneve Virtual Network Identifier +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + Deserialize, + Serialize, + JsonSchema, +)] +pub struct Vni(u32); + +impl Vni { + const MAX_VNI: u32 = 1 << 24; + + /// Create a new random VNI. + pub fn random() -> Self { + use rand::Rng; + Self(rand::thread_rng().gen_range(0..=Self::MAX_VNI)) + } +} + +impl From for u32 { + fn from(vni: Vni) -> u32 { + vni.0 + } +} + +impl TryFrom for Vni { + type Error = Error; + + fn try_from(x: u32) -> Result { + if x <= Self::MAX_VNI { + Ok(Self(x)) + } else { + Err(Error::internal_error( + format!("Invalid Geneve VNI: {}", x).as_str(), + )) + } + } +} + +impl TryFrom for Vni { + type Error = Error; + + fn try_from(x: i32) -> Result { + Self::try_from(u32::try_from(x).map_err(|_| { + Error::internal_error(format!("Invalid Geneve VNI: {}", x).as_str()) + })?) + } +} + /// A `NetworkInterface` represents a virtual network interface device. #[derive(ObjectIdentity, Clone, Debug, Deserialize, JsonSchema, Serialize)] pub struct NetworkInterface { diff --git a/common/src/sql/dbinit.sql b/common/src/sql/dbinit.sql index aabafc84eac..61b95eec652 100644 --- a/common/src/sql/dbinit.sql +++ b/common/src/sql/dbinit.sql @@ -592,6 +592,13 @@ CREATE TABLE omicron.public.vpc ( system_router_id UUID NOT NULL, dns_name STRING(63) NOT NULL, + /* + * The Geneve Virtual Network Identifier for this VPC. Note that this is a + * 24-bit unsigned value, properties which are checked in the application, + * not the database. + */ + vni INT4 NOT NULL, + /* The IPv6 prefix allocated to subnets. */ ipv6_prefix INET NOT NULL, @@ -606,6 +613,11 @@ CREATE UNIQUE INDEX ON omicron.public.vpc ( ) WHERE time_deleted IS NULL; +CREATE UNIQUE INDEX ON omicron.public.vpc ( + vni +) WHERE + time_deleted IS NULL; + CREATE TABLE omicron.public.vpc_subnet ( /* Identity metadata (resource) */ id UUID PRIMARY KEY, diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index 2362f8cebf0..a6b2ba3e46d 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -65,6 +65,7 @@ use diesel::query_dsl::methods::LoadQuery; use diesel::upsert::excluded; use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; use omicron_common::api; +use omicron_common::api::external; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; @@ -77,6 +78,7 @@ use omicron_common::api::external::{ CreateResult, IdentityMetadataCreateParams, }; use omicron_common::bail_unless; +use sled_agent_client::types as sled_client_types; use std::convert::{TryFrom, TryInto}; use std::net::Ipv6Addr; use std::sync::Arc; @@ -1365,6 +1367,85 @@ impl DataStore { Ok(()) } + /// Return the information about an instance's network interfaces required + /// for the sled agent to instantiate them via OPTE. + /// + /// OPTE requires information that's currently split across the network + /// interface and VPC subnet tables. This query just joins those for each + /// NIC in the given instance. + pub(crate) async fn derive_guest_network_interface_info( + &self, + opctx: &OpContext, + authz_instance: &authz::Instance, + ) -> ListResultVec { + opctx.authorize(authz::Action::ListChildren, authz_instance).await?; + + use db::schema::network_interface; + use db::schema::vpc; + use db::schema::vpc_subnet; + + // The record type for the results of the below JOIN query + #[derive(Debug, diesel::Queryable)] + struct NicInfo { + name: db::model::Name, + ip: ipnetwork::IpNetwork, + mac: db::model::MacAddr, + ipv4_block: db::model::Ipv4Net, + ipv6_block: db::model::Ipv6Net, + vni: db::model::Vni, + slot: i16, + } + + impl From for sled_client_types::NetworkInterface { + fn from(nic: NicInfo) -> sled_client_types::NetworkInterface { + let ip_subnet = if nic.ip.is_ipv4() { + external::IpNet::V4(nic.ipv4_block.0) + } else { + external::IpNet::V6(nic.ipv6_block.0) + }; + sled_client_types::NetworkInterface { + name: sled_client_types::Name::from(&nic.name.0), + ip: nic.ip.ip().to_string(), + mac: sled_client_types::MacAddr::from(nic.mac.0), + subnet: sled_client_types::IpNet::from(ip_subnet), + vni: sled_client_types::Vni::from(nic.vni.0), + slot: u8::try_from(nic.slot).unwrap(), + } + } + } + + let rows = network_interface::table + .filter(network_interface::instance_id.eq(authz_instance.id())) + .filter(network_interface::time_deleted.is_null()) + .inner_join( + vpc_subnet::table + .on(network_interface::subnet_id.eq(vpc_subnet::id)), + ) + .inner_join(vpc::table.on(vpc_subnet::vpc_id.eq(vpc::id))) + .order_by(network_interface::slot) + // TODO-cleanup: Having to specify each column again is less than + // ideal, but we can't derive `Selectable` since this is the result + // of a JOIN and not from a single table. DRY this out if possible. + .select(( + network_interface::name, + network_interface::ip, + network_interface::mac, + vpc_subnet::ipv4_block, + vpc_subnet::ipv6_block, + vpc::vni, + network_interface::slot, + )) + .get_results_async::(self.pool_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel_pool(e, ErrorHandler::Server) + })?; + Ok(rows + .into_iter() + .map(sled_client_types::NetworkInterface::from) + .collect()) + } + /// List network interfaces associated with a given instance. pub async fn instance_list_network_interfaces( &self, diff --git a/nexus/src/db/model/mod.rs b/nexus/src/db/model/mod.rs index fcae80caaa5..21ece88b3fe 100644 --- a/nexus/src/db/model/mod.rs +++ b/nexus/src/db/model/mod.rs @@ -40,6 +40,7 @@ mod ssh_key; mod u16; mod update_artifact; mod user_builtin; +mod vni; mod volume; mod vpc; mod vpc_firewall_rule; @@ -84,6 +85,7 @@ pub use snapshot::*; pub use ssh_key::*; pub use update_artifact::*; pub use user_builtin::*; +pub use vni::*; pub use volume::*; pub use vpc::*; pub use vpc_firewall_rule::*; diff --git a/nexus/src/db/model/vni.rs b/nexus/src/db/model/vni.rs new file mode 100644 index 00000000000..73750d9c31f --- /dev/null +++ b/nexus/src/db/model/vni.rs @@ -0,0 +1,42 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use diesel::backend::Backend; +use diesel::backend::RawValue; +use diesel::deserialize; +use diesel::deserialize::FromSql; +use diesel::query_builder::bind_collector::RawBytesBindCollector; +use diesel::serialize; +use diesel::serialize::ToSql; +use diesel::sql_types; +use omicron_common::api::external; + +#[derive(Clone, Debug, Copy, AsExpression, FromSqlRow)] +#[diesel(sql_type = sql_types::Int4)] +pub struct Vni(pub external::Vni); + +impl ToSql for Vni +where + DB: Backend>, + i32: ToSql, +{ + fn to_sql<'b>( + &'b self, + out: &mut serialize::Output<'b, '_, DB>, + ) -> serialize::Result { + // Reborrowing is necessary to ensure that the lifetime of the temporary + // i32 created here and `out` is the same, i.e., that `'b = '_`. + i32::try_from(u32::from(self.0)).unwrap().to_sql(&mut out.reborrow()) + } +} + +impl FromSql for Vni +where + DB: Backend, + i32: FromSql, +{ + fn from_sql(bytes: RawValue) -> deserialize::Result { + Ok(Vni(external::Vni::try_from(i32::from_sql(bytes)?)?)) + } +} diff --git a/nexus/src/db/model/vpc.rs b/nexus/src/db/model/vpc.rs index 43052d61d68..b8bb20cad1b 100644 --- a/nexus/src/db/model/vpc.rs +++ b/nexus/src/db/model/vpc.rs @@ -4,6 +4,7 @@ use super::{Generation, Ipv6Net, Name, VpcFirewallRule}; use crate::db::collection_insert::DatastoreCollection; +use crate::db::model::Vni; use crate::db::schema::{vpc, vpc_firewall_rule}; use crate::defaults; use crate::external_api::params; @@ -20,6 +21,7 @@ pub struct Vpc { pub project_id: Uuid, pub system_router_id: Uuid, + pub vni: Vni, pub ipv6_prefix: Ipv6Net, pub dns_name: Name, @@ -54,6 +56,7 @@ impl Vpc { identity, project_id, system_router_id, + vni: Vni(external::Vni::random()), ipv6_prefix, dns_name: params.dns_name.into(), firewall_gen: Generation::new(), diff --git a/nexus/src/db/schema.rs b/nexus/src/db/schema.rs index 6b7e05df824..66d25ea7a8f 100644 --- a/nexus/src/db/schema.rs +++ b/nexus/src/db/schema.rs @@ -333,6 +333,7 @@ table! { time_deleted -> Nullable, project_id -> Uuid, system_router_id -> Uuid, + vni -> Int4, ipv6_prefix -> Inet, dns_name -> Text, firewall_gen -> Int8, diff --git a/nexus/src/nexus.rs b/nexus/src/nexus.rs index 33987abde25..dded9e6abe9 100644 --- a/nexus/src/nexus.rs +++ b/nexus/src/nexus.rs @@ -1847,22 +1847,10 @@ impl Nexus { }); } - let nics: Vec = self + let nics = self .db_datastore - .instance_list_network_interfaces( - &opctx, - &authz_instance, - &DataPageParams { - marker: None, - direction: dropshot::PaginationOrder::Ascending, - limit: std::num::NonZeroU32::new(MAX_NICS_PER_INSTANCE) - .unwrap(), - }, - ) - .await? - .iter() - .map(|x| x.clone().into()) - .collect(); + .derive_guest_network_interface_info(&opctx, &authz_instance) + .await?; // Ask the sled agent to begin the state change. Then update the // database to reflect the new intermediate state. If this update is @@ -1873,7 +1861,7 @@ impl Nexus { runtime: sled_agent_client::types::InstanceRuntimeState::from( db_instance.runtime().clone(), ), - nics: nics.iter().map(|nic| nic.clone().into()).collect(), + nics, disks: disk_reqs, cloud_init_bytes: Some(base64::encode( db_instance.generate_cidata()?, diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index 5e2447dd37c..87207d8941c 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -28,7 +28,6 @@ use omicron_common::api::external::Generation; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::InstanceState; use omicron_common::api::external::Name; -use omicron_common::api::external::NetworkInterface; use omicron_common::api::internal::nexus::InstanceRuntimeState; use omicron_common::backoff::{self, BackoffError}; use rand::{rngs::StdRng, RngCore, SeedableRng}; @@ -326,9 +325,9 @@ async fn sic_allocate_network_interface_ids( async fn sic_create_network_interfaces( sagactx: ActionContext, -) -> Result>, ActionError> { +) -> Result<(), ActionError> { match sagactx.saga_params().create_params.network_interfaces { - params::InstanceNetworkInterfaceAttachment::None => Ok(None), + params::InstanceNetworkInterfaceAttachment::None => Ok(()), params::InstanceNetworkInterfaceAttachment::Default => { sic_create_default_network_interface(&sagactx).await } @@ -345,9 +344,9 @@ async fn sic_create_network_interfaces( async fn sic_create_custom_network_interfaces( sagactx: &ActionContext, interface_params: &[params::NetworkInterfaceCreate], -) -> Result>, ActionError> { +) -> Result<(), ActionError> { if interface_params.is_empty() { - return Ok(Some(vec![])); + return Ok(()); } let osagactx = sagactx.user_data(); @@ -381,7 +380,6 @@ async fn sic_create_custom_network_interfaces( ))); } - let mut interfaces = Vec::with_capacity(interface_params.len()); if ids.len() != interface_params.len() { return Err(ActionError::action_failed(Error::internal_error( "found differing number of network interface IDs and interface \ @@ -392,7 +390,7 @@ async fn sic_create_custom_network_interfaces( // TODO-correctness: It seems racy to fetch the subnet and create the // interface in separate requests, but outside of a transaction. This // should probably either be in a transaction, or the - // `subnet_create_network_interface` function/query needs some JOIN + // `instance_create_network_interface` function/query needs some JOIN // on the `vpc_subnet` table. let (.., authz_subnet, db_subnet) = LookupPath::new(&opctx, &datastore) .vpc_id(authz_vpc.id()) @@ -406,7 +404,7 @@ async fn sic_create_custom_network_interfaces( interface_id, instance_id, authz_vpc.id(), - db_subnet, + db_subnet.clone(), mac, params.identity.clone(), params.ip, @@ -422,8 +420,8 @@ async fn sic_create_custom_network_interfaces( .await; use crate::db::subnet_allocation::NetworkInterfaceError; - let interface = match result { - Ok(interface) => Ok(interface), + match result { + Ok(_) => Ok(()), // Detect the specific error arising from this node being partially // completed. @@ -453,30 +451,19 @@ async fn sic_create_custom_network_interfaces( instance_id; "primary_key" => interface_id.to_string(), ); - - // Refetch the interface itself, to serialize it for the next - // saga node. - LookupPath::new(&opctx, &datastore) - .instance_id(authz_instance.id()) - .network_interface_name(&db::model::Name( - params.identity.name.clone(), - )) - .fetch() - .await - .map(|(.., db_interface)| db_interface) + Ok(()) } Err(e) => Err(e.into_external()), } .map_err(ActionError::action_failed)?; - interfaces.push(NetworkInterface::from(interface)) } - Ok(Some(interfaces)) + Ok(()) } /// Create the default network interface for an instance during the create saga async fn sic_create_default_network_interface( sagactx: &ActionContext, -) -> Result>, ActionError> { +) -> Result<(), ActionError> { let osagactx = sagactx.user_data(); let datastore = osagactx.datastore(); let saga_params = sagactx.saga_params(); @@ -519,13 +506,13 @@ async fn sic_create_default_network_interface( interface_id, instance_id, authz_vpc.id(), - db_subnet, + db_subnet.clone(), mac, interface_params.identity.clone(), interface_params.ip, ) .map_err(ActionError::action_failed)?; - let interface = datastore + datastore .instance_create_network_interface( &opctx, &authz_subnet, @@ -535,17 +522,17 @@ async fn sic_create_default_network_interface( .await .map_err(db::subnet_allocation::NetworkInterfaceError::into_external) .map_err(ActionError::action_failed)?; - Ok(Some(vec![interface.into()])) + Ok(()) } async fn sic_create_network_interfaces_undo( sagactx: ActionContext, ) -> Result<(), anyhow::Error> { - // We issue a request to delete any interfaces associated with this - // instance. In the case we failed partway through allocating interfaces, - // we won't have cached the interface records in the saga log, but they're - // definitely still in the database. Just delete every interface that - // exists, even if there are zero such records. + // We issue a request to delete any interfaces associated with this instance. + // In the case we failed partway through allocating interfaces, we need to + // clean up any previously-created interface records from the database. + // Just delete every interface that exists, even if there are zero such + // records. let osagactx = sagactx.user_data(); let datastore = osagactx.datastore(); let saga_params = sagactx.saga_params(); diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 740f9cd2dde..760183232d0 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -856,6 +856,51 @@ "destroyed" ] }, + "IpNet": { + "description": "An `IpNet` represents an IP network, either IPv4 or IPv6.", + "oneOf": [ + { + "type": "object", + "properties": { + "V4": { + "$ref": "#/components/schemas/Ipv4Net" + } + }, + "required": [ + "V4" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "V6": { + "$ref": "#/components/schemas/Ipv6Net" + } + }, + "required": [ + "V6" + ], + "additionalProperties": false + } + ] + }, + "Ipv4Net": { + "example": "192.168.1.0/24", + "title": "An IPv4 subnet", + "description": "An IPv4 subnet, including prefix and subnet mask", + "type": "string", + "pattern": "(^(10\\.(25[0-5]|[1-2][0-4][0-9]|[1-9][0-9]|[0-9]\\.){2}(25[0-5]|[1-2][0-4][0-9]|[1-9][0-9]|[0-9])/(1[0-9]|2[0-8]|[8-9]))$)|(^(172\\.16\\.(25[0-5]|[1-2][0-4][0-9]|[1-9][0-9]|[0-9])\\.(25[0-5]|[1-2][0-4][0-9]|[1-9][0-9]|[0-9])/(1[2-9]|2[0-8]))$)|(^(192\\.168\\.(25[0-5]|[1-2][0-4][0-9]|[1-9][0-9]|[0-9])\\.(25[0-5]|[1-2][0-4][0-9]|[1-9][0-9]|[0-9])/(1[6-9]|2[0-8]))$)", + "maxLength": 18 + }, + "Ipv6Net": { + "example": "fd12:3456::/64", + "title": "An IPv6 subnet", + "description": "An IPv6 subnet, including prefix and subnet mask", + "type": "string", + "pattern": "^(fd|FD)[0-9a-fA-F]{2}:((([0-9a-fA-F]{1,4}\\:){6}[0-9a-fA-F]{1,4})|(([0-9a-fA-F]{1,4}:){1,6}:))/(6[4-9]|[7-9][0-9]|1[0-1][0-9]|12[0-6])$", + "maxLength": 43 + }, "MacAddr": { "example": "ff:ff:ff:ff:ff:ff", "title": "A MAC address", @@ -873,76 +918,38 @@ "maxLength": 63 }, "NetworkInterface": { - "description": "A `NetworkInterface` represents a virtual network interface device.", + "description": "Information required to construct a virtual network interface for a guest", "type": "object", "properties": { - "description": { - "description": "human-readable free-form text about a resource", - "type": "string" - }, - "id": { - "description": "unique, immutable, system-controlled identifier for each resource", - "type": "string", - "format": "uuid" - }, - "instance_id": { - "description": "The Instance to which the interface belongs.", - "type": "string", - "format": "uuid" - }, "ip": { - "description": "The IP address assigned to this interface.", "type": "string", "format": "ip" }, "mac": { - "description": "The MAC address assigned to this interface.", - "allOf": [ - { - "$ref": "#/components/schemas/MacAddr" - } - ] + "$ref": "#/components/schemas/MacAddr" }, "name": { - "description": "unique, mutable, user-controlled identifier for each resource", - "allOf": [ - { - "$ref": "#/components/schemas/Name" - } - ] - }, - "subnet_id": { - "description": "The subnet to which the interface belongs.", - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/Name" }, - "time_created": { - "description": "timestamp when this resource was created", - "type": "string", - "format": "date-time" + "slot": { + "type": "integer", + "format": "uint8", + "minimum": 0 }, - "time_modified": { - "description": "timestamp when this resource was last modified", - "type": "string", - "format": "date-time" + "subnet": { + "$ref": "#/components/schemas/IpNet" }, - "vpc_id": { - "description": "The VPC to which the interface belongs.", - "type": "string", - "format": "uuid" + "vni": { + "$ref": "#/components/schemas/Vni" } }, "required": [ - "description", - "id", - "instance_id", "ip", "mac", "name", - "subnet_id", - "time_created", - "time_modified", - "vpc_id" + "slot", + "subnet", + "vni" ] }, "ServiceEnsureBody": { @@ -1020,6 +1027,12 @@ "zone" ] }, + "Vni": { + "description": "A Geneve Virtual Network Identifier", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, "VolumeConstructionRequest": { "oneOf": [ { diff --git a/sled-agent-client/src/lib.rs b/sled-agent-client/src/lib.rs index 73a9bda0d67..61b19959f91 100644 --- a/sled-agent-client/src/lib.rs +++ b/sled-agent-client/src/lib.rs @@ -193,37 +193,46 @@ impl From for omicron_common::api::external::DiskState { } } -impl From - for types::NetworkInterface -{ - fn from(s: omicron_common::api::external::NetworkInterface) -> Self { - Self { - description: s.identity.description.clone(), - id: s.identity.id, - name: (&s.identity.name).into(), - time_created: s.identity.time_created, - time_modified: s.identity.time_modified, - ip: s.ip.to_string(), - instance_id: s.instance_id, - mac: s.mac.into(), - subnet_id: s.subnet_id, - vpc_id: s.vpc_id, - } - } -} - impl From<&omicron_common::api::external::Name> for types::Name { fn from(s: &omicron_common::api::external::Name) -> Self { Self(<&str>::from(s).to_string()) } } +impl From for types::Vni { + fn from(v: omicron_common::api::external::Vni) -> Self { + Self(u32::from(v)) + } +} + impl From for types::MacAddr { fn from(s: omicron_common::api::external::MacAddr) -> Self { Self(s.0.to_string()) } } +impl From for types::Ipv4Net { + fn from(n: omicron_common::api::external::Ipv4Net) -> Self { + Self(n.to_string()) + } +} + +impl From for types::Ipv6Net { + fn from(n: omicron_common::api::external::Ipv6Net) -> Self { + Self(n.to_string()) + } +} + +impl From for types::IpNet { + fn from(s: omicron_common::api::external::IpNet) -> Self { + use omicron_common::api::external::IpNet; + match s { + IpNet::V4(v4) => Self::V4(v4.into()), + IpNet::V6(v6) => Self::V6(v6.into()), + } + } +} + impl From for types::UpdateArtifact { diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index ffb1cd6b9c6..b35599c0dcd 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -17,6 +17,7 @@ crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] } futures = "0.3.21" ipnetwork = "0.18" +libc = "0.2.123" macaddr = { version = "1.0.1", features = [ "serde_std" ] } nexus-client = { path = "../nexus-client" } omicron-common = { path = "../common" } @@ -46,6 +47,10 @@ uuid = { version = "0.8", features = [ "serde", "v4" ] } vsss-rs = { version = "2.0.0-pre0", default-features = false, features = ["std"] } zone = "0.1" +[target.'cfg(target_os = "illumos")'.dependencies] +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "cb1767c" } +opte = { git = "https://github.com/oxidecomputer/opte", rev = "cb1767c", features = [ "api", "std" ] } + [dev-dependencies] expectorate = "1.0.5" http = "0.2.7" diff --git a/sled-agent/src/illumos/running_zone.rs b/sled-agent/src/illumos/running_zone.rs index 7dbfbc05cd8..08227d42ec9 100644 --- a/sled-agent/src/illumos/running_zone.rs +++ b/sled-agent/src/illumos/running_zone.rs @@ -8,6 +8,7 @@ use crate::illumos::addrobj::AddrObject; use crate::illumos::svc::wait_for_service; use crate::illumos::vnic::{Vnic, VnicAllocator}; use crate::illumos::zone::{AddressRequest, ZONE_PREFIX}; +use crate::opte::OptePort; use ipnetwork::IpNetwork; use slog::Logger; use std::path::PathBuf; @@ -228,13 +229,13 @@ impl RunningZone { // TODO(https://github.com/oxidecomputer/omicron/issues/725) // // Re-initialize guest_vnic state by inspecting the zone. - guest_vnics: vec![], + opte_ports: vec![], }, }) } - pub fn get_guest_vnics(&self) -> &Vec { - &self.inner.guest_vnics + pub fn get_opte_ports(&self) -> &Vec { + &self.inner.opte_ports } } @@ -279,8 +280,8 @@ pub struct InstalledZone { // NIC used for control plane communication. control_vnic: Vnic, - // Other NICs being used by the zone. - guest_vnics: Vec, + // OPTE devices for the guest network interfaces + opte_ports: Vec, } impl InstalledZone { @@ -312,7 +313,7 @@ impl InstalledZone { unique_name: Option<&str>, datasets: &[zone::Dataset], devices: &[zone::Device], - vnics: Vec, + opte_ports: Vec, ) -> Result { let control_vnic = vnic_allocator.new_control(None).map_err(|err| { InstallZoneError::CreateVnic { @@ -325,9 +326,9 @@ impl InstalledZone { let zone_image_path = PathBuf::from(&format!("/opt/oxide/{}.tar.gz", service_name)); - let vnic_names: Vec = vnics + let net_device_names: Vec = opte_ports .iter() - .map(|vnic| vnic.name().to_string()) + .map(|port| port.vnic().name().to_string()) .chain(std::iter::once(control_vnic.name().to_string())) .collect(); @@ -337,7 +338,7 @@ impl InstalledZone { &zone_image_path, &datasets, &devices, - vnic_names, + net_device_names, ) .map_err(|err| InstallZoneError::InstallZone { zone: zone_name.to_string(), @@ -349,7 +350,7 @@ impl InstalledZone { log: log.new(o!("zone" => zone_name.clone())), name: zone_name, control_vnic, - guest_vnics: vnics, + opte_ports, }) } } diff --git a/sled-agent/src/illumos/vnic.rs b/sled-agent/src/illumos/vnic.rs index d200e4f4bc0..c3d118ad1ea 100644 --- a/sled-agent/src/illumos/vnic.rs +++ b/sled-agent/src/illumos/vnic.rs @@ -4,7 +4,6 @@ //! API for controlling a single instance. -use crate::common::vlan::VlanID; use crate::illumos::dladm::{ CreateVnicError, DeleteVnicError, PhysicalLink, VNIC_PREFIX, VNIC_PREFIX_CONTROL, @@ -50,19 +49,6 @@ impl VnicAllocator { } } - /// Creates a new NIC, intended for usage by the guest. - pub fn new_guest( - &self, - mac: Option, - vlan: Option, - ) -> Result { - let allocator = self.new_superscope("Guest"); - let name = allocator.next(); - debug_assert!(name.starts_with(VNIC_PREFIX)); - Dladm::create_vnic(&self.data_link, &name, mac, vlan)?; - Ok(Vnic { name, deleted: false }) - } - /// Creates a new NIC, intended for allowing Propolis to communicate /// with the control plane. pub fn new_control( @@ -110,8 +96,8 @@ pub struct Vnic { impl Vnic { /// Takes ownership of an existing VNIC. - pub fn wrap_existing(name: String) -> Self { - Vnic { name, deleted: false } + pub fn wrap_existing>(name: S) -> Self { + Vnic { name: name.as_ref().to_owned(), deleted: false } } /// Deletes a NIC (if it has not already been deleted). diff --git a/sled-agent/src/illumos/zone.rs b/sled-agent/src/illumos/zone.rs index 35f2edbe45a..c2ed47cf784 100644 --- a/sled-agent/src/illumos/zone.rs +++ b/sled-agent/src/illumos/zone.rs @@ -502,14 +502,15 @@ impl Zones { Ok(()) } - // Ensures a link local IPv6 exists for the object. - // - // This is necessary for allocating IPv6 addresses on illumos. - // - // For more context, see: - // + /// Ensures a link-local IPv6 exists with the name provided in `addrobj`. + /// + /// A link-local address is necessary for allocating a static address on an + /// interface on illumos. + /// + /// For more context, see: + /// #[allow(clippy::needless_lifetimes)] - fn ensure_has_link_local_v6_address<'a>( + pub fn ensure_has_link_local_v6_address<'a>( zone: Option<&'a str>, addrobj: &AddrObject, ) -> Result<(), crate::illumos::ExecutionError> { diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index bf0cce1fa57..bb38f84991d 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -4,9 +4,8 @@ //! API for controlling a single instance. -use crate::common::{ - instance::{Action as InstanceAction, InstanceStates, PROPOLIS_PORT}, - vlan::VlanID, +use crate::common::instance::{ + Action as InstanceAction, InstanceStates, PROPOLIS_PORT, }; use crate::illumos::running_zone::{InstalledZone, RunningZone}; use crate::illumos::svc::wait_for_service; @@ -14,18 +13,21 @@ use crate::illumos::vnic::VnicAllocator; use crate::illumos::zone::{AddressRequest, PROPOLIS_ZONE_PREFIX}; use crate::instance_manager::InstanceTicket; use crate::nexus::NexusClient; +use crate::opte::OptePort; +use crate::opte::OptePortAllocator; +use crate::params::NetworkInterface; use crate::params::{ InstanceHardware, InstanceMigrateParams, InstanceRuntimeStateRequested, }; use anyhow::anyhow; use futures::lock::{Mutex, MutexGuard}; -use omicron_common::api::external::NetworkInterface; use omicron_common::api::internal::nexus::InstanceRuntimeState; use omicron_common::backoff; use propolis_client::api::DiskRequest; use propolis_client::Client as PropolisClient; use slog::Logger; use std::net::IpAddr; +use std::net::Ipv6Addr; use std::net::SocketAddr; use std::sync::Arc; use tokio::task::JoinHandle; @@ -70,6 +72,12 @@ pub enum Error { #[error(transparent)] ZoneInstall(#[from] crate::illumos::running_zone::InstallZoneError), + + #[error("serde_json failure: {0}")] + SerdeJsonError(#[from] serde_json::Error), + + #[error(transparent)] + Opte(#[from] crate::opte::Error), } // Issues read-only, idempotent HTTP requests at propolis until it responds with @@ -190,8 +198,11 @@ struct InstanceInner { // NIC-related properties vnic_allocator: VnicAllocator, + + // OPTE port related properties + underlay_addr: Ipv6Addr, + port_allocator: OptePortAllocator, requested_nics: Vec, - vlan: Option, // Disk related properties requested_disks: Vec, @@ -271,14 +282,15 @@ impl InstanceInner { ) -> Result<(), Error> { let PropolisSetup { client, running_zone } = setup; - // TODO: Store slot in NetworkInterface, make this more stable. let nics = self .requested_nics .iter() - .enumerate() - .map(|(i, _)| propolis_client::api::NetworkInterfaceRequest { - name: running_zone.get_guest_vnics()[i].name().to_string(), - slot: propolis_client::api::Slot(i as u8), + .zip(running_zone.get_opte_ports().iter()) + .map(|(nic, port)| propolis_client::api::NetworkInterfaceRequest { + // TODO-correctness: Remove `.vnic()` call when we use the port + // directly. + name: port.vnic().name().to_string(), + slot: propolis_client::api::Slot(nic.slot), }) .collect(); @@ -372,8 +384,9 @@ mockall::mock! { log: Logger, id: Uuid, vnic_allocator: VnicAllocator, + underlay_addr: Ipv6Addr, + port_allocator: OptePortAllocator, initial: InstanceHardware, - vlan: Option, nexus_client: Arc, ) -> Result; pub async fn start( @@ -401,21 +414,25 @@ impl Instance { /// * `vnic_allocator`: A unique (to the sled) ID generator to /// refer to a VNIC. (This exists because of a restriction on VNIC name /// lengths, otherwise the UUID would be used instead). + /// * `underlay_addr`: The IPv6 underlay address for the sled hosting this + /// instance. + /// * `port_allocator`: A unique (to the sled) ID generator to + /// refer to an OPTE port for the guest network interfaces. /// * `initial`: State of the instance at initialization time. /// * `nexus_client`: Connection to Nexus, used for sending notifications. - /// * `vlan`: An optional VLAN ID for tagging guest VNICs. // TODO: This arg list is getting a little long; can we clean this up? pub fn new( log: Logger, id: Uuid, vnic_allocator: VnicAllocator, + underlay_addr: Ipv6Addr, + port_allocator: OptePortAllocator, initial: InstanceHardware, - vlan: Option, nexus_client: Arc, ) -> Result { info!(log, "Instance::new w/initial HW: {:?}", initial); let instance = InstanceInner { - log: log.new(o!("instance id" => id.to_string())), + log: log.new(o!("instance_id" => id.to_string())), // NOTE: Mostly lies. properties: propolis_client::api::InstanceProperties { id, @@ -432,10 +449,11 @@ impl Instance { propolis_id: initial.runtime.propolis_uuid, propolis_ip: initial.runtime.propolis_addr.unwrap().ip(), vnic_allocator, + underlay_addr, + port_allocator, requested_nics: initial.nics, requested_disks: initial.disks, cloud_init_bytes: initial.cloud_init_bytes, - vlan, state: InstanceStates::new(initial.runtime), running_state: None, nexus_client, @@ -446,28 +464,31 @@ impl Instance { Ok(Instance { inner }) } + fn create_opte_ports( + &self, + inner: &mut MutexGuard<'_, InstanceInner>, + ) -> Result, Error> { + let mut ports = Vec::with_capacity(inner.requested_nics.len()); + for nic in inner.requested_nics.iter() { + let vni = crate::opte::Vni::new(nic.vni).expect("Invalid VNI"); + let port = inner.port_allocator.new_port( + nic.ip, + *nic.mac, + ipnetwork::IpNetwork::from(nic.subnet), + vni, + inner.underlay_addr, + )?; + info!(inner.log, "created OPTE port for guest"; "port_info" => ?port); + ports.push(port); + } + Ok(ports) + } + async fn setup_propolis_locked( &self, inner: &mut MutexGuard<'_, InstanceInner>, ) -> Result { - // Instantiate all guest-requested VNICs. - // - // TODO: Ideally, we'd allocate VNICs directly within the Zone. - // However, this seems to have been a SmartOS feature which - // doesn't exist in illumos. - // - // https://github.com/illumos/ipd/blob/master/ipd/0003/README.md - let guest_nics = inner - .requested_nics - .clone() - .into_iter() - .map(|nic| { - inner - .vnic_allocator - .new_guest(Some(nic.mac), inner.vlan) - .map_err(|e| e.into()) - }) - .collect::, Error>>()?; + let opte_ports = self.create_opte_ports(inner)?; // Create a zone for the propolis instance, using the previously // configured VNICs. @@ -485,7 +506,7 @@ impl Instance { zone::Device { name: "/dev/vmmctl".to_string() }, zone::Device { name: "/dev/viona".to_string() }, ], - guest_nics, + opte_ports, ) .await?; @@ -652,6 +673,7 @@ mod test { use super::*; use crate::illumos::dladm::PhysicalLink; use crate::mocks::MockNexusClient; + use crate::opte::OptePortAllocator; use crate::params::InstanceStateRequested; use chrono::Utc; use omicron_common::api::external::{ @@ -722,14 +744,18 @@ mod test { "Test".to_string(), PhysicalLink("mylink".to_string()), ); + let port_allocator = OptePortAllocator::new(); let nexus_client = MockNexusClient::default(); let inst = Instance::new( log.clone(), test_uuid(), vnic_allocator, + std::net::Ipv6Addr::new( + 0xfd00, 0x1de, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + ), + port_allocator, new_initial_instance(), - None, Arc::new(nexus_client), ) .unwrap(); diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index 2e5970374c6..bd6e4dff4ce 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -4,16 +4,17 @@ //! API for controlling multiple instances on a sled. -use crate::common::vlan::VlanID; use crate::illumos::dladm::PhysicalLink; use crate::illumos::vnic::VnicAllocator; use crate::nexus::NexusClient; +use crate::opte::OptePortAllocator; use crate::params::{ InstanceHardware, InstanceMigrateParams, InstanceRuntimeStateRequested, }; use omicron_common::api::internal::nexus::InstanceRuntimeState; use slog::Logger; use std::collections::BTreeMap; +use std::net::Ipv6Addr; use std::sync::{Arc, Mutex}; use uuid::Uuid; @@ -38,8 +39,9 @@ struct InstanceManagerInternal { /// A mapping from a Sled Agent "Instance ID" to ("Propolis ID", [Instance]). instances: Mutex>, - vlan: Option, vnic_allocator: VnicAllocator, + underlay_addr: Ipv6Addr, + port_allocator: OptePortAllocator, } /// All instances currently running on the sled. @@ -51,17 +53,18 @@ impl InstanceManager { /// Initializes a new [`InstanceManager`] object. pub fn new( log: Logger, - vlan: Option, nexus_client: Arc, physical_link: PhysicalLink, + underlay_addr: Ipv6Addr, ) -> InstanceManager { InstanceManager { inner: Arc::new(InstanceManagerInternal { log, nexus_client, instances: Mutex::new(BTreeMap::new()), - vlan, vnic_allocator: VnicAllocator::new("Instance", physical_link), + underlay_addr, + port_allocator: OptePortAllocator::new(), }), } } @@ -108,16 +111,14 @@ impl InstanceManager { // Instance does not exist or one does but we're performing // a intra-sled migration. Either way - create an instance info!(&self.inner.log, "new instance"); - let instance_log = self - .inner - .log - .new(o!("instance" => instance_id.to_string())); + let instance_log = self.inner.log.new(o!()); let instance = Instance::new( instance_log, instance_id, self.inner.vnic_allocator.clone(), + self.inner.underlay_addr, + self.inner.port_allocator.clone(), initial_hardware, - self.inner.vlan, self.inner.nexus_client.clone(), )?; let instance_clone = instance.clone(); @@ -258,9 +259,11 @@ mod test { let im = InstanceManager::new( log, - None, nexus_client, PhysicalLink("mylink".to_string()), + std::net::Ipv6Addr::new( + 0xfd00, 0x1de, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + ), ); // Verify that no instances exist. @@ -278,7 +281,7 @@ mod test { let ticket = Arc::new(std::sync::Mutex::new(None)); let ticket_clone = ticket.clone(); let instance_new_ctx = MockInstance::new_context(); - instance_new_ctx.expect().return_once(move |_, _, _, _, _, _| { + instance_new_ctx.expect().return_once(move |_, _, _, _, _, _, _| { let mut inst = MockInstance::default(); inst.expect_clone().return_once(move || { let mut inst = MockInstance::default(); @@ -338,16 +341,18 @@ mod test { let im = InstanceManager::new( log, - None, nexus_client, PhysicalLink("mylink".to_string()), + std::net::Ipv6Addr::new( + 0xfd00, 0x1de, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + ), ); let ticket = Arc::new(std::sync::Mutex::new(None)); let ticket_clone = ticket.clone(); let instance_new_ctx = MockInstance::new_context(); let mut seq = mockall::Sequence::new(); - instance_new_ctx.expect().return_once(move |_, _, _, _, _, _| { + instance_new_ctx.expect().return_once(move |_, _, _, _, _, _, _| { let mut inst = MockInstance::default(); // First call to ensure (start + transition). inst.expect_clone().times(1).in_sequence(&mut seq).return_once( diff --git a/sled-agent/src/lib.rs b/sled-agent/src/lib.rs index 80951b04639..d63698402b9 100644 --- a/sled-agent/src/lib.rs +++ b/sled-agent/src/lib.rs @@ -26,6 +26,7 @@ pub mod illumos; mod instance; mod instance_manager; mod nexus; +mod opte; mod params; pub mod rack_setup; pub mod server; diff --git a/sled-agent/src/opte/mock_opte.rs b/sled-agent/src/opte/mock_opte.rs new file mode 100644 index 00000000000..e048b449817 --- /dev/null +++ b/sled-agent/src/opte/mock_opte.rs @@ -0,0 +1,185 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Mock / empty interface to the Oxide Packet Transformation Engine (OPTE), for +//! building the sled agent on non-illumos systems. + +use crate::illumos::vnic::Vnic; +use ipnetwork::IpNetwork; +use macaddr::MacAddr6; +use slog::Logger; +use std::net::IpAddr; +use std::net::Ipv6Addr; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; +use std::sync::Arc; + +#[derive(Debug, Clone, Copy)] +pub struct Vni(u32); + +impl Vni { + pub fn new(n: N) -> Result + where + N: Into, + { + let x = n.into(); + if x <= 0x00_FF_FF_FF { + Ok(Self(x)) + } else { + Err(Error::InvalidArgument(format!("invalid VNI: {}", x))) + } + } +} + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Invalid argument: {0}")] + InvalidArgument(String), +} + +#[derive(Debug, Clone)] +pub struct OptePortAllocator { + value: Arc, +} + +impl OptePortAllocator { + pub fn new() -> Self { + Self { value: Arc::new(AtomicU64::new(0)) } + } + + fn next(&self) -> String { + format!("opte{}", self.next_id()) + } + + fn next_id(&self) -> u64 { + self.value.fetch_add(1, Ordering::SeqCst) + } + + pub fn new_port( + &self, + ip: IpAddr, + mac: MacAddr6, + subnet: IpNetwork, + vni: Vni, + underlay_ip: Ipv6Addr, + ) -> Result { + // TODO-completess: Remove IPv4 restrictions once OPTE supports virtual + // IPv6 networks. + if matches!(ip, IpAddr::V6(_)) { + return Err(Error::InvalidArgument(String::from( + "IPv6 not yet supported", + ))); + } + let gateway = Gateway::from_subnet(&subnet); + if matches!(gateway.ip, IpAddr::V6(_)) { + return Err(Error::InvalidArgument(String::from( + "IPv6 not yet supported", + ))); + } + let boundary_services = BoundaryServices::default(); + let name = self.next(); + if matches!(subnet.network(), IpAddr::V6(_)) { + return Err(Error::InvalidArgument(String::from( + "IPv6 not yet supported", + ))); + } + Ok(OptePort { + name, + ip, + subnet, + mac, + vni, + underlay_ip, + gateway, + boundary_services, + vnic: None, + }) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct BoundaryServices { + pub ip: Ipv6Addr, + pub vni: Vni, +} + +impl Default for BoundaryServices { + fn default() -> Self { + // TODO-completeness: Don't hardcode this. + // + // Boundary Services will be started on several Sidecars during rack + // setup, and those addresses will need to be propagated here. + const BOUNDARY_SERVICES_ADDR: Ipv6Addr = + Ipv6Addr::new(0xfd00, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01); + let boundary_services_vni = Vni::new(99_u32).unwrap(); + + Self { ip: BOUNDARY_SERVICES_ADDR, vni: boundary_services_vni } + } +} + +/// Information about the gateway for an OPTE port +#[derive(Debug, Clone, Copy)] +#[allow(dead_code)] +pub struct Gateway { + mac: MacAddr6, + ip: IpAddr, +} + +// The MAC address that OPTE exposes to guest NICs, i.e., the MAC of the virtual +// gateway OPTE operates as for each guest. See +// https://github.com/oxidecomputer/omicron/pull/955#discussion_r856432498 for +// more context on the genesis of this, but this is just a reserved address +// within the "system" portion of the virtual MAC address space. +const OPTE_VIRTUAL_GATEWAY_MAC: MacAddr6 = + MacAddr6::new(0xa8, 0x40, 0x25, 0xff, 0x77, 0x77); + +impl Gateway { + pub fn from_subnet(subnet: &IpNetwork) -> Self { + Self { + mac: OPTE_VIRTUAL_GATEWAY_MAC, + + // See RFD 21, section 2.2 table 1 + ip: subnet + .iter() + .nth(1) + .expect("IP subnet must have at least 1 address"), + } + } +} + +/// A port on the OPTE "virtual switch", which corresponds to one guest network +/// interface. +#[derive(Debug)] +#[allow(dead_code)] +pub struct OptePort { + name: String, + ip: IpAddr, + subnet: IpNetwork, + mac: MacAddr6, + vni: Vni, + underlay_ip: Ipv6Addr, + gateway: Gateway, + boundary_services: BoundaryServices, + vnic: Option, +} + +impl OptePort { + /// Return the VNIC used to link OPTE and Viona. + // TODO-correctness: Remove this once we can put Viona directly on top of an + // OPTE port device. + pub fn vnic(&self) -> &Vnic { + self.vnic.as_ref().unwrap() + } +} + +impl Drop for OptePort { + fn drop(&mut self) { + self.vnic.take(); + } +} + +pub fn initialize_xde_driver(log: &Logger) -> Result<(), Error> { + slog::warn!(log, "`xde` driver is a fiction on non-illumos systems"); + Ok(()) +} diff --git a/sled-agent/src/opte/mod.rs b/sled-agent/src/opte/mod.rs new file mode 100644 index 00000000000..007ec6f22d9 --- /dev/null +++ b/sled-agent/src/opte/mod.rs @@ -0,0 +1,11 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Interactions with the Oxide Packet Transformation Engine (OPTE) + +#[cfg_attr(target_os = "illumos", path = "opte.rs")] +#[cfg_attr(not(target_os = "illumos"), path = "mock_opte.rs")] +mod inner; + +pub use inner::*; diff --git a/sled-agent/src/opte/opte.rs b/sled-agent/src/opte/opte.rs new file mode 100644 index 00000000000..6f62dd021ef --- /dev/null +++ b/sled-agent/src/opte/opte.rs @@ -0,0 +1,298 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Interactions with the Oxide Packet Transformation Engine (OPTE) + +use crate::illumos::addrobj; +use crate::illumos::addrobj::AddrObject; +use crate::illumos::dladm; +use crate::illumos::dladm::Dladm; +use crate::illumos::dladm::PhysicalLink; +use crate::illumos::vnic::Vnic; +use crate::illumos::zone::Zones; +use ipnetwork::IpNetwork; +use macaddr::MacAddr6; +use opte::api::IpCidr; +use opte::api::Ipv4Cidr; +use opte::api::Ipv4PrefixLen; +use opte::api::MacAddr; +pub use opte::api::Vni; +use opte::oxide_vpc::api::AddRouterEntryIpv4Req; +use opte::oxide_vpc::api::RouterTarget; +use opte_ioctl::OpteHdl; +use slog::Logger; +use std::net::IpAddr; +use std::net::Ipv6Addr; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; +use std::sync::Arc; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Failure interacting with the OPTE ioctl(2) interface: {0}")] + Opte(#[from] opte_ioctl::Error), + + #[error("Failed to wrap OPTE port in a VNIC: {0}")] + CreateVnic(#[from] dladm::CreateVnicError), + + #[error("Failed to create an IPv6 link-local address for xde underlay devices: {0}")] + UnderlayDevice(#[from] crate::illumos::ExecutionError), + + #[error(transparent)] + BadAddrObj(#[from] addrobj::ParseError), +} + +#[derive(Debug, Clone)] +pub struct OptePortAllocator { + value: Arc, +} + +impl OptePortAllocator { + pub fn new() -> Self { + Self { value: Arc::new(AtomicU64::new(0)) } + } + + fn next(&self) -> String { + format!("opte{}", self.next_id()) + } + + fn next_id(&self) -> u64 { + self.value.fetch_add(1, Ordering::SeqCst) + } + + pub fn new_port( + &self, + ip: IpAddr, + mac: MacAddr6, + subnet: IpNetwork, + vni: Vni, + underlay_ip: Ipv6Addr, + ) -> Result { + // TODO-completess: Remove IPv4 restrictions once OPTE supports virtual + // IPv6 networks. + let private_ip = match ip { + IpAddr::V4(ip) => Ok(ip), + IpAddr::V6(_) => Err(opte_ioctl::Error::InvalidArgument( + String::from("IPv6 is not yet supported for guest interfaces"), + )), + }?; + let gateway = Gateway::from_subnet(&subnet); + let gateway_ip = match gateway.ip { + IpAddr::V4(ip) => Ok(ip), + IpAddr::V6(_) => Err(opte_ioctl::Error::InvalidArgument( + String::from("IPv6 is not yet supported for guest interfaces"), + )), + }?; + let boundary_services = BoundaryServices::default(); + let name = self.next(); + let hdl = OpteHdl::open(OpteHdl::DLD_CTL)?; + hdl.create_xde( + &name, + MacAddr::from(mac.into_array()), + private_ip, + MacAddr::from(gateway.mac.into_array()), + gateway_ip, + boundary_services.ip, + boundary_services.vni, + vni, + underlay_ip, + /* passthru = */ false, + )?; + + // Add a router entry for this interface's subnet, directing traffic to the + // VPC subnet. + match subnet.network() { + IpAddr::V4(ip) => { + let prefix = + Ipv4PrefixLen::new(subnet.prefix()).map_err(|e| { + opte_ioctl::Error::InvalidArgument(format!( + "Invalid IPv4 subnet prefix: {}", + e + )) + })?; + let cidr = Ipv4Cidr::new(opte::api::Ipv4Addr::from(ip), prefix); + let route = AddRouterEntryIpv4Req { + port_name: name.clone(), + dest: cidr, + target: RouterTarget::VpcSubnet(IpCidr::Ip4(cidr)), + }; + hdl.add_router_entry_ip4(&route)?; + } + IpAddr::V6(_) => { + return Err(opte_ioctl::Error::InvalidArgument(String::from( + "IPv6 not yet supported", + )) + .into()); + } + } + + // Create a VNIC on top of this device, to hook Viona into. + // + // Viona is the illumos MAC provider that implements the VIRTIO + // specification.It sits on top of a MAC provider, which is responsible + // for delivering frames to the underlying data link. The guest includes + // a driver that handles the virtio-net specification on their side, + // which talks to Viona. + // + // In theory, Viona work with any MAC provider. However, there are + // implicit assumptions, in both Viona _and_ MAC, that require Viona to + // be built on top of a VNIC specifically. There is probably a good deal + // of work required to relax that assumption, so in the meantime, we + // create a superfluous VNIC on the OPTE device, solely so Viona can use + // it. + let vnic = { + let phys = PhysicalLink(name.clone()); + let vnic_name = format!("v{}", name); + Dladm::create_vnic( + &phys, + &vnic_name, + Some(omicron_common::api::external::MacAddr(mac)), + None, + )?; + Some(Vnic::wrap_existing(vnic_name)) + }; + + Ok(OptePort { + name, + ip, + subnet, + mac, + vni, + underlay_ip, + gateway, + boundary_services, + vnic, + }) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct BoundaryServices { + pub ip: Ipv6Addr, + pub vni: Vni, +} + +impl Default for BoundaryServices { + fn default() -> Self { + // TODO-completeness: Don't hardcode this. + // + // Boundary Services will be started on several Sidecars during rack + // setup, and those addresses will need to be propagated here. + const BOUNDARY_SERVICES_ADDR: Ipv6Addr = + Ipv6Addr::new(0xfd00, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01); + let boundary_services_vni = Vni::new(99_u32).unwrap(); + + Self { ip: BOUNDARY_SERVICES_ADDR, vni: boundary_services_vni } + } +} + +/// Information about the gateway for an OPTE port +#[derive(Debug, Clone, Copy)] +pub struct Gateway { + mac: MacAddr6, + ip: IpAddr, +} + +// The MAC address that OPTE exposes to guest NICs, i.e., the MAC of the virtual +// gateway OPTE operates as for each guest. See +// https://github.com/oxidecomputer/omicron/pull/955#discussion_r856432498 for +// more context on the genesis of this, but this is just a reserved address +// within the "system" portion of the virtual MAC address space. +const OPTE_VIRTUAL_GATEWAY_MAC: MacAddr6 = + MacAddr6::new(0xa8, 0x40, 0x25, 0xff, 0x77, 0x77); + +impl Gateway { + pub fn from_subnet(subnet: &IpNetwork) -> Self { + Self { + mac: OPTE_VIRTUAL_GATEWAY_MAC, + + // See RFD 21, section 2.2 table 1 + ip: subnet + .iter() + .nth(1) + .expect("IP subnet must have at least 1 address"), + } + } +} + +/// A port on the OPTE "virtual switch", which corresponds to one guest network +/// interface. +#[derive(Debug)] +#[allow(dead_code)] +pub struct OptePort { + name: String, + ip: IpAddr, + subnet: IpNetwork, + mac: MacAddr6, + vni: Vni, + underlay_ip: Ipv6Addr, + gateway: Gateway, + boundary_services: BoundaryServices, + // TODO-correctness: Remove this once we can put Viona directly on top of an + // OPTE port device. + // + // Note that this will always be `Some(_)`. It is wrapped in an optional to + // ensure we can drop the VNIC before we drop the OPTE port itself. + vnic: Option, +} + +impl OptePort { + /// Return the VNIC used to link OPTE and Viona. + // TODO-correctness: Remove this once we can put Viona directly on top of an + // OPTE port device. + pub fn vnic(&self) -> &Vnic { + self.vnic.as_ref().unwrap() + } +} + +impl Drop for OptePort { + fn drop(&mut self) { + self.vnic.take(); + if let Ok(hdl) = OpteHdl::open(OpteHdl::DLD_CTL) { + if hdl.delete_xde(&self.name).is_ok() { + return; + } + } + eprintln!("WARNING: Failed to delete OPTE port '{}'", self.name); + } +} + +/// Initialize the underlay devices required for the xde kernel module. +/// +/// The xde driver needs information about the physical devices out which it can +/// send traffic from the guests. +pub fn initialize_xde_driver(log: &Logger) -> Result<(), Error> { + let underlay_nics = find_chelsio_links()?; + info!(log, "using '{:?}' as data links for xde driver", underlay_nics); + if underlay_nics.len() < 2 { + return Err(Error::Opte(opte_ioctl::Error::InvalidArgument( + String::from("There must be at least two underlay NICs"), + ))); + } + for nic in &underlay_nics { + let addrobj = AddrObject::new(&nic.0, "linklocal")?; + Zones::ensure_has_link_local_v6_address(None, &addrobj)?; + } + match OpteHdl::open(OpteHdl::DLD_CTL)? + .set_xde_underlay(&underlay_nics[0].0, &underlay_nics[1].0) + { + Ok(_) => Ok(()), + // TODO-correctness: xde provides no way to get the current underlay + // devices we're using, but we'd probably like the further check that + // those are exactly what we're giving it now. + Err(opte_ioctl::Error::CommandError( + _, + opte::api::OpteError::System { errno: libc::EEXIST, .. }, + )) => Ok(()), + Err(e) => Err(e.into()), + } +} + +fn find_chelsio_links() -> Result, Error> { + // TODO-correctness: This should eventually be determined by a call to + // `Dladm` to get the real Chelsio links on a Gimlet. These will likely be + // called `cxgbeN`, but we explicitly call them `netN` to be clear that + // they're likely VNICs for the time being. + Ok((0..2).map(|i| PhysicalLink(format!("net{}", i))).collect()) +} diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 21c0182a22e..87fde05d7c6 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -2,16 +2,29 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use omicron_common::api::external::NetworkInterface; +use omicron_common::api::external; use omicron_common::api::internal::nexus::{ DiskRuntimeState, InstanceRuntimeState, }; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::fmt::{Debug, Display, Formatter, Result as FormatResult}; -use std::net::{Ipv6Addr, SocketAddr}; +use std::net::IpAddr; +use std::net::Ipv6Addr; +use std::net::SocketAddr; use uuid::Uuid; +/// Information required to construct a virtual network interface for a guest +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct NetworkInterface { + pub name: external::Name, + pub ip: IpAddr, + pub mac: external::MacAddr, + pub subnet: external::IpNet, + pub vni: external::Vni, + pub slot: u8, +} + /// Used to request a Disk state change #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] #[serde(rename_all = "lowercase", tag = "state", content = "instance")] diff --git a/sled-agent/src/server.rs b/sled-agent/src/server.rs index 00c141a0358..ae51a90c83e 100644 --- a/sled-agent/src/server.rs +++ b/sled-agent/src/server.rs @@ -61,7 +61,7 @@ impl Server { let sa_log = log.new(o!( "component" => "SledAgent", - "server" => config.id.clone().to_string() + "sled_id" => config.id.clone().to_string() )); let sled_agent = SledAgent::new(&config, sa_log, nexus_client.clone(), addr) diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 46090319783..e4017304d1c 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -63,6 +63,9 @@ pub enum Error { #[error("Error updating: {0}")] Download(#[from] crate::updates::Error), + + #[error("Error managing guest networking: {0}")] + Opte(#[from] crate::opte::Error), } impl From for omicron_common::api::external::Error { @@ -101,7 +104,6 @@ impl SledAgent { sled_address: SocketAddrV6, ) -> Result { let id = &config.id; - let vlan = config.vlan; info!(&log, "created sled agent"; "id" => ?id); let data_link = if let Some(link) = config.data_link.clone() { @@ -134,6 +136,9 @@ impl SledAgent { ) .map_err(|err| Error::SledSubnet { err })?; + // Initialize the xde kernel driver with the underlay devices. + crate::opte::initialize_xde_driver(&log)?; + // Identify all existing zones which should be managed by the Sled // Agent. // @@ -156,8 +161,15 @@ impl SledAgent { // // This should be accessible via: // $ dladm show-linkprop -c -p zone -o LINK,VALUE + // + // Note that this currently deletes only VNICs that start with the + // prefix the sled-agent uses. We'll need to generate an alert or + // otherwise handle VNICs that we _don't_ expect. let vnics = Dladm::get_vnics()?; - for vnic in vnics { + for vnic in vnics + .iter() + .filter(|vnic| vnic.starts_with(crate::illumos::dladm::VNIC_PREFIX)) + { warn!(log, "Deleting VNIC: {}", vnic); Dladm::delete_vnic(&vnic)?; } @@ -181,9 +193,9 @@ impl SledAgent { } let instances = InstanceManager::new( log.clone(), - vlan, nexus_client.clone(), data_link.clone(), + *sled_address.ip(), ); let services = ServiceManager::new(log.clone(), data_link.clone(), None).await?; diff --git a/smf/sled-agent/config.toml b/smf/sled-agent/config.toml index 84d44293e46..45fc6669dfa 100644 --- a/smf/sled-agent/config.toml +++ b/smf/sled-agent/config.toml @@ -8,8 +8,11 @@ id = "fb0f7546-4d46-40ca-9d56-cbb810684ca7" nexus_address = "[fd00:1122:3344:0101::3]:12221" # A file-backed zpool can be manually created with the following: -# $ truncate -s 10GB testpool.vdev -# $ zpool create oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b testpool.vdev +# # truncate -s 10GB testpool.vdev +# # zpool create oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b "$PWD/testpool.vdev" +# +# Note that you'll need to create one such zpool for each below, with a +# different vdev for each. zpools = [ "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b", "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03", diff --git a/tools/install_opte.sh b/tools/install_opte.sh index 85f34674951..b2c49694edb 100755 --- a/tools/install_opte.sh +++ b/tools/install_opte.sh @@ -59,16 +59,20 @@ function sha_from_url { curl -L "$SHA_URL" 2> /dev/null | cut -d ' ' -f 1 } -# The `helios-netdev` provides the XDE kernel driver and the `opteadm` userland -# tool for interacting with it. -HELIOS_NETDEV_REPO_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G11AT7E4XV9J1J54GE2YDJT6/CB4WF4BVgnbvf5NI573z9osAV2LNIKogPtWJ5sfW2cNxUYQO/01G11ATFVTWAC2HSNV148PQ4ER/01G11B5MPQRBX3Q5EF45YDAW6Q/opte-0.1.60.p5p" -HELIOS_NETDEV_REPO_SHA_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G11AT7E4XV9J1J54GE2YDJT6/CB4WF4BVgnbvf5NI573z9osAV2LNIKogPtWJ5sfW2cNxUYQO/01G11ATFVTWAC2HSNV148PQ4ER/01G11B5MR60H4N13NJKGWEEA69/opte-0.1.60.p5p.sha256" +# `helios-netdev` provides the xde kernel driver and the `opteadm` userland tool +# for interacting with it. +HELIOS_NETDEV_BASE_URL="https://buildomat.eng.oxide.computer/public/file/oxidecomputer/opte/repo" +HELIOS_NETDEV_COMMIT="cb1767c80d4e9d97cb79901eed3c9d08e1fb3826" +HELIOS_NETDEV_REPO_URL="$HELIOS_NETDEV_BASE_URL/$HELIOS_NETDEV_COMMIT/opte.p5p" +HELIOS_NETDEV_REPO_SHA_URL="$HELIOS_NETDEV_BASE_URL/$HELIOS_NETDEV_COMMIT/opte.p5p.sha256" HELIOS_NETDEV_REPO_PATH="$XDE_DIR/$(basename "$HELIOS_NETDEV_REPO_URL")" -# The XDE repo provides a full OS/Net incorporation, with updated kernel bits +# The xde repo provides a full OS/Net incorporation, with updated kernel bits # that the `xde` kernel module and OPTE rely on. -XDE_REPO_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0ZKH44GQF88GB0GQBG9TQGW/7eOYj8L8E4MLrtvdTgGMyMu5qjYTRheV250bEvh2OkBrggX4/01G0ZKHBQ33K40S5ABZMRNWS5P/01G0ZYDDRXQ3Y4E5SG9QX8N9FK/repo.p5p" -XDE_REPO_SHA_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0ZKH44GQF88GB0GQBG9TQGW/7eOYj8L8E4MLrtvdTgGMyMu5qjYTRheV250bEvh2OkBrggX4/01G0ZKHBQ33K40S5ABZMRNWS5P/01G0ZYDJDMJAYHFV9Z6XVE30X5/repo.p5p.sha256" +XDE_REPO_BASE_URL="https://buildomat.eng.oxide.computer/public/file/oxidecomputer/os-build/xde" +XDE_REPO_COMMIT="485065f3b3292e2198db0629341492672b1e29f7" +XDE_REPO_URL="$XDE_REPO_BASE_URL/$XDE_REPO_COMMIT/repo.p5p" +XDE_REPO_SHA_URL="$XDE_REPO_BASE_URL/$XDE_REPO_COMMIT/repo.p5p.sha256" XDE_REPO_PATH="$XDE_DIR/$(basename "$XDE_REPO_URL")" # Download and verify the package repositorieies @@ -80,7 +84,7 @@ download_and_check_sha "$XDE_REPO_URL" "$(sha_from_url "$XDE_REPO_SHA_URL")" # provides newer versions of the packages. pkg set-publisher --non-sticky helios-dev -# Add the OPTE and XDE repositories and update packages. +# Add the OPTE and xde repositories and update packages. pkg set-publisher -p "$HELIOS_NETDEV_REPO_PATH" --search-first pkg set-publisher -p "$XDE_REPO_PATH" --search-first