Initial integration with Oxide Packet Transformation Engine

common/src/api/external/mod.rs

@@ -1170,6 +1170,15 @@ impl FromStr for IpNet {
     }
 }
 
+impl From<IpNet> for ipnetwork::IpNetwork {
+    fn from(net: IpNet) -> ipnetwork::IpNetwork {
+        match net {
+            IpNet::V4(net) => ipnetwork::IpNetwork::from(net.0),
+            IpNet::V6(net) => ipnetwork::IpNetwork::from(net.0),
+        }
+    }
+}
+
 /// A `RouteTarget` describes the possible locations that traffic matching a
 /// route destination can be sent.
 #[derive(
@@ -1668,6 +1677,62 @@ impl JsonSchema for MacAddr {
     }
 }
 
+/// A Geneve Virtual Network Identifier
+#[derive(
+    Debug,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    Hash,
+    PartialOrd,
+    Ord,
+    Deserialize,
+    Serialize,
+    JsonSchema,
+)]
+pub struct Vni(u32);
+
+impl Vni {
+    const MAX_VNI: u32 = 1 << 24;
+
+    /// Create a new random VNI.
+    pub fn random() -> Self {
+        use rand::Rng;
+        Self(rand::thread_rng().gen_range(0..=Self::MAX_VNI))
+    }
+}
+
+impl From<Vni> for u32 {
+    fn from(vni: Vni) -> u32 {
+        vni.0
+    }
+}
+
+impl TryFrom<u32> for Vni {
+    type Error = Error;
+
+    fn try_from(x: u32) -> Result<Self, Error> {
+        if x <= Self::MAX_VNI {
+            Ok(Self(x))
+        } else {
+            Err(Error::internal_error(
+                format!("Invalid Geneve VNI: {}", x).as_str(),
+            ))
+        }
+    }
+}
+
+impl TryFrom<i32> for Vni {
+    type Error = Error;
+
+    fn try_from(x: i32) -> Result<Self, Error> {
+        Self::try_from(u32::try_from(x).map_err(|_| {
+            Error::internal_error(format!("Invalid Geneve VNI: {}", x).as_str())
+        })?)
+    }
+}
+
 /// A `NetworkInterface` represents a virtual network interface device.
 #[derive(ObjectIdentity, Clone, Debug, Deserialize, JsonSchema, Serialize)]
 pub struct NetworkInterface {

common/src/sql/dbinit.sql

@@ -592,6 +592,13 @@ CREATE TABLE omicron.public.vpc (
     system_router_id UUID NOT NULL,
     dns_name STRING(63) NOT NULL,
 
+    /*
+     * The Geneve Virtual Network Identifier for this VPC. Note that this is a
+     * 24-bit unsigned value, properties which are checked in the application,
+     * not the database.
+     */
+    vni INT4 NOT NULL,
+
     /* The IPv6 prefix allocated to subnets. */
     ipv6_prefix INET NOT NULL,
 
@@ -606,6 +613,11 @@ CREATE UNIQUE INDEX ON omicron.public.vpc (
 ) WHERE
     time_deleted IS NULL;
 
+CREATE UNIQUE INDEX ON omicron.public.vpc (
+    vni
+) WHERE
+    time_deleted IS NULL;
+
 CREATE TABLE omicron.public.vpc_subnet (
     /* Identity metadata (resource) */
     id UUID PRIMARY KEY,

nexus/src/db/datastore.rs

@@ -65,6 +65,7 @@ use diesel::query_dsl::methods::LoadQuery;
 use diesel::upsert::excluded;
 use diesel::{ExpressionMethods, QueryDsl, SelectableHelper};
 use omicron_common::api;
+use omicron_common::api::external;
 use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::DeleteResult;
 use omicron_common::api::external::Error;
@@ -77,6 +78,7 @@ use omicron_common::api::external::{
     CreateResult, IdentityMetadataCreateParams,
 };
 use omicron_common::bail_unless;
+use sled_agent_client::types as sled_client_types;
 use std::convert::{TryFrom, TryInto};
 use std::net::Ipv6Addr;
 use std::sync::Arc;
@@ -1365,6 +1367,85 @@ impl DataStore {
         Ok(())
     }
 
+    /// Return the information about an instance's network interfaces required
+    /// for the sled agent to instantiate them via OPTE.
+    ///
+    /// OPTE requires information that's currently split across the network
+    /// interface and VPC subnet tables. This query just joins those for each
+    /// NIC in the given instance.
+    pub(crate) async fn derive_guest_network_interface_info(
+        &self,
+        opctx: &OpContext,
+        authz_instance: &authz::Instance,
+    ) -> ListResultVec<sled_client_types::NetworkInterface> {
+        opctx.authorize(authz::Action::ListChildren, authz_instance).await?;
+
+        use db::schema::network_interface;
+        use db::schema::vpc;
+        use db::schema::vpc_subnet;
+
+        // The record type for the results of the below JOIN query
+        #[derive(Debug, diesel::Queryable)]
+        struct NicInfo {
+            name: db::model::Name,
+            ip: ipnetwork::IpNetwork,
+            mac: db::model::MacAddr,
+            ipv4_block: db::model::Ipv4Net,
+            ipv6_block: db::model::Ipv6Net,
+            vni: db::model::Vni,
+            slot: i16,
+        }
+
+        impl From<NicInfo> for sled_client_types::NetworkInterface {
+            fn from(nic: NicInfo) -> sled_client_types::NetworkInterface {
+                let ip_subnet = if nic.ip.is_ipv4() {
+                    external::IpNet::V4(nic.ipv4_block.0)
+                } else {
+                    external::IpNet::V6(nic.ipv6_block.0)
+                };
+                sled_client_types::NetworkInterface {
+                    name: sled_client_types::Name::from(&nic.name.0),
+                    ip: nic.ip.ip().to_string(),
+                    mac: sled_client_types::MacAddr::from(nic.mac.0),
+                    subnet: sled_client_types::IpNet::from(ip_subnet),
+                    vni: sled_client_types::Vni::from(nic.vni.0),
+                    slot: u8::try_from(nic.slot).unwrap(),
+                }
+            }
+        }
+
+        let rows = network_interface::table
+            .filter(network_interface::instance_id.eq(authz_instance.id()))
+            .filter(network_interface::time_deleted.is_null())
+            .inner_join(
+                vpc_subnet::table
+                    .on(network_interface::subnet_id.eq(vpc_subnet::id)),
+            )
+            .inner_join(vpc::table.on(vpc_subnet::vpc_id.eq(vpc::id)))
+            .order_by(network_interface::slot)
+            // TODO-cleanup: Having to specify each column again is less than
+            // ideal, but we can't derive `Selectable` since this is the result
+            // of a JOIN and not from a single table. DRY this out if possible.
+            .select((
+                network_interface::name,
+                network_interface::ip,
+                network_interface::mac,
+                vpc_subnet::ipv4_block,
+                vpc_subnet::ipv6_block,
+                vpc::vni,
+                network_interface::slot,
+            ))
+            .get_results_async::<NicInfo>(self.pool_authorized(opctx).await?)
+            .await
+            .map_err(|e| {
+                public_error_from_diesel_pool(e, ErrorHandler::Server)
+            })?;
+        Ok(rows
+            .into_iter()
+            .map(sled_client_types::NetworkInterface::from)
+            .collect())
+    }
+
     /// List network interfaces associated with a given instance.
     pub async fn instance_list_network_interfaces(
         &self,

nexus/src/db/model/mod.rs

@@ -40,6 +40,7 @@ mod ssh_key;
 mod u16;
 mod update_artifact;
 mod user_builtin;
+mod vni;
 mod volume;
 mod vpc;
 mod vpc_firewall_rule;
@@ -84,6 +85,7 @@ pub use snapshot::*;
 pub use ssh_key::*;
 pub use update_artifact::*;
 pub use user_builtin::*;
+pub use vni::*;
 pub use volume::*;
 pub use vpc::*;
 pub use vpc_firewall_rule::*;

nexus/src/db/model/vni.rs

@@ -0,0 +1,42 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use diesel::backend::Backend;
+use diesel::backend::RawValue;
+use diesel::deserialize;
+use diesel::deserialize::FromSql;
+use diesel::query_builder::bind_collector::RawBytesBindCollector;
+use diesel::serialize;
+use diesel::serialize::ToSql;
+use diesel::sql_types;
+use omicron_common::api::external;
+
+#[derive(Clone, Debug, Copy, AsExpression, FromSqlRow)]
+#[diesel(sql_type = sql_types::Int4)]
+pub struct Vni(pub external::Vni);
+
+impl<DB> ToSql<sql_types::Int4, DB> for Vni
+where
+    DB: Backend<BindCollector = RawBytesBindCollector<DB>>,
+    i32: ToSql<sql_types::Int4, DB>,
+{
+    fn to_sql<'b>(
+        &'b self,
+        out: &mut serialize::Output<'b, '_, DB>,
+    ) -> serialize::Result {
+        // Reborrowing is necessary to ensure that the lifetime of the temporary
+        // i32 created here and `out` is the same, i.e., that `'b = '_`.
+        i32::try_from(u32::from(self.0)).unwrap().to_sql(&mut out.reborrow())
+    }
+}
+
+impl<DB> FromSql<sql_types::Int4, DB> for Vni
+where
+    DB: Backend,
+    i32: FromSql<sql_types::Int4, DB>,
+{
+    fn from_sql(bytes: RawValue<DB>) -> deserialize::Result<Self> {
+        Ok(Vni(external::Vni::try_from(i32::from_sql(bytes)?)?))
+    }
+}

nexus/src/db/model/vpc.rs

@@ -4,6 +4,7 @@
 
 use super::{Generation, Ipv6Net, Name, VpcFirewallRule};
 use crate::db::collection_insert::DatastoreCollection;
+use crate::db::model::Vni;
 use crate::db::schema::{vpc, vpc_firewall_rule};
 use crate::defaults;
 use crate::external_api::params;
@@ -20,6 +21,7 @@ pub struct Vpc {
 
     pub project_id: Uuid,
     pub system_router_id: Uuid,
+    pub vni: Vni,
     pub ipv6_prefix: Ipv6Net,
     pub dns_name: Name,
 
@@ -54,6 +56,7 @@ impl Vpc {
             identity,
             project_id,
             system_router_id,
+            vni: Vni(external::Vni::random()),
             ipv6_prefix,
             dns_name: params.dns_name.into(),
             firewall_gen: Generation::new(),

nexus/src/db/schema.rs

@@ -333,6 +333,7 @@ table! {
         time_deleted -> Nullable<Timestamptz>,
         project_id -> Uuid,
         system_router_id -> Uuid,
+        vni -> Int4,
         ipv6_prefix -> Inet,
         dns_name -> Text,
         firewall_gen -> Int8,

nexus/src/nexus.rs

@@ -1847,22 +1847,10 @@ impl Nexus {
             });
         }
 
-        let nics: Vec<external::NetworkInterface> = self
+        let nics = self
             .db_datastore
-            .instance_list_network_interfaces(
-                &opctx,
-                &authz_instance,
-                &DataPageParams {
-                    marker: None,
-                    direction: dropshot::PaginationOrder::Ascending,
-                    limit: std::num::NonZeroU32::new(MAX_NICS_PER_INSTANCE)
-                        .unwrap(),
-                },
-            )
-            .await?
-            .iter()
-            .map(|x| x.clone().into())
-            .collect();
+            .derive_guest_network_interface_info(&opctx, &authz_instance)
+            .await?;
 
         // Ask the sled agent to begin the state change.  Then update the
         // database to reflect the new intermediate state.  If this update is
@@ -1873,7 +1861,7 @@ impl Nexus {
             runtime: sled_agent_client::types::InstanceRuntimeState::from(
                 db_instance.runtime().clone(),
             ),
-            nics: nics.iter().map(|nic| nic.clone().into()).collect(),
+            nics,
             disks: disk_reqs,
             cloud_init_bytes: Some(base64::encode(
                 db_instance.generate_cidata()?,