fix(kubernetes): keep a late stop snapshot from reviving a stopped sandbox

crates/openshell-driver-kubernetes/src/driver.rs

@@ -6923,17 +6923,13 @@ async fn sandbox_from_object_with_sandbox_runtime_readiness(
     namespace: &str,
     obj: DynamicObject,
 ) -> Result<(String, Sandbox), String> {
-    let bootstrapping = sandbox_runtime_bootstrap_in_progress(&obj);
     let sandbox_id = sandbox_id_from_object(&obj).unwrap_or_default();
     let object_namespace = obj
         .metadata
         .namespace
         .clone()
         .unwrap_or_else(|| namespace.to_string());
-    let (name, mut sandbox) = sandbox_from_object(namespace, obj.clone())?;
-    if bootstrapping {
-        mark_sandbox_runtime_bootstrapping(&mut sandbox);
-    }
+    let (name, mut sandbox) = sandbox_from_object_with_bootstrap_state(namespace, &obj)?;
     if !sandbox_id.is_empty() {
         let dependencies = Box::pin(generation_control_availability(
             client,
@@ -6972,11 +6968,33 @@ async fn sandbox_from_object_with_sandbox_runtime_readiness(
     Ok((name, sandbox))
 }
 
+fn sandbox_from_object_with_bootstrap_state(
+    namespace: &str,
+    obj: &DynamicObject,
+) -> Result<(String, Sandbox), String> {
+    let (name, mut sandbox) = sandbox_from_object(namespace, obj.clone())?;
+    if sandbox_runtime_bootstrap_in_progress(obj) {
+        if sandbox_runtime_bootstrap_operation(obj) == Some("stop") {
+            mark_sandbox_runtime_stopping(&mut sandbox);
+        } else {
+            mark_sandbox_runtime_bootstrapping(&mut sandbox);
+        }
+    }
+    Ok((name, sandbox))
+}
+
+/// A stop holds the bootstrap annotations until its completion patch, but it
+/// is not a start: the gateway reads `Bootstrapping=True` as an accepted
+/// start that may replace a settled Stopped phase. Withhold the suspension
+/// until the stop completes, as a start does.
+fn mark_sandbox_runtime_stopping(sandbox: &mut Sandbox) {
+    clear_sandbox_runtime_operation_conditions(sandbox);
+    mark_sandbox_runtime_control_unavailable(sandbox);
+}
+
 fn mark_sandbox_runtime_bootstrapping(sandbox: &mut Sandbox) {
+    clear_sandbox_runtime_operation_conditions(sandbox);
     if let Some(status) = sandbox.status.as_mut() {
-        status.conditions.retain(|condition| {
-            condition.r#type != SANDBOX_SUSPENDED_CONDITION && condition.r#type != "Bootstrapping"
-        });
         status.conditions.push(SandboxCondition {
             r#type: "Bootstrapping".to_string(),
             status: "True".to_string(),
@@ -6988,6 +7006,14 @@ fn mark_sandbox_runtime_bootstrapping(sandbox: &mut Sandbox) {
     mark_sandbox_runtime_control_unavailable(sandbox);
 }
 
+fn clear_sandbox_runtime_operation_conditions(sandbox: &mut Sandbox) {
+    if let Some(status) = sandbox.status.as_mut() {
+        status.conditions.retain(|condition| {
+            condition.r#type != SANDBOX_SUSPENDED_CONDITION && condition.r#type != "Bootstrapping"
+        });
+    }
+}
+
 fn mark_sandbox_runtime_control_unavailable(sandbox: &mut Sandbox) {
     const REASON: &str = "DependenciesNotReady";
     const MESSAGE: &str = "sandbox-runtime enforcement dependencies are not ready";
@@ -14451,6 +14477,104 @@ mod tests {
         }));
     }
 
+    fn sandbox_cr_mid_operation(operation: Option<&str>) -> DynamicObject {
+        let mut annotations = BTreeMap::from([
+            (
+                ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING.to_string(),
+                "true".to_string(),
+            ),
+            (
+                ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE.to_string(),
+                SandboxRuntimeBootstrapPhase::Suspending
+                    .as_str()
+                    .to_string(),
+            ),
+        ]);
+        if let Some(operation) = operation {
+            annotations.insert(
+                ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION.to_string(),
+                operation.to_string(),
+            );
+        }
+        DynamicObject {
+            types: None,
+            metadata: ObjectMeta {
+                name: Some("alpha--work".to_string()),
+                namespace: Some("default".to_string()),
+                annotations: Some(annotations),
+                labels: Some(BTreeMap::from([
+                    (LABEL_SANDBOX_ID.to_string(), "uuid-123".to_string()),
+                    (LABEL_SANDBOX_NAME.to_string(), "work".to_string()),
+                    (LABEL_SANDBOX_WORKSPACE.to_string(), "alpha".to_string()),
+                    (
+                        LABEL_MANAGED_BY.to_string(),
+                        LABEL_MANAGED_BY_VALUE.to_string(),
+                    ),
+                ])),
+                ..Default::default()
+            },
+            data: serde_json::json!({
+                "status": {
+                    "conditions": [
+                        {"type": "Ready", "status": "False", "reason": "SandboxSuspended"},
+                        {"type": "Suspended", "status": "True", "reason": "PodTerminated"},
+                    ]
+                }
+            }),
+        }
+    }
+
+    fn has_condition(sandbox: &Sandbox, r#type: &str, status: &str) -> bool {
+        sandbox.status.as_ref().is_some_and(|sandbox_status| {
+            sandbox_status
+                .conditions
+                .iter()
+                .any(|condition| condition.r#type == r#type && condition.status == status)
+        })
+    }
+
+    // The gateway treats Bootstrapping=True as an accepted start and lets it
+    // replace a settled Stopped phase. A snapshot taken while a stop still
+    // holds its bootstrap annotations can arrive after that stop settled
+    // (#5195), so it must not claim a starting generation.
+    #[test]
+    fn stop_snapshot_does_not_report_a_starting_generation() {
+        let (_, sandbox) = sandbox_from_object_with_bootstrap_state(
+            "default",
+            &sandbox_cr_mid_operation(Some("stop")),
+        )
+        .unwrap();
+
+        assert!(!has_condition(&sandbox, "Bootstrapping", "True"));
+        // Suspension is still published only once the stop completes.
+        assert!(!has_condition(
+            &sandbox,
+            SANDBOX_SUSPENDED_CONDITION,
+            "True"
+        ));
+        assert!(!has_condition(&sandbox, "Ready", "True"));
+    }
+
+    #[test]
+    fn start_snapshots_report_a_starting_generation() {
+        for operation in [Some("create"), Some("restart"), None] {
+            let (_, sandbox) = sandbox_from_object_with_bootstrap_state(
+                "default",
+                &sandbox_cr_mid_operation(operation),
+            )
+            .unwrap();
+
+            assert!(
+                has_condition(&sandbox, "Bootstrapping", "True"),
+                "{operation:?}"
+            );
+            assert!(
+                !has_condition(&sandbox, SANDBOX_SUSPENDED_CONDITION, "True"),
+                "{operation:?}"
+            );
+        }
+    }
+
     #[test]
     fn completed_sandbox_runtime_bootstrap_preserves_real_suspension() {
         let sandbox = Sandbox {

crates/openshell-server/src/compute/mod.rs

@@ -14327,6 +14327,66 @@ mod tests {
         assert_eq!(current.phase(), SandboxPhase::Provisioning as i32);
     }
 
+    // Gateway side of the Kubernetes stop contract. While a stop holds its
+    // bootstrap annotations, the driver withdraws readiness and reports
+    // neither Bootstrapping nor Suspended. Such a snapshot keeps the stop in
+    // Stopping, never Provisioning or Error, and cannot revive the sandbox
+    // after the stop settles. This passes without the #5195 fix; the driver
+    // test stop_snapshot_does_not_report_a_starting_generation covers the
+    // regression.
+    #[tokio::test]
+    async fn kubernetes_stop_snapshots_settle_without_reviving_the_sandbox() {
+        for session_connected in [false, true] {
+            let runtime = test_runtime(Arc::new(TestDriver::default())).await;
+            let sandbox = sandbox_record("sb-stopping", "sandbox-stopping", SandboxPhase::Stopping);
+            runtime.store.put_message(&sandbox).await.unwrap();
+            if session_connected {
+                register_test_supervisor_session(&runtime, sandbox.object_id());
+            }
+
+            let mut in_flight = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
+            in_flight.status = Some(make_driver_status(make_driver_condition(
+                "DependenciesNotReady",
+                "sandbox-runtime enforcement dependencies are not ready",
+            )));
+            let mut settled_status = make_driver_status(make_driver_condition(
+                "SandboxSuspended",
+                "Sandbox is suspended",
+            ));
+            settled_status.conditions.push(DriverCondition {
+                r#type: "Suspended".to_string(),
+                status: "True".to_string(),
+                reason: "PodTerminated".to_string(),
+                message: "Pod terminated".to_string(),
+                transition_time: None,
+            });
+            let mut settled = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
+            settled.status = Some(settled_status);
+
+            for (step, snapshot, expected) in [
+                ("in-flight stop", &in_flight, SandboxPhase::Stopping),
+                ("completed stop", &settled, SandboxPhase::Stopped),
+                ("late in-flight stop", &in_flight, SandboxPhase::Stopped),
+            ] {
+                runtime
+                    .apply_sandbox_update(snapshot.clone())
+                    .await
+                    .unwrap();
+                let current = runtime
+                    .store
+                    .get_message::<Sandbox>(sandbox.object_id())
+                    .await
+                    .unwrap()
+                    .unwrap();
+                assert_eq!(
+                    current.phase(),
+                    expected as i32,
+                    "{step} (session_connected={session_connected})"
+                );
+            }
+        }
+    }
+
     #[tokio::test]
     async fn suspended_bootstrap_snapshot_does_not_revive_stopped_sandbox() {
         let runtime = test_runtime(Arc::new(TestDriver::default())).await;