Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions internal/agent/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -278,10 +278,42 @@ func (r *ImpVMReconciler) maybeLaunchRunner(ctx context.Context, vm *impdevv1alp
if r.Recorder != nil {
r.Recorder.Event(vm, corev1.EventTypeWarning, "RunnerLaunchFailed", res.Err.Error())
}
return
}
if res.HandoffDone && res.ExitCode == 0 {
if _, err := r.finishRunnerLaunchSucceeded(context.Background(), vm); err != nil {
log.Error(err, "failed to mark VM after runner exit", "vm", vm.Name)
}
}
}()
}

// finishRunnerLaunchSucceeded makes a completed one-time runner terminal so the
// pool controller can delete it and restore idle capacity. The handoff runs
// asynchronously, so fetch a fresh object rather than patching a stale copy.
func (r *ImpVMReconciler) finishRunnerLaunchSucceeded(ctx context.Context, vm *impdevv1alpha1.ImpVM) (ctrl.Result, error) {
key := client.ObjectKeyFromObject(vm)
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
latest := &impdevv1alpha1.ImpVM{}
if err := r.Get(ctx, key, latest); err != nil {
return client.IgnoreNotFound(err)
}
switch latest.Status.Phase {
case impdevv1alpha1.VMPhaseFailed,
impdevv1alpha1.VMPhaseSucceeded,
impdevv1alpha1.VMPhaseTerminating:
return nil
}
base := latest.DeepCopy()
latest.Status.Phase = impdevv1alpha1.VMPhaseSucceeded
latest.Status.StartedAt = nil
latest.Status.IP = ""
latest.Status.RuntimePID = 0
return r.Status().Patch(ctx, latest, client.MergeFrom(base))
})
return ctrl.Result{}, err
}

// finishRunnerLaunchFailed makes a failed guest handoff terminal. A JIT runner
// cannot start without its one-time payload, so leaving it Running would keep a
// VM that can never accept work. The handoff runs asynchronously, so fetch a
Expand Down
24 changes: 24 additions & 0 deletions internal/agent/reconciler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,30 @@ var _ = Describe("ImpVM Agent: runner JIT handoff", func() {
}).Should(Succeed())
})

It("marks the VM Succeeded when the one-time runner exits successfully", func() {
vm := &impdevv1alpha1.ImpVM{
ObjectMeta: metav1.ObjectMeta{Name: "tc-runner-handoff-success", Namespace: "default"},
Spec: impdevv1alpha1.ImpVMSpec{NodeName: testNode, RunnerConfigSecret: "jitconfig"},
}
Expect(k8sClient.Create(ctx, vm)).To(Succeed())
DeferCleanup(func() { _ = k8sClient.Delete(ctx, vm) })
base := vm.DeepCopy()
vm.Status.Phase = impdevv1alpha1.VMPhaseRunning
Expect(k8sClient.Status().Patch(ctx, vm, client.MergeFrom(base))).To(Succeed())

r := newReconciler(&vsockStubDriver{StubDriver: NewStubDriver()})
r.runnerLaunch = func(context.Context, *impdevv1alpha1.ImpVM) runnerlaunch.Result {
return runnerlaunch.Result{HandoffDone: true, ExitCode: 0}
}
r.maybeLaunchRunner(ctx, vm)

updated := &impdevv1alpha1.ImpVM{}
Eventually(func(g Gomega) {
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(vm), updated)).To(Succeed())
g.Expect(updated.Status.Phase).To(Equal(impdevv1alpha1.VMPhaseSucceeded))
}).Should(Succeed())
})

It("does not overwrite a terminal status with a late handoff failure", func() {
vm := &impdevv1alpha1.ImpVM{
ObjectMeta: metav1.ObjectMeta{Name: "tc-runner-handoff-late-failure", Namespace: "default"},
Expand Down
Loading