Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
541c1c3
Add app disable and enable RPCs
jcasimir Oct 7, 2026
d078529
Hold a disabled app's services at zero
jcasimir Oct 7, 2026
b02e44b
Add miren app disable and enable commands
jcasimir Oct 7, 2026
24e90eb
Pause a disabled app's scheduled tasks
jcasimir Oct 8, 2026
11c71bd
Serve a disabled app's 503 from ingress as a holding page
jcasimir Oct 8, 2026
d567970
Show disabled and maintenance as one section in app status
jcasimir Oct 8, 2026
db63eb7
Point app disable and maintenance mode docs at each other
jcasimir Oct 8, 2026
74b3f3b
Report a disabled app as disabled in status regardless of version
jcasimir Oct 13, 2026
6762580
Grant app disable and enable to the admin workload roles
jcasimir Oct 13, 2026
55e150c
Skip scheduled ticks from a disable the scheduler never saw
jcasimir Oct 13, 2026
1882e3f
Resume app schedules from the latest enable time
evanphx Oct 13, 2026
403b488
Merge branch 'main' into app-suspend-resume
jcasimir Oct 14, 2026
eee6e75
Skip pending scheduled runs when their app is disabled
evanphx Oct 15, 2026
00d1c94
Merge remote-tracking branch 'up/main' into app-suspend-resume
jcasimir Oct 29, 2026
c153c7c
Merge main into app disable and enable
phinze Nov 5, 2026
f80af16
Authorize app disable and enable against the resolved app
phinze Nov 5, 2026
4b3bd33
Refuse to scale up a disabled app's pool by hand
phinze Nov 5, 2026
9d0a38e
Model a disabled app as one component
phinze Nov 5, 2026
facab53
Let deployers disable and enable apps
phinze Nov 5, 2026
c8a77e1
Finish enabling once the flag is clear, and never seed a disabled pool
phinze Nov 5, 2026
696f174
Refuse to build a new version of a disabled app
phinze Nov 5, 2026
c048e8a
Say what disable leaves running
phinze Nov 5, 2026
2c77e4b
Freeze a disabled app
phinze Nov 5, 2026
b1cbc5b
Give a disabled app one refusal
phinze Nov 6, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
340 changes: 333 additions & 7 deletions api/app/app_v1alpha/rpc.gen.go

Large diffs are not rendered by default.

31 changes: 31 additions & 0 deletions api/app/rpc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -660,6 +660,12 @@ interfaces:
- name: sourceValue
type: string
doc: "Normalized image reference or auto-detected stack name, depending on sourceKind"
- name: disabledAt
type: standard.Timestamp
doc: "When the app was disabled with app disable; unset when it is not disabled"
- name: disabledReason
type: string
doc: "Why the app was disabled, as given to app disable"
- name: setWorkloadRole
doc: |
Set the authorization role that identity tokens minted for this app's
Expand Down Expand Up @@ -780,6 +786,31 @@ interfaces:
type: int32
- name: stopped_sandboxes
type: int32
- name: disable
http:
post: /apps/{app}/disable
body: "*"
parameters:
- name: app
type: string
- name: reason
type: string
doc: "Optional explanation, shown by app status"
results:
- name: scaled_pools
type: int32
doc: "Pools scaled to zero by this call"
- name: enable
http:
post: /apps/{app}/enable
body: "*"
parameters:
- name: app
type: string
results:
- name: restored_pools
type: int32
doc: "Fixed-mode pools returned to their configured count"

- name: listVersions
doc: List an app's version history newest first, including replaced and ephemeral versions.
Expand Down
101 changes: 85 additions & 16 deletions api/core/core_v1alpha/schema.gen.go

Large diffs are not rendered by default.

23 changes: 23 additions & 0 deletions api/core/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,29 @@ kinds:
type: ref
doc: Reference to the initial ConfigVersion entity created before the first deploy

disabled:
type: component
doc: >-
Operator-initiated disabled state, set by `miren app disable`. When
present, every service is held at zero instances, anything that would
change the active version is refused, and requests get a 503. Absent
means the app runs normally.
attrs:
at:
type: time
doc: When the app was disabled
reason:
type: string
doc: Why the app was disabled, shown by `miren app status`

schedules_resumed_at:
type: time
doc: >-
When `miren app enable` last resumed the app's schedules. The run
scheduler discards pending ticks up to this time, including ticks due
before the disable, so a pause it never observed between sweeps still
counts.

workload_role:
type: string
doc: >-
Expand Down
2 changes: 1 addition & 1 deletion api/run/run_v1alpha/schema.gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ func (o *Run) InitSchema(sb *schema.SchemaBuilder) {
sb.Singleton("dev.miren.run/status.timed_out")
sb.Singleton("dev.miren.run/status.canceled")
sb.Singleton("dev.miren.run/status.skipped")
sb.Ref("status", "dev.miren.run/run.status", schema.Doc("Indexed because the deadline sweep lists non-terminal runs on a short\ninterval; scanning every run of every app to find them would not hold up.\n\n\"skipped\" is a scheduled run that never executed because its predecessor\nwas still going. It exists so a skipped tick is visible rather than\nlooking like a gap, since \"my job stopped running\" and \"my job got\nslower than its interval\" are otherwise indistinguishable from outside.\n"), schema.Indexed, schema.Choices(RunStatusPendingId, RunStatusRunningId, RunStatusSucceededId, RunStatusFailedId, RunStatusTimedOutId, RunStatusCanceledId, RunStatusSkippedId))
sb.Ref("status", "dev.miren.run/run.status", schema.Doc("Indexed because the deadline sweep lists non-terminal runs on a short\ninterval; scanning every run of every app to find them would not hold up.\n\n\"skipped\" is scheduled work discarded because its predecessor was still\ngoing or the app was disabled before the next attempt could start.\nIt exists so a skipped tick is visible rather than looking like a gap,\nsince \"my job stopped running\" and \"my job got slower than its interval\"\nare otherwise indistinguishable from outside.\n"), schema.Indexed, schema.Choices(RunStatusPendingId, RunStatusRunningId, RunStatusSucceededId, RunStatusFailedId, RunStatusTimedOutId, RunStatusCanceledId, RunStatusSkippedId))
sb.String("task", "dev.miren.run/run.task", schema.Doc("Task name from app.toml"))
sb.String("tick", "dev.miren.run/run.tick", schema.Doc("Scheduled runs only: the RFC 3339 instant this run claims. Every replica\nderives the same ticks from the stored calendar expression, so creating\nthe run is a create-if-absent on an id derived from this, and etcd picks\nthe single winner.\n"))
sb.String("timeout", "dev.miren.run/run.timeout", schema.Doc("Go duration bounding the run; empty means the platform default, \"0\" unbounded"))
Expand Down
9 changes: 5 additions & 4 deletions api/run/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,11 @@ kinds:
Indexed because the deadline sweep lists non-terminal runs on a short
interval; scanning every run of every app to find them would not hold up.

"skipped" is a scheduled run that never executed because its predecessor
was still going. It exists so a skipped tick is visible rather than
looking like a gap, since "my job stopped running" and "my job got
slower than its interval" are otherwise indistinguishable from outside.
"skipped" is scheduled work discarded because its predecessor was still
going or the app was disabled before the next attempt could start.
It exists so a skipped tick is visible rather than looking like a gap,
since "my job stopped running" and "my job got slower than its interval"
are otherwise indistinguishable from outside.

attempt:
type: int
Expand Down
42 changes: 42 additions & 0 deletions cli/commands/app_disable.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package commands

import (
"miren.dev/runtime/api/app/app_v1alpha"
)

func AppDisable(ctx *Context, opts struct {
Reason string `short:"r" long:"reason" description:"Why the app is disabled, shown by app status"`
AppCentric
}) error {
crudcl, err := ctx.RPCClient("dev.miren.runtime/app")
if err != nil {
return err
}

result, err := app_v1alpha.NewCrudClient(crudcl).Disable(ctx, opts.App, opts.Reason)
if err != nil {
return err
}

ctx.Printf("Disabled app %s (scaled %d pools to zero)\n", opts.App, result.ScaledPools())
ctx.Printf("Run `miren app enable -a %s` to start it again.\n", opts.App)
return nil
}

func AppEnable(ctx *Context, opts struct {
AppCentric
}) error {
crudcl, err := ctx.RPCClient("dev.miren.runtime/app")
if err != nil {
return err
}

result, err := app_v1alpha.NewCrudClient(crudcl).Enable(ctx, opts.App)
if err != nil {
return err
}

ctx.Printf("Enabled app %s (restored %d fixed pools; autoscaled services start on the next request)\n",
opts.App, result.RestoredPools())
return nil
}
2 changes: 2 additions & 0 deletions cli/commands/app_list.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,8 @@ func AppList(ctx *Context, opts struct {
} else {
status = infoYellow.Render("⚠️ 0 (fixed)")
}
case "disabled":
status = infoYellow.Render("⏸ disabled")
case "healthy":
status = infoGreen.Render(fmt.Sprintf("%d%s", a.ReadyInstances(), modeSuffix))
case "degraded", "starting":
Expand Down
40 changes: 38 additions & 2 deletions cli/commands/app_status.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

"miren.dev/runtime/api/app/app_v1alpha"
"miren.dev/runtime/api/deployment/deployment_v1alpha"
"miren.dev/runtime/pkg/rpc/standard"
"miren.dev/runtime/pkg/ui"
)

Expand Down Expand Up @@ -80,8 +81,13 @@ func AppStatus(ctx *Context, opts AppStatusOptions) error {
if source := formatSource(appResult.SourceKind(), appResult.SourceValue()); source != "" {
items = append(items, ui.NewNamedValue("Source", source))
}
if down := appResult.MaintenanceRoutes(); len(down) > 0 {
items = append(items, ui.NewNamedValue("Maintenance", fmt.Sprintf("%s serving a holding page", maintenanceRouteList(down))))
var disabledAt *time.Time
if appResult.HasDisabledAt() {
at := standard.FromTimestamp(appResult.DisabledAt())
disabledAt = &at
}
if label, lines := notServingLines(disabledAt, appResult.DisabledReason(), appResult.MaintenanceRoutes()); label != "" {
items = append(items, ui.NewNamedValue(label, strings.Join(lines, "; ")))
}

if activeDeployment != nil && activeDeployment.HasDeployment() {
Expand Down Expand Up @@ -224,6 +230,8 @@ func printAppStatusJSON(
Source *sourceJSON `json:"source,omitempty"`
WorkloadRole string `json:"workload_role,omitempty"`
MaintenanceRoutes []string `json:"maintenance_routes,omitempty"`
DisabledAt *time.Time `json:"disabled_at,omitempty"`
DisabledReason string `json:"disabled_reason,omitempty"`
Configuration *configuration `json:"configuration,omitempty"`
ActiveDeployment *deploymentJSON `json:"active_deployment,omitempty"`
RecentDeployments []deploymentJSON `json:"recent_deployments,omitempty"`
Expand All @@ -234,8 +242,13 @@ func printAppStatusJSON(
Cluster: cluster,
WorkloadRole: appResult.WorkloadRole(),
MaintenanceRoutes: appResult.MaintenanceRoutes(),
DisabledReason: appResult.DisabledReason(),
Services: health,
}
if appResult.HasDisabledAt() {
at := standard.FromTimestamp(appResult.DisabledAt())
output.DisabledAt = &at
}
if healthErr != nil {
output.HealthError = healthErr.Error()
}
Expand Down Expand Up @@ -290,3 +303,26 @@ func maintenanceRouteList(hosts []string) string {

return strings.Join(named, ", ")
}

// notServingLines explains why an app, or some of its routes, isn't serving.
// A disabled app and a route in maintenance read as one section: the app is
// off, and those routes show their maintenance page instead of the disabled
// one.
func notServingLines(disabledAt *time.Time, reason string, maintenanceRoutes []string) (string, []string) {
if disabledAt == nil {
if len(maintenanceRoutes) == 0 {
return "", nil
}
return "Maintenance", []string{fmt.Sprintf("%s serving a holding page", maintenanceRouteList(maintenanceRoutes))}
}

line := fmt.Sprintf("disabled since %s", disabledAt.Local().Format(time.RFC1123))
if reason != "" {
line += " — " + reason
}
lines := []string{line}
if len(maintenanceRoutes) > 0 {
lines = append(lines, fmt.Sprintf("in maintenance instead: %s", maintenanceRouteList(maintenanceRoutes)))
}
return "Not serving", lines
}
55 changes: 55 additions & 0 deletions cli/commands/app_status_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package commands

import (
"testing"
"time"

"github.com/stretchr/testify/assert"
)

func TestNotServingLines(t *testing.T) {
at := time.Date(2026, 10, 8, 17, 0, 0, 0, time.UTC)
since := "disabled since " + at.Local().Format(time.RFC1123)

tests := []struct {
name string
disabledAt *time.Time
reason string
routes []string
wantLabel string
wantLines []string
}{
{name: "serving", wantLabel: "", wantLines: nil},
{
name: "maintenance only",
routes: []string{"shop.example.com"},
wantLabel: "Maintenance",
wantLines: []string{"shop.example.com serving a holding page"},
},
{
name: "disabled with a reason",
disabledAt: &at,
reason: "moved to the new cluster",
wantLabel: "Not serving",
wantLines: []string{since + " — moved to the new cluster"},
},
{
name: "disabled with routes in maintenance",
disabledAt: &at,
routes: []string{"shop.example.com", ""},
wantLabel: "Not serving",
wantLines: []string{
since,
"in maintenance instead: shop.example.com, the default route",
},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
label, lines := notServingLines(tt.disabledAt, tt.reason, tt.routes)
assert.Equal(t, tt.wantLabel, label)
assert.Equal(t, tt.wantLines, lines)
})
}
}
28 changes: 28 additions & 0 deletions cli/commands/commands.go
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,20 @@ miren deployment create --format jsonl | jq -c 'select(.event == "build_step")'
Body: "miren app restart -s web",
}),
))
d.Dispatch("app disable", Infer("app disable", "Hold every service of an app at zero instances", AppDisable,
WithDescription(appDisableDescription),
WithExample(mflags.Example{
Name: "Disable an app",
Body: "miren app disable -a myapp --reason \"Moved to the new cluster\"",
}),
))
d.Dispatch("app enable", Infer("app enable", "Start a disabled app again", AppEnable,
WithDescription(appEnableDescription),
WithExample(mflags.Example{
Name: "Enable an app",
Body: "miren app enable -a myapp",
}),
))
d.Dispatch("app set-workload-role", Infer("app set-workload-role", "Set the API role for an app's sandbox identity tokens", AppSetWorkloadRole,
WithExample(mflags.Example{
Name: "Let an app's workloads read and deploy their own app",
Expand Down Expand Up @@ -1498,6 +1512,20 @@ Use restart to:
You do not need to restart after ` + "`" + `miren env set` + "`" + `, ` + "`" + `miren env delete` + "`" + `, ` + "`" + `miren addon create` + "`" + `, or ` + "`" + `miren addon delete` + "`" + `. Each of those already creates a new version and rolls out new sandboxes automatically. A manual restart on top only adds a redundant rollout.
:::`

const appDisableDescription = `Disable turns an app off. Every service is held at zero instances: running sandboxes shut down, scheduled tasks pause, and visitors get a 503 holding page. The app is frozen until you enable it: deploys, rollbacks, and config changes are refused. The app keeps its versions, routes, environment, disks, and addons, so ` + "`" + `miren app enable` + "`" + ` brings it back as it was. Task runs already in progress finish, and addons keep running.

Disabled is an explicit state, so ` + "`" + `miren app list` + "`" + ` and ` + "`" + `miren app status` + "`" + ` report it as such rather than as an app that failed to start.

:::note[Manual tasks still run]
` + "`" + `miren app run` + "`" + ` keeps working, which leaves room for a backup or a migration while the app is off. Scheduled tasks don't fire while the app is disabled, and enabling it resumes the schedule from that moment without running the ticks it missed.
:::

:::tip[Taking only traffic down?]
To show visitors a holding page while the app keeps running, for a migration for example, put its routes into maintenance with ` + "`" + `miren route down` + "`" + ` instead.
:::`

const appEnableDescription = `Enable turns a disabled app back on. Fixed-mode services return to their configured count right away; autoscaled services stay at zero and start on the next request.`

const secretSetDescription = `Stores a value in the cluster's secret store. The value is encrypted at rest and is never echoed, logged, or written to disk by this command — it travels to the server, which holds the only key.

Each write mints a new immutable version and prints its handle, e.g. ` + "`" + `payments/stripe-key@x1A` + "`" + `. Storing a value identical to the current one is reported as unchanged rather than minting a duplicate, so re-running the command is safe.
Expand Down
4 changes: 4 additions & 0 deletions cli/commands/deploy_events.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,10 @@ func outcomeEvent(o terminalOutcome) deployevents.Outcome {
return deployevents.OutcomeHealthy
case outcomeScaledToZero:
return deployevents.OutcomeScaledToZero
case outcomeDisabled:
// Published as scaled to zero, which it is, rather than widening the
// event vocabulary for a variant of the same state.
return deployevents.OutcomeScaledToZero
case outcomeNoService:
// Keep the published value for compatibility. It predates static-only
// apps and means the deployment has no service process to wait for.
Expand Down
10 changes: 10 additions & 0 deletions cli/commands/health_wait.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,9 @@ const (
// decisionNoService: the app has no long-running process by design. Its
// tasks or static content are available, which is as "up" as it gets.
decisionNoService
// decisionDisabled: the app is disabled, so the version is deployed but
// held at zero until the app is enabled.
decisionDisabled
)

// decideActivation maps a snapshot to a terminal decision (or "keep waiting").
Expand Down Expand Up @@ -125,6 +128,8 @@ func decideActivation(snap healthSnapshot) activationDecision {
return decisionNoService
case snap.health == apphealth.Idle:
return decisionScaledToZero
case snap.health == apphealth.Disabled:
return decisionDisabled
case snap.ready > 0:
return decisionHealthy
default:
Expand Down Expand Up @@ -189,6 +194,7 @@ const (
outcomeHealthy terminalOutcome = iota
outcomeScaledToZero
outcomeNoService
outcomeDisabled
outcomeCrashed
outcomeTimeout
outcomeCanceled
Expand All @@ -210,6 +216,8 @@ func pollOutcome(ctx context.Context, getter appInfoGetter, appName, versionID s
return outcomeScaledToZero, snap, true
case decisionNoService:
return outcomeNoService, snap, true
case decisionDisabled:
return outcomeDisabled, snap, true
case decisionCrashed:
return outcomeCrashed, snap, true
default:
Expand Down Expand Up @@ -240,6 +248,8 @@ func healthOutcomeText(versionDisplay string, outcome terminalOutcome, snap heal
// Deliberately not the scaled-to-zero wording: nothing went to sleep,
// this app never had a long-running process to begin with.
return fmt.Sprintf("Version %s deployed — no long-running service required", versionDisplay), true
case outcomeDisabled:
return fmt.Sprintf("Version %s deployed — the app is disabled; run `miren app enable` to start it", versionDisplay), true
case outcomeCrashed:
detail := ""
if snap.crashCount > 0 {
Expand Down
1 change: 1 addition & 0 deletions cli/commands/health_wait_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ func TestDecideActivation(t *testing.T) {
// Without its own arm this falls to ready > 0, which an app without a
// service never satisfies -- the deploy would time out despite succeeding.
{"service-free app is deployed", healthSnapshot{versionActive: true, health: apphealth.Ready, ready: 0, desired: 0}, decisionNoService},
{"disabled app is deployed", healthSnapshot{versionActive: true, health: apphealth.Disabled, ready: 0, desired: 0}, decisionDisabled},
{"one of many serving", healthSnapshot{versionActive: true, health: apphealth.Degraded, ready: 1, desired: 3}, decisionHealthy},
{"fully healthy", healthSnapshot{versionActive: true, health: apphealth.Healthy, ready: 2, desired: 2}, decisionHealthy},
{"active but nothing serving", healthSnapshot{versionActive: true, health: apphealth.Starting, ready: 0, desired: 1}, decisionWait},
Expand Down
Loading
Loading