From 541c1c3efba04347cacf7fd55827ee6500bdce81 Mon Sep 17 00:00:00 2001 From: Jeff Casimir Date: Wed, 7 Oct 2026 09:37:00 -0600 Subject: [PATCH 01/21] Add app disable and enable RPCs An App gains disabled_at and disabled_reason, an explicit record that the operator turned it off, as opposed to an app at zero because something broke. Disable marks the app and scales its pools to zero; enable clears the mark and returns fixed-mode pools of the active version to their configured count, leaving autoscaled services to start on the next request. Restart refuses on a disabled app, since restoring fixed counts would undo it until the next resync. App health reports a new disabled state, and getConfiguration returns when and why. Co-Authored-By: Claude Opus 5.5 --- api/app/app_v1alpha/rpc.gen.go | 340 +++++++++++++++++++++++++++- api/app/rpc.yml | 31 +++ api/core/core_v1alpha/schema.gen.go | 26 ++- api/core/schema.yml | 11 + pkg/apphealth/apphealth.go | 3 + servers/app/app.go | 23 ++ servers/app/disable.go | 147 ++++++++++++ servers/app/disable_test.go | 111 +++++++++ servers/app/runtime.go | 6 + 9 files changed, 690 insertions(+), 8 deletions(-) create mode 100644 servers/app/disable.go create mode 100644 servers/app/disable_test.go diff --git a/api/app/app_v1alpha/rpc.gen.go b/api/app/app_v1alpha/rpc.gen.go index e6bee754f..b1ac288f4 100644 --- a/api/app/app_v1alpha/rpc.gen.go +++ b/api/app/app_v1alpha/rpc.gen.go @@ -2666,13 +2666,15 @@ func (v *CrudGetConfigurationArgs) UnmarshalJSON(data []byte) error { } type crudGetConfigurationResultsData struct { - Configuration *Configuration `cbor:"0,keyasint,omitempty" json:"configuration,omitempty"` - VersionId *string `cbor:"1,keyasint,omitempty" json:"versionId,omitempty"` - VersionShortId *string `cbor:"2,keyasint,omitempty" json:"versionShortId,omitempty"` - WorkloadRole *string `cbor:"3,keyasint,omitempty" json:"workloadRole,omitempty"` - MaintenanceRoutes *[]string `cbor:"4,keyasint,omitempty" json:"maintenanceRoutes,omitempty"` - SourceKind *string `cbor:"5,keyasint,omitempty" json:"sourceKind,omitempty"` - SourceValue *string `cbor:"6,keyasint,omitempty" json:"sourceValue,omitempty"` + Configuration *Configuration `cbor:"0,keyasint,omitempty" json:"configuration,omitempty"` + VersionId *string `cbor:"1,keyasint,omitempty" json:"versionId,omitempty"` + VersionShortId *string `cbor:"2,keyasint,omitempty" json:"versionShortId,omitempty"` + WorkloadRole *string `cbor:"3,keyasint,omitempty" json:"workloadRole,omitempty"` + MaintenanceRoutes *[]string `cbor:"4,keyasint,omitempty" json:"maintenanceRoutes,omitempty"` + SourceKind *string `cbor:"5,keyasint,omitempty" json:"sourceKind,omitempty"` + SourceValue *string `cbor:"6,keyasint,omitempty" json:"sourceValue,omitempty"` + DisabledAt *standard.Timestamp `cbor:"7,keyasint,omitempty" json:"disabledAt,omitempty"` + DisabledReason *string `cbor:"8,keyasint,omitempty" json:"disabledReason,omitempty"` } type CrudGetConfigurationResults struct { @@ -2709,6 +2711,14 @@ func (v *CrudGetConfigurationResults) SetSourceValue(sourceValue string) { v.data.SourceValue = &sourceValue } +func (v *CrudGetConfigurationResults) SetDisabledAt(disabledAt *standard.Timestamp) { + v.data.DisabledAt = disabledAt +} + +func (v *CrudGetConfigurationResults) SetDisabledReason(disabledReason string) { + v.data.DisabledReason = &disabledReason +} + func (v *CrudGetConfigurationResults) MarshalCBOR() ([]byte, error) { return cbor.Marshal(v.data) } @@ -3461,6 +3471,148 @@ func (v *CrudRestartResults) UnmarshalJSON(data []byte) error { return json.Unmarshal(data, &v.data) } +type crudDisableArgsData struct { + App *string `cbor:"0,keyasint,omitempty" json:"app,omitempty"` + Reason *string `cbor:"1,keyasint,omitempty" json:"reason,omitempty"` +} + +type CrudDisableArgs struct { + call rpc.Call + data crudDisableArgsData +} + +func (v *CrudDisableArgs) HasApp() bool { + return v.data.App != nil +} + +func (v *CrudDisableArgs) App() string { + if v.data.App == nil { + return "" + } + return *v.data.App +} + +func (v *CrudDisableArgs) HasReason() bool { + return v.data.Reason != nil +} + +func (v *CrudDisableArgs) Reason() string { + if v.data.Reason == nil { + return "" + } + return *v.data.Reason +} + +func (v *CrudDisableArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *CrudDisableArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *CrudDisableArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *CrudDisableArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type crudDisableResultsData struct { + ScaledPools *int32 `cbor:"0,keyasint,omitempty" json:"scaled_pools,omitempty"` +} + +type CrudDisableResults struct { + call rpc.Call + data crudDisableResultsData +} + +func (v *CrudDisableResults) SetScaledPools(scaled_pools int32) { + v.data.ScaledPools = &scaled_pools +} + +func (v *CrudDisableResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *CrudDisableResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *CrudDisableResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *CrudDisableResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type crudEnableArgsData struct { + App *string `cbor:"0,keyasint,omitempty" json:"app,omitempty"` +} + +type CrudEnableArgs struct { + call rpc.Call + data crudEnableArgsData +} + +func (v *CrudEnableArgs) HasApp() bool { + return v.data.App != nil +} + +func (v *CrudEnableArgs) App() string { + if v.data.App == nil { + return "" + } + return *v.data.App +} + +func (v *CrudEnableArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *CrudEnableArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *CrudEnableArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *CrudEnableArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type crudEnableResultsData struct { + RestoredPools *int32 `cbor:"0,keyasint,omitempty" json:"restored_pools,omitempty"` +} + +type CrudEnableResults struct { + call rpc.Call + data crudEnableResultsData +} + +func (v *CrudEnableResults) SetRestoredPools(restored_pools int32) { + v.data.RestoredPools = &restored_pools +} + +func (v *CrudEnableResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *CrudEnableResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *CrudEnableResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *CrudEnableResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + type CrudNew struct { rpc.Call args CrudNewArgs @@ -3773,6 +3925,58 @@ func (t *CrudRestart) Results() *CrudRestartResults { return results } +type CrudDisable struct { + rpc.Call + args CrudDisableArgs + results CrudDisableResults +} + +func (t *CrudDisable) Args() *CrudDisableArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *CrudDisable) Results() *CrudDisableResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + +type CrudEnable struct { + rpc.Call + args CrudEnableArgs + results CrudEnableResults +} + +func (t *CrudEnable) Args() *CrudEnableArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *CrudEnable) Results() *CrudEnableResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + type Crud interface { New(ctx context.Context, state *CrudNew) error SetConfiguration(ctx context.Context, state *CrudSetConfiguration) error @@ -3786,6 +3990,8 @@ type Crud interface { SetInitialEnvVars(ctx context.Context, state *CrudSetInitialEnvVars) error DeleteEnvVar(ctx context.Context, state *CrudDeleteEnvVar) error Restart(ctx context.Context, state *CrudRestart) error + Disable(ctx context.Context, state *CrudDisable) error + Enable(ctx context.Context, state *CrudEnable) error } type reexportCrud struct { @@ -3840,6 +4046,14 @@ func (reexportCrud) Restart(ctx context.Context, state *CrudRestart) error { panic("not implemented") } +func (reexportCrud) Disable(ctx context.Context, state *CrudDisable) error { + panic("not implemented") +} + +func (reexportCrud) Enable(ctx context.Context, state *CrudEnable) error { + panic("not implemented") +} + func (t reexportCrud) CapabilityClient() rpc.Client { return t.client } @@ -4035,6 +4249,38 @@ func AdaptCrud(t Crud) *rpc.Interface { return t.Restart(ctx, &CrudRestart{Call: call}) }, }, + { + Name: "disable", + InterfaceName: "Crud", + Index: 0, + Public: false, + Params: []string{"app", "reason"}, + HTTP: &rpc.HTTPBinding{ + Verb: "POST", + Path: "/api/v1/apps/{app}/disable", + Body: "*", + PathParams: []string{"app"}, + }, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.Disable(ctx, &CrudDisable{Call: call}) + }, + }, + { + Name: "enable", + InterfaceName: "Crud", + Index: 0, + Public: false, + Params: []string{"app"}, + HTTP: &rpc.HTTPBinding{ + Verb: "POST", + Path: "/api/v1/apps/{app}/enable", + Body: "*", + PathParams: []string{"app"}, + }, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.Enable(ctx, &CrudEnable{Call: call}) + }, + }, } return rpc.NewInterface(methods, t) @@ -4203,6 +4449,25 @@ func (v *CrudClientGetConfigurationResults) SourceValue() string { return *v.data.SourceValue } +func (v *CrudClientGetConfigurationResults) HasDisabledAt() bool { + return v.data.DisabledAt != nil +} + +func (v *CrudClientGetConfigurationResults) DisabledAt() *standard.Timestamp { + return v.data.DisabledAt +} + +func (v *CrudClientGetConfigurationResults) HasDisabledReason() bool { + return v.data.DisabledReason != nil +} + +func (v *CrudClientGetConfigurationResults) DisabledReason() string { + if v.data.DisabledReason == nil { + return "" + } + return *v.data.DisabledReason +} + func (v CrudClient) GetConfiguration(ctx context.Context, app string) (*CrudClientGetConfigurationResults, error) { args := CrudGetConfigurationArgs{} args.data.App = &app @@ -4523,6 +4788,67 @@ func (v CrudClient) Restart(ctx context.Context, app string, service string) (*C return &CrudClientRestartResults{client: v.Client, data: ret}, nil } +type CrudClientDisableResults struct { + client rpc.Client + data crudDisableResultsData +} + +func (v *CrudClientDisableResults) HasScaledPools() bool { + return v.data.ScaledPools != nil +} + +func (v *CrudClientDisableResults) ScaledPools() int32 { + if v.data.ScaledPools == nil { + return 0 + } + return *v.data.ScaledPools +} + +func (v CrudClient) Disable(ctx context.Context, app string, reason string) (*CrudClientDisableResults, error) { + args := CrudDisableArgs{} + args.data.App = &app + args.data.Reason = &reason + + var ret crudDisableResultsData + + err := v.Call(ctx, "disable", &args, &ret) + if err != nil { + return nil, err + } + + return &CrudClientDisableResults{client: v.Client, data: ret}, nil +} + +type CrudClientEnableResults struct { + client rpc.Client + data crudEnableResultsData +} + +func (v *CrudClientEnableResults) HasRestoredPools() bool { + return v.data.RestoredPools != nil +} + +func (v *CrudClientEnableResults) RestoredPools() int32 { + if v.data.RestoredPools == nil { + return 0 + } + return *v.data.RestoredPools +} + +func (v CrudClient) Enable(ctx context.Context, app string) (*CrudClientEnableResults, error) { + args := CrudEnableArgs{} + args.data.App = &app + + var ret crudEnableResultsData + + err := v.Call(ctx, "enable", &args, &ret) + if err != nil { + return nil, err + } + + return &CrudClientEnableResults{client: v.Client, data: ret}, nil +} + type userQueryWhoAmIArgsData struct{} type UserQueryWhoAmIArgs struct { diff --git a/api/app/rpc.yml b/api/app/rpc.yml index db7979089..f95b17293 100644 --- a/api/app/rpc.yml +++ b/api/app/rpc.yml @@ -573,6 +573,12 @@ interfaces: - name: sourceValue type: string doc: "Normalized image reference or auto-detected stack name, depending on sourceKind" + - name: disabledAt + type: standard.Timestamp + doc: "When the app was disabled with app disable; unset when it is not disabled" + - name: disabledReason + type: string + doc: "Why the app was disabled, as given to app disable" - name: setHost http: put: /apps/{app}/host @@ -702,6 +708,31 @@ interfaces: type: int32 - name: stopped_sandboxes type: int32 + - name: disable + http: + post: /apps/{app}/disable + body: "*" + parameters: + - name: app + type: string + - name: reason + type: string + doc: "Optional explanation, shown by app status" + results: + - name: scaled_pools + type: int32 + doc: "Pools scaled to zero by this call" + - name: enable + http: + post: /apps/{app}/enable + body: "*" + parameters: + - name: app + type: string + results: + - name: restored_pools + type: int32 + doc: "Fixed-mode pools returned to their configured count" - name: UserQuery methods: diff --git a/api/core/core_v1alpha/schema.gen.go b/api/core/core_v1alpha/schema.gen.go index 11dde87cd..eacc1a65d 100644 --- a/api/core/core_v1alpha/schema.gen.go +++ b/api/core/core_v1alpha/schema.gen.go @@ -1267,6 +1267,8 @@ const ( AppActiveDeploymentId = entity.Id("dev.miren.core/app.active_deployment") AppActiveVersionId = entity.Id("dev.miren.core/app.active_version") AppDeploymentLockId = entity.Id("dev.miren.core/app.deployment_lock") + AppDisabledAtId = entity.Id("dev.miren.core/app.disabled_at") + AppDisabledReasonId = entity.Id("dev.miren.core/app.disabled_reason") AppInitialConfigId = entity.Id("dev.miren.core/app.initial_config") AppProjectId = entity.Id("dev.miren.core/app.project") AppWorkloadRoleId = entity.Id("dev.miren.core/app.workload_role") @@ -1277,6 +1279,8 @@ type App struct { ActiveDeployment entity.Id `cbor:"active_deployment,omitempty" json:"active_deployment,omitempty"` ActiveVersion entity.Id `cbor:"active_version,omitempty" json:"active_version,omitempty"` DeploymentLock DeploymentLock `cbor:"deployment_lock,omitempty" json:"deployment_lock"` + DisabledAt time.Time `cbor:"disabled_at,omitempty" json:"disabled_at"` + DisabledReason string `cbor:"disabled_reason,omitempty" json:"disabled_reason,omitempty"` InitialConfig entity.Id `cbor:"initial_config,omitempty" json:"initial_config,omitempty"` Project entity.Id `cbor:"project,omitempty" json:"project,omitempty"` WorkloadRole string `cbor:"workload_role,omitempty" json:"workload_role,omitempty"` @@ -1293,6 +1297,12 @@ func (o *App) Decode(e entity.AttrGetter) { if a, ok := e.Get(AppDeploymentLockId); ok && a.Value.Kind() == entity.KindComponent { o.DeploymentLock.Decode(a.Value.Component()) } + if a, ok := e.Get(AppDisabledAtId); ok && a.Value.Kind() == entity.KindTime { + o.DisabledAt = a.Value.Time() + } + if a, ok := e.Get(AppDisabledReasonId); ok && a.Value.Kind() == entity.KindString { + o.DisabledReason = a.Value.String() + } if a, ok := e.Get(AppInitialConfigId); ok && a.Value.Kind() == entity.KindId { o.InitialConfig = a.Value.Id() } @@ -1330,6 +1340,12 @@ func (o *App) Encode() (attrs []entity.Attr) { if !o.DeploymentLock.Empty() { attrs = append(attrs, entity.Component(AppDeploymentLockId, o.DeploymentLock.Encode())) } + if !entity.Empty(o.DisabledAt) { + attrs = append(attrs, entity.Time(AppDisabledAtId, o.DisabledAt)) + } + if !entity.Empty(o.DisabledReason) { + attrs = append(attrs, entity.String(AppDisabledReasonId, o.DisabledReason)) + } if !entity.Empty(o.InitialConfig) { attrs = append(attrs, entity.Ref(AppInitialConfigId, o.InitialConfig)) } @@ -1354,6 +1370,12 @@ func (o *App) Empty() bool { if !o.DeploymentLock.Empty() { return false } + if !entity.Empty(o.DisabledAt) { + return false + } + if !entity.Empty(o.DisabledReason) { + return false + } if !entity.Empty(o.InitialConfig) { return false } @@ -1371,6 +1393,8 @@ func (o *App) InitSchema(sb *schema.SchemaBuilder) { sb.Ref("active_version", "dev.miren.core/app.active_version", schema.Doc("The version of the project that should be used")) sb.Component("deployment_lock", "dev.miren.core/app.deployment_lock", schema.Doc("The expiring lock held by the deployment currently mutating this app")) (&DeploymentLock{}).InitSchema(sb.Builder("app.deployment_lock")) + sb.Time("disabled_at", "dev.miren.core/app.disabled_at", schema.Doc("When the app was disabled with `miren app disable`. While set, every service is held at zero instances, nothing boots for a deploy, and requests get a 503. The zero time means the app is not disabled.")) + sb.String("disabled_reason", "dev.miren.core/app.disabled_reason", schema.Doc("Why the app was disabled, as given to `miren app disable`")) sb.Ref("initial_config", "dev.miren.core/app.initial_config", schema.Doc("Reference to the initial ConfigVersion entity created before the first deploy")) sb.Ref("project", "dev.miren.core/app.project", schema.Doc("The project that the app belongs to")) sb.String("workload_role", "dev.miren.core/app.workload_role", schema.Doc("The authorization role that identity tokens minted for this app's sandboxes authenticate as (see pkg/workloadroles). Empty means the default (app-readonly). Cluster-scoped roles may only be set by an operator, never via app.toml.")) @@ -3843,5 +3867,5 @@ func init() { (&Secret{}).InitSchema(sb) (&SecretVersion{}).InitSchema(sb) }) - schema.RegisterEncodedSchema("dev.miren.core", "v1alpha", []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xdc\\۲\xec6\xd1~\x8d??\t\x10\xce\x04\xe2$\x84p&\xc5\xe1\x82*nx\x04\x97\xc6\xd6x\xb4ƶ\xbce\xcd\xec5\xb9ۜ)\n\xaax\x05\xf6\u07b9\xe1\xf9\xe0\x9a\xb2Z\x92\xa5\xb6lI\x9eE\x15\xc5\xcd*\xb5\xac\xfet\xean\xb5Z\xbd\xe6Uݓ\x8e\xf65\xbd\x16\x1d\x13\xb4/*.(=\xb3\xbe\x1e\xff\xf1ܯ}o\xaa-\xc80|\xaax\x04\xfaJ\x86\x01\xf8\xfeu\xacyGX\x8f@\x8fGF\xdbz\xfc\xfd\xcb\x03\xab\x1f\xbf\xb4d.H%ٕ\x965\x1dZ~\xebh/U7ϖ\xd5\xf26\xd0\x03\xab\x15\xd0\xdb\xeb@W*F\xc6{\x98 \xaa\xd3\x10\xaf&\x88/\x04 \xe6\xdeʖWg\x85\xc1q\xe5\x04\xc2*\xde\r\xbc\xa7\xbd\x9cK\xb0>\x18\xb7\b\xe0\xa6,\xd8o\xd4<\xdfA\x83D@\x05\xa9\x9e]\x98\xa0uI`\xd9\xcen\xc54\xd0Z\xb2\x8e*\xa8oF\xa0\x1c\x9a\xd5\n\xac\xf3\xab&\xb8\xe3(\x05\xeb\x1b\x05\xf8\xf5\b }\x1c\x98\xa0\xa3\x19ڃCۑ5zg\x9a\xeb\a\xa4\x1dN\xa4\x1d\x04눸\x95\xd3\n\xf53\xe2\x04\xb8\xba\xf1\xacg\x92\x91\xb6\xacx\x7fd\rl<\xaase\xe7\xff\x03\x10\x83\xe0\x0f\xb4\x82\x816\x86p\x99>\x1f`z\xceŹ\xe5\xa4.\x05o)\xac\x98_\xe5\xac\xd8\xe6D+2\f\x8ba)\xb5\x1bi%\xa8\x96\xac\vj\x00\xdfRd\xe9\xd7j\n_\x0e\xf2\x17\xd5E\x88i\xcf\\\xbd\xe1\xb8rk\xfd4\xce@\xe4I1ת\x94:\xf7#\xb0?~&4}\xbd\x130\xff+j\xa1?\xa6,\xc0\v5\xf07\xc3\x00\x05\x7f\xdeS\xa1\xba\xa0PL\x1d\xbb\x91\x94\x052\x98L!ّ\x98\xd1c\xabj\xbe\xa6\f\xff\xb7j\xf8x\x85\f\xc2daT\x17\x93\x18m\v\xad\xe1\xe8Hώt\x04y?Y\nk\xf8Wc\xfce\xcd\x1a\x03\xc3q\xa5\x83\xa6\f\xeeg\xd7\xd0FI\xe4eT G]V\x06\x82\xf6\x97\xee<\xfd)\xaf\xa4\xbd\xd0\xf1\xefG0\xe7\x8b\xe5\x06&}\x00\x9c\x88\xa8N\xecJ\x97\x1d\x9af\xfa\xfb\xe6֞\xcc\xe8\xc2{\xdbQIj\"Ixo\xcd\xd7$;\x1f\\\x1b\x83P\xb4\xe4@۱\xeeH\x7f\xfb'\xac\x90\xae\x99V\x88\xaarP\xb6-\x80\xd2\xc8\xf9\x0f\xde\xe2ϭ\xf1mY\xc4\xed\x953\x10\x8bI\xa9\x95\x9bm\xba>/\xdfX=FR\x96\xefoj\x16o\xadb\xac+\a>\xff}\x9e\xd2.\xd9\xc9Rx\xed\xb0AE\b\xaeA=\xbb\x15\x18\a{E\x0eN\xd5^FI\x859\x91\x1f\x1c\x1a\xa3|e\x03\x85wCK\xe5\xec&\xb4^\rVԍyA\x91\xd6\xe5\xe1\x06\xf3r+\"\x8e\x11\x82-°)\x9b\xfe\xa7\x8d\t+\x90\x82\\\xe4\xa9\xec\xa8<\xf1Z\xaf\xbfS\x81W\x0e\x9bI\x17\x88v\x84\xb5p6@1\xa6@.sD\xf7\xc2\xce\x1d\xb0rѐ\x9e}B$\xe3\xbd\xd9}\x8e+1`X\xa4\x01p\xbc\x1cf}6D\x9a$\x02\xc0䮍\x92t\xa0Nl&\xd3\xd4\x02@.#\x15弦\x0f\x0e\x9d3\x19ŦW\xa51D\xced\x14\x8f\xdd\x196\x93\xa9G\xff\x83A;܂G\xa5#\xdcT\b.ʎ\x8e#i\xb4\x9b\xe8Wa\xfd۰L\r\x93%\xeb\x8f\x1c,\x93\xa52\xaf$\x01\xc0t\xb5\xc3V\xdd (\x9d\xe3\xe0G\x1du\x19o\xc9*\xefA\x90\xbe\x02\xf7\xf1\xa8˘\xf7\x1bk\xbc\x15\xef:&K\xe8\xd2\x11\xae1\xf4\x01\xa3~-\x82\xeaK\xfd\xb0\xa8\x8d\xd9\x12\x8b\xc7ƲfB\x82\xd9F\xe8!e\xa4\x7f\x0e\x06\x15\x14{Q\x1f\xca#\xd3\x01\xcf\xc6\x10\xd1k\x91b\x9d\x9a\x8e\xb7Q\xd2\x0e\x04\xc0\xa1\xa3\xee\xbf\x02h)\x19\xa9r\xbf\xf8\x05\x04\xa1\xf3\xab\xd2\xc6\xd1\xf1K/K\x1b\xb1|ph\f\xb0\bO(\x80\xc8\xcd\x0eK/0m\xc5\x19\x83\x92\x01l\x83\xe0WVS\xa1\xe2a\v\xf39U\xaa}-m;%\xe2\x96\n\x86\xd2^ҖW\xa4]\xf4h\xb8\n\xf5\x99\xaa/\xeb\x8dT\xddq|\xd62I\x17\xeag[\xc1\xf7\xe0y\x03\x13\x14\x94\xd4%\xef[\xf0N\xd9L\xfa\xceqX\x1aG\xf6\t-\x9b\x83\xb6Z\x9a0v!\xe8\xd2j>\xe5\xb6h\x97\x01ʱ\xd3Q3\xaa\xe9\x18\xf7\x87\xcdd\xaa\xa9\x04\xb5~\x1d\x1a\x9aUT\xda_\x1dCPMd\xc4\f\x14\x19f\x80\xf6\xd7\x14#\xf0\x87\xa0,\xd3\xfeZ\x1cH5\xb9\x13\xb0膈-\xdf\xc4Xӱ\x12l\xb0\x8e\xf5٭@\x008\xde?\xf1\x9f)\bI5\x15bw\x97\x89APx\x0f\x03\xa5\xb0ԶdM\x8c#\xedG&\xd9UG\bf\xd2g\xc5\xe6A\xb1\xc6E\xeb\xff\x02lJ3\xc1<@1\xf9\xf9\x88\xf6\xd7`\x18\xd4n8\xeb\xcc]\x88B\x11\x8fg\x11Q7\x9c\x11C\xb7ʷrP\a]q\x8fɉ\xca\xccd̙\xf7\x11\xd4\x0e\xcd\bp\x8b\x88\x98Z\x0f\xc1;\x84\xa1\xe2\t\x0fa\x05\x98\xa2\x7f\xbf\vJ\x98b\x8f\xed\xcb\xc2\x18\x03\x13\xafiiw\x86ͤ\xb7=\xe1\x0e\xb7<\xaf\xc5\x01\x01\x1c\x82K^\xf1v\xe3\xdcR\xfbcۙs\v\xa8\xf0\x13P%\xaba\xa1膧\x90\xd5P]\xea\x8d\x06\x97zؘ\xa3\x15\x9c\x1a\xcb̶AWܯB\xb7y\xbb\xe9\xbaPV\xbc\x87\xb7\xd5\n\f\xd9\x18\xfa\x10\x11\xb6\x8f3\x84-\x00\x9f.z8\xa8\x19\x00+:^\xeb5S%,\x88\xef'@L\xdb\xcb\xfaQ\x92~rɕS\xe7Wy\xe2\xf9\xbd\x04\xc4\xc9\xce\xd3Q\x8e\xe5@\x85ŁG\xf4\xf0'\xaf\x87\x0f\x13z\x18'\xf7\xa8\xac\xf9\xf3\xbe\xaciK`3\x87E-^\x8e$\xe8\xd3E*\b\xd7\xcd\x1d\x16\xb5\xa9\xd2)t\x1fN\x17\xdb79#;\xc1`\xbe\x91/I\x84,k&heÊ\x1cWb\x9b\xbbrA\xbb\x12\xc1ȡ\xa5\xee\x05\xcd\xd6\xdd\x7fA3P\xe9\x1e\x0f\xbe3\x18\x844\xb7\a\x87G,w\x8eﳰ`\x16e\xd3\x03\xc2\xf7n˕\xe4\x06a\x13m\xb9\x13}!\xbc\xbf3\x7f\xdc!\xc2G\x88\xe5\xdd\xed\x15Y\x19\xdaN?\x01)\tF\xb1\x97\xc2\xe4'\x97\xa1:7F\xf6\xde\x06\x14\x1dN\xb4\xa3\x82\xb4%ʐ\x92\xc1/~\x10\x14\xbf\x19\x86\x81U^\x00(%\xae\x8c=\x80\x84\x01\xa5l\xf5˕W\x15\v\xf3\xba`\xca\xf1,/\x02\x80\xd8L\xc6\x14\xc8\x05IL\\\xd9Z\xa5]\xb9+[A\xcb\x15\xe9\xde\x19\xb4\x04\x84\xf43\x1a\xab<\xf0\xabw=\xe7Q\xed\xc1\xa1c\xaa\xe7 \x98G\x9f\xc6\x10\xb1@\x87\xe6\x9dF\x0fN\x81*\xc5̔\xe6\xba\xe3\x99J#\xec6\x16z\xe3\xa2\xc23J\"YUzan\x8e+c\xa1,\x17p\xf5\xed!e\xd4nV\xc9bQ\xd5\x03ę\xdeJ\xc1\xa5z\xba\xd2·H\xb4s\x9a\xa4\x8b\x1d\xb6\x95.J\u058bw\xd0nxhG\xc1\xbb\xd2\x1c{'K\xc5\xd2\x0e<\fA\x9f\v2\f\xfa\x00d3i\x1c\xbf\xe0û\a\x91\x9a\xa5\xf6\xaa\xaey\xbf|\xe8\xd1\xc9gӷ㑰\x96\x06.\xcf\xd0\x04\xbe\x9e\x04\x95l\x9a\xdcZ\x1a\x9b\xf9\xfe\xa0'35](\x96ijZ\x04\xe5ћ\xa6\xe4v\xad\x8f\xba\x9c*\x8e\xad\v\x14\x96G\xce\xea\xaa<\xb0\xbef}\xb3\"\x8fn\x93\xf4\xccr쳹(\xc1\xa7\xacס\x9ch\x8f\xabj\t\xeb&ϼfӄܠ\xc0\xb0\xf8\x16\xb1\xf8\xa8\xa3b\xb3\xa3\xf4\x97 |Tb\xa4mo\x11_\xef\x16\xdc\x03\x91\x92\nm\x9c\f\x91*\r\\\xc1\xcdh\xc1.\xbdu\xc8r\x90\xb1\x10{Hl\x1c/:\x10~\xd4\xe5\x98\xc9\xf1\xf87B\xe9[o\xf7\x1e\x86α*\xddE\xe4\xb82Y\xb5\\\xe8\xa5\x02O\xaa\xe5{\xa3Z\xee\xf0\xd9\xee7J\x174\xec\xff\xf88A\x05\vZT\xc47\x0e\xb4\x02?A\x95\"J\x84\xb7\xcc~7s\x9f@\xd2\xdfᾟ\x02WT-\xefiI&\xa3^\xaa\x1bE\xef\x85\ae\xf0{d\"\xa8\xe7\"\xbd\xe7\xf4\x1d\xfbx\xf7\xecb\xb1ş\xeeG\xd6\x05\xed\xedh\"U\a\x1e\xdf\x04\xfc\t^\xa1+p\x83\xad\xc4\xed\x83ܡ\xb9[\xd9z\xf5x\xd6\xf8F\x17\x86N{\xe1V\xe2\xf7n\x12\xe0=\xaf\xd7\xef&\x89Y\xce\x03\xf6_\xd4*\x7f+k\xe4\x05\x11\x8d;\xfcZ\xd1xy?\xcaÌe:\xbcJWp\a\xd3\x0f͞3B\xb2i\x1a\x1d\xea*\xdd\xe5N\xd3\xe8P\x1f\xb1 \xed/\xf6#\xe7\xc5n\x7f\xb5\xbf\xa3\xfbB\xba\xbf\xdc\xdf\xf1\xceH\xef==>m\x00\xf8\xf1\xad\xd9n\x9a\xfe\x9c\xee^\x87\xe2ґ\xd1\xee\xcbT\xf90OI2\x93U~\xb8c\ni\xb9,\x99\x8a\x97\x9d\xea\x92v\xa0\"\xfc\xecL\x98]\xb3\xc8H\x94\xc1\xef2I\xf8\x11W#ӂ\xc7\xd3l>\x9dP\x7f\xb4\a\xf5\x7f#\vG\xa8\xde[ڐ\xea\xe6\x0f\xe2\xc5\xe3\xcf\xeeY\x16w\xa8\xf7B\xb9\x13z\xf1\xf8\U000fbc1c\xe4\xa3\x1f\xef\x01J\xccM\xdae|\xe2\xa9K?\xd8\x05\x1b\x7fmٵ\x14w'>=_\x9eCs*T\x9a\xeb~G\x82\xd4\ay\xa7OV\x8eT\xa6\x9dJN\xa1\xcaܧ\xdc\f\xab\\\xaf;\x9a\x81\x95)\xaf\xc9\tZ\x99ꕑ\xbf\x95yj%\xa6w}7\x1fu\xf7\xd3\xc5e\xa9U&\x1f,ӫ\xdb\xca\x12{\xb5C\\:*\x05\xab\xe0V\xd0\x18\"\xa2\xa5\x1f\xe5i\xa9FM\xbfDej\x94\xc6/h\xaf\xb2\x11`*\x86\xf0%)ӿ2Ȭ\x97T\\\x89\xce~\xb2ԝ\x9ae\xd0\xd7\x7f\x94\xe1.ԍ\xe4\xbeL\xe7\xcaB^\x0e-\x83X\xe0Q\x97\xed\xf2n\xdfn\xdeXʿ\x06}\xb9#X\x11qH3Ѷ\x16*\xd3F$\xe7Ff\xee)\xa4H:\u05c8֫\xb9Ӳ%\xe7cf\x9a\xaa}Y\x9a\x99\x17\xd0\xccD\xcḍ$)\x8f3\xd3\\\xe5\xa4y\xee\x1a\xee\x8a@\xef\xb9V\xfdW'\x89\x06\xeeI\n\xf5\xc5\xe3O\ue665\xe9\xfb^\x18\x93ƺk\x0fwf\xb9\x06\xbcw\x85\xb7\x9d|\xb8dR\x03\xffv\xda\xc03\xb3\f3\x02\xf5:g\xa3fpQ\x7fph\f\xf8\x9d\x1c@\xc8s\x18\x8c#\xf5lY\x8d-\x1fN7\t\xc3K\x82\"nP\x91\x99\x1e\xb9\x81\x9db\xe7\xfe\x98q\x06*\xd0h\x9c^\xad\x00\xfe\xb7\xe3-\xc0\xfb\xff1&\x82\x9e~\xe9K;h-p\xde\xcd,g\x91\xa3ײ4\xbf}\x06\xe3\x825\f\x06y\xd4\xe5}~\xc0\f\x99t\xcdK3g3h\xe2\x1d/w\xfa\t\x17\xbc4\xfb5C\xee\xbe\xdd9\x06\x14\xd4T_\xed\xd2\"\x0e0\x80\x8e<\xce\t\xe6p|\xf7\xa8\xce\xf3\f\xd2,(`?\x89\xbf\fPb\xf2\xda\xf5#Rc\x88\xcd\xc4\xfc-\xb4\xb1:\xd1\xfa\xa2C\xfb'K\xdd3@\xd7Kn\xd6\x1c\xe4,@\xc1\x9aF\x87\x8a\x1bC\xa4\xe7\xd7\xf8\x92\x91aJMf\xb4{\xa0\xb0\xb9\xf2I\f\xaa\x85K7\xa8i\a\xad\x05N\v\xa1\xa5Y\xe9\x19\xf4?`\xa5g\xf0M+\x9dfRg\xb0'4\xa93蓚T\a\xf6\xa9L\xea\f\xb9ۤ\x8e\xb3\xe2X\xb4\xed4[g\x00\xdb?X\xea\xa77\x85\xf3\xb4\xe0\xf7.#yZ~\xa3\xf44H\x9c\xcd\xef\xe3\x14\x15\x1bNTH\xfa\xa8\xd3S\x1czZ=z\xb8I\xed\xa0\xe3\\+\x04t\xa6g\xf3\x10p\xd4\xe5X\x82\x1dB\x00RK\x04\x94\xdd\xc41\x9cو\xb9%\x91z\xf3\xa1\x18~_3\xa1\xba`\xaa-5Q\xbdS\xcdFh\xb6\xd8\x06\xd5\xcc|f5\x1d\xa5\u0dd5\x9f\x96T\xff\xe1\x03\xdfW\xfe{Λ\x82\x1abّʘ_C\xe2\x85ĩ\xd6\bGg2\x975\x85\xdf+>\xbb\x15\xf3\xaen\v\xae\x8f\x89۞\xc7\xd3t\a\x87\x9f\x87\xae\xc80\xac\xfdD\xb4\xf91\u05cd߲\xb5\xbf\x99\xba\xf5\x83\xaf\x91_\xdf4_矚\xdc\xfc\x91N\xf7gz\"\xbfI\xe9e\xd4\xc7~\xd2'!\xdf\xd9m\xe1\xe7pFӣ\x93\x8c\x89\xdf\x06mc\x8a\xfd\xf97\x00\x00\x00\xff\xff\x01\x00\x00\xff\xff\xf2A@\x85\t\\\x00\x00")) + schema.RegisterEncodedSchema("dev.miren.core", "v1alpha", []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xdc\\۲\xec6\xd1~\x8d??\t\x10\xce\x042I\b\xe1L\x8a\xc3\x05U\xdc\xf0\b.\x8d\xad\xf1h\x8dmyK\x9aYk\xe5ns\xa6(\xa8\xe2\x15\xd8k\xe7\x86\xe7\x83k\xca:Yjk\xac\x96gQEq3\xa5\x96ݟNݭV\xab\xc7O\xcd@z:4\xf4\xb2뙠î\xe6\x82\xd2\x13\x1b\x1a\xf9\x8f\xfb\xb8\xf6\xbd\xa9vG\xc6\xf1S\xcd#\xc0S2\x8e\x86\xef_\x87\x86\xf7\x84\r\x00\xf4p`\xb4k\xe4\x1f_\xedY\xf3\xf0\xa5%\xf3\x8eԊ]h\xd5б\xe3\x8f=\x1d\x94n\xe6ŲZ=\x8et\xcf\x1a\r\xf4\xf6u\xa0\v\x15\x92\xf1\xc1\f\x10\xd4Y\x88\xa7\t\xe2\v\t\x88\xb9\xb5\xaa\xe3\xf5IcpX9\x81\xb0\x9a\xf7#\x1f\xe8\xa0撙\x1f\x88\xbbK\xe0b&\xec7z\x9c\xef\x80N\x02\xa0\x1d\xa9_\x9c\x99\xa0ME̴\x9d\u008a\xa9\xa3\x8db=\xd5P\xdf\xcc@\x054k4X\x1fWMp\a\xa9\x04\x1bZ\r\xf8\xf5\f }\x18\x99\xa0\xd2u\xed.\xa0}\xcfZ\xbb2\xed\xe5\x03ҍGҍ\x82\xf5D\xb2\v]6\xe8^\xb3\xcfW\x97\xf6\xe8z\x97^۞*\xd2\x10E\xd2k랢\xf6\x9b\xe4\xdc8\x84]G\xf6\xb4\x93MO\x86\xc7\x7f\x9a\x19\xb25\xd3\fQ]Nʶ\a\xd0\x1a9\xff\xc0%\xfe\xdc5\xbe5\x8b\xb8>s\x0eb1(=s\xf3\xdeb\xf7\xed7\xaeng\x98\xe9\xfb\x9b\x1e\xc5[W1\xae+\aܒb\x9e\xcaO\xd9\xd1Sp\xee\xa0A\x05\b\xa1A=\x85\x15\x10\azg\x01Nݝ\xa5\xa2\xc2y\x06w\x01\rQ\xbe\xb2\x82\xc2\xfb\xb1\xa3jީ\xbb\xa8\x06*\xeaʸL\x916\xd5\xfe\xd1n\xf9AE\xc6A\x03\xb0\xbb4,f\xd1\xff\xb42`\r\xb2#gu\xacz\xaa\x8e\xbc\xb1\xf3\x1fT\xc0\x99\x83f2\x04\xa2=a\x9d\xd9\x1bL1\xa7@!sF\xf7\xd2N\xa6a\xe5\xa2%\x03\xfb\x84(\xc6\a\xb7\xfa\x1cVB\xc0\xb4H\x1b@y\xde\xcf\xfa\xec\b\x9c$\x1a\x80ɗ\x93\x8a\xf4F\x9d\xd8L\xe2\xd4\u0080\x9c%\x15\xd5<\xa7w\x01]2\x18\xcdfg\xa5uD\xc9`4\x8f_\x196\x93ح\xffΡ\xed\x1f\x93[e \xdcT\b.\xaa\x9eJIZ\xeb&\xc6UP\xffV,S\xcbTņ\x037\x96\xc9S\x85G\xa3\x04 ^\xed\xa0Uw\bZ\xe7\xb8\xf1\xa3\x0e\xb6\f\x97\xe4*\xef^\x90\xa16\xee\xe3\xc1\x96!\xef7\xae\xf1ּ\uf66aL\x93\x81p\xc9\xd4\x03\x88\xfa\xb5\fj,\xf5\xe3\xa26gK<\x1e\x93UÄ2f\xf3\xe8)\xed\xe8\xec9\xef\x92\xc6\xc4s\x87\xd2\xd3&\xe4&\xa91\x9e[БK\xa6\xb80\xad\xdf\x054ĀN\xa6ǐGb\xf6ѩ\x90;\x92z\xae\xe9<Ć\xb6R\x82\xd2\xeaH\xa4Y\xe2\x17\xcbj\xb4\xcb\xdd25!'\x0f\x83\x81\\\xa3&\xec\x8b\xd7\xf9\xf9H\x85\xb6\xb3\xc6@\xccd\xee@\x1ab\x9cUͭ\x89i\x1d\x91\x13\xeb\x80\x7f$\xfa\f\x06#4\xcb\xeaЭI\xefG\x06\xefH\xa4\xe9\r5Eؗ\xddu^\xc9Ϣ\x0e\xc3B\xce\xfc\xaa\xe4\x93\x02\x17G*\"\x02\xd7\xe4.\xa0\xe3\x18Bz\x9bv\x18\xa9\x13\x04r\x9dBG\xad\x05'^\xcc&0\x81,\xe6\xddE\xef\x9c\xd7gMq\"\xf8\xe0\xde\xc0\x98\xe0\xbf&=\x9f\x00dG\x9a\x9e\r\x95\xe2'\xea<Ϡ\"g\x8f#\xa0k\x8e3Ԛ\x88ɞ\x9a\xac\xe7\xec\xa80\xfe\x97\x88\xe0x\xf6 \x82s\b\"7+[\xdb\xdb˨\x1f@\xc3L\xeb\xef_\xa7f\xc3\xf0띀\fMx\b;\xfa\xbaL\xf7\xde\xc9v\xcf\xc3\xe3\x039P\xd4\x1c\x82\x832\x82순\xc3\xea\xb9%\x15\x17V[s\xe5\b\xac]\xf63\x92TU;T:(\xf18r6\xb8\xc8\xe4L\xc3^B=\xb1\b#\x17\xca\x06\x99\xa6\xd2\xc4U\xb3A\xad-\x9f\x1dI\xb4|\xbe\xee\xf6\xe5sP(\a\xeau\xea\xb0\xea\x10v\r\x93\xa7\xb0\x9b\xd4Td\xfa\xf8>\xbe\x8f\xa6\x05LO\xff\x9c\f*h\xf6]\xb3\xaf\x0e\xcc\x06<[Gd\x8fE\x9auzU>JE{#\x00\x01\x9du\xff5@G\x89\xa4\xda\xfd\xe2g#\b}\\\x85\xebG\xcfσ\xaa|\xc4\xf2.\xa0!\xc0\"<\xa1\x012';(\xbd\x86i-Θ\x94\f\xc36\n~a\r\x15:\x1e\xb60\x9fS\xa5^\xd7ʿ\xa7E\xdcS\xc9P\xda+\xda\xf1\x9at\x8b\x16\x1d\xd7N?\xa6\xfa\xc9\xf5\x97t\xddA\xbe蘢\v\xf5\xf3o\x99\xe7\xc9\xfd\xc6\fPP\xd2T|\xe8\x8cw\xcaf2v\x8e\xd3\xd2(\xd9'\xb4j\xf7\xd6jY\xc2م\xa4Kk\xf9\xb4\xdbb]\x06S\xce펖Q\x0fǹ?l&\xb1\xa6Ҩ\xf5\xebT\u05fc\xa2\xd2\xe1\x12\x18\x82z\"3f`W`\x06\xe8p\xc1\x18\x81?$e\x99\x0e\x97ݞԓ;a&\xdd\x11\xb9\xe9\x9b\x18\x1b*k\xc1F\xefX\x9f\xc2\n\x00\x00\xe3\xfd\x13\xff\x89\x1a!\xa9\xa7B\xee\xec21\bj\xee\xe5\x8cRxj]\xb2&FI\a\xc9\x14\xbb\xd8\b\xc1LƬ\xd0\xa2\xc3%\x19\x06\xf5\v\xcezw\x16\xa2\xa6\b\xfb\xb3\x88\xa8;Ό\xa1\xbb\xcawe\xa3N\xba\xe2\x11S\x10\x95\x99ɜ3\x1f#\xe8\x15\x9a\x11\xcc)\"cj#\x84h\x136\x15ϸ\tk@\x8c\xfe\xfd.)a\x9a=\xb7.\vcl\x98xC+\xbf2l&\xa3\xe5I7\xb8\xe6y-6\b\xc3!\xb8\xe25\xefV\xf6-\xbd>\xfe=\xb7o\x19*}\x05T\xabz\\(\xba\xe3٩z\xac\xcf\xcd\xca\v\xe7f\\\x19\xa3\x17\x9c\x06\xca̺A\xd7\xdcO\xa9Ӽ_t[\xa8j>\x98\xbb\xd5\xda\x182\x99z\x90\x11\xb6\x8f\v\x84-\x01\x8f\x17=\x18\xd4L\x80\xedz\xde\xd89\xd3%(\x88\xef# \xa6\xe5e\x83Td\x98\\r\xed\xd4\xc5U\x91x~\x0f\x818\xd9y*\x95\xacF*<\x8e\xb9DO?\x8aZ\xf8\x10т\x9cܣ\xaa\xe1\xf7C\xd5Ў\x98\xc5\x1c\x17\xb5p:P\xd0dz\xd2\x10\xa1\x9b;.j\xb1\xd2)l\x1bA\x13\xeb'9';\xc9`\xbe\x93/E\x84\xaa\x1a&h\xedÊ\x1cVB\x9b{\xe5\x80v!\x82霏\xe0\x80\xe6\xebn?\xa09(\xbc\xc7\x03\xcf\f\x0e\x01\xe7\xf6\xc0\xf0\x88\xe7.\xf1}\x16\x16̣\xacz@\xf0\xdc\xed\xb9Pn\x104ў\x1b\xe9\v\xc1\xf5\x9d\xf9\xf3\x0e\x11\xdcB<\xeff\xaf\xc8\xcb\xd0z\xfa\x89\x91\x92d\x14{)Lq\x92\x1b\xa8\vcd\xef\xad@\xd1\xf1H{*HW\x81L-\x95|\x12\aA\xe1\x9da\x1aX\xe7\x05\x18\xa5\x84\x95\xb9\v\x904\xa0R\x9d\xbd\xb9\x8a\xaara\xde\x10L;\x9e\xd5Y\x18 6\x939\x05\nA\x90\x89+k\xb3\xb4)we-hyE\xba7\x06-\r\x02~\x8f\x86*o\xf8\xf5\xbd^p\xa9v\x17\xd09\xd5\v\x10ܥO\xeb\x88\\\xa0\xc3\xf2N\xbd7N\x81.\xe5̔\xe5\xba\xe1\x9a\xca\"l6\x16v\xe1\xb2\xc2#\x15Q\xac\xae\xa207\x87\x95\xb9PV\bx\xf5\xee\x01\xd3\xeb0\xabd1\xa9\xfa\x02\xe2D\x1f+\xc1\x95\xbe\xba\xb2·H\xb4\v^\xc1\x8b\x1d\xb4\x95!Jэw\xd2nDh\a\xc1\xfb\xcam{GO\xe5\xd2\x0e\"\fA\xef\x05\x19G\xbb\x01\xb2\x99t\x8e_\xf2\xe2=\x82\xc0f\xa9=5\r\x1f\x96\x17=6\xf9lzv8\x10\xd6\xd1\xc4\xe1ټb\x9e\x1e\x05Ul\x1aܵ46\xf7\xfc\xce\x0efzu\xa1X\xeeU\xf7FR\x1e\xa3a*\xee\xe7\xfa`\xcbXq\xecB\xa0\xb4hh\xb7\xda\x10\xfe&\bn\x95\x10i\xdd[\x84ǻ\x05\xf7H\x94\xa2\xc2\x1a'G`\xa5\x81k\xb8\x19-\xd9d4\x0fE\x0e2\x14\xe2\b\x89Iy\xb6\x81\xf0\x83-\xe7LNĿ\x12J_\xbb\xbb\x8f0l\x8eU\x15N\"\x87\x95h\xd5\n\xa1\x97\n<\xa9V\xec\x8dZ\xb9\x83{{\xfc\x12^Р\xff\x13\xe3$\x15,iQ\x01\x9f\x1cim\xfc\x04]\xca(\x11\\2\xff܍}\x02\xc1\xdf\xc3}\x1f\x03\xb7\xab;>ЊLF\xbd\xd2'\x8a!\n\x0f\xaa\xe4\xf3\xcc@@\xcb;|\xcb\xf8\x15\xfbx\xf3\xe8r\xb1şnG\xb6\x05\xeb\xedX\x02\xab\x03\x0fo\x1a\xfc\t^\xa3kp\x87\xad\xc5\xed\x83Ү\x85K\xd9E\xf5p\xd4\xf0D\x97\x86\xc6\xddpk\xf1{\x17\x05x\xcb\xed\xf5\xbb(1+\xb9\xc0\xfe\x8b\x9e\xe5o\x15\xf5|GD\x1bv\xbf\xd14\x9cޏ\xca0s\x99\x0eOx\x05\x0f0\xe3\xd0\xec\xa9 $\x8b\xd3\xe8TSx\x97\x1b\xa7ѩ6rA\xda_lG.\x8b\xdd\xfej{C\xb7\x85t\x7f\xb9\xbdፑ\xde[Z|\xde\x00\xf0\xc3[\xb3\xddt\xed\x05ͽNť3\xbdݖ\xa9\xf2a\x99\x92\x14&\xab\xfcp\xc3\x10p\xb9,\x85\x8aW\x9c\xea\x82\xdbP\x01~q&̦Q\x14$\xca\xc0{\x19\x14~\xc6\xd5(\xb4\xe0\xf94\x9bO'\xd4\x1fmA\xfd\xdf\xc8\xc2\x11\xba\xf5\x8e\xb6\xa4~\x8c;\xf1\xf2\xe1g\xb7LK\xd8\xd5[\xa1\xc2\x01\xbd|\xf8\xf9MXA\xf2я\xb7\x00!s\x936\x19\x9f|\xea\xd2\x0f6\xc1\xe6o[6M\xc5͉O\xf7\xcb}hN\x85¹\xee7$H}P\xb6\xfb\x14\xe5H\x15\xda)t\nU\xe1:\x95fX\x95z\xdd\xd9\f\xacByE'h\x15\xaaWA\xfeVᮅL\xef\xfan9\xea櫋\xf3R\xab\\>X\xa1W\xb7\x96%\xf6\xb4A\\z\xaa\x04\xabͩ\xa0uDFK?*\xd3R\x8b\x8a?D\x15j\x94\xc5\xdf\xd1Ag#\x98\xa18\"\x96\xa4B\xff\xca!\xb3AQq!6\xfb\xc9S7j\x96C\xbf\xfeQ\x86\x9bPW\x92\xfb\n\x9d+\x0fy\xdew\xcc\xc4\x02\x0f\xb6\xec\xa7w\xfdt\xf3\xc6R\xfe-\xe8\xab\r\xc1\x8a\x8cCZ\x88\xb66Q\x856\x02\x9d\x1bY\xb8\xa6&E28FtQ͍\x96\r\x9d\x8fYh\xaa\xb6ei\x16\x1e@\v\x135\v\xb7\x12T\x1eg\xa1\xb9*I\xf3\xdc\xd4\xdd+\x02\xbd\xe5X\xf5_\x9d$\x9a8'iԗ\x0f?\xb9e\x94\xae\xed[a\\\x1a\xeb\xa65ܘ\xe5\x9a\xf0\xde5\xdez\xf2\xe1\x92Iw\xfc۸\x8e\x17f\x19\x16\x04\xeam\xceF\xc3\xccA\xfd.\xa0!\xe0wJ\x00M\x9e\xc3\xe8\x1c\xa9\x17\xcbjh\xf9`\xbaI\x1a^\x11\x10q3\x15\x85\xe9\x91+\xd8\xf8\x8f\xc9\xe1\xf6@\r\x9a\x8d\xd3\xeb\x19\x80\x7f;^\x03\xbc\xfd\x8f1\x19t\xfc\xa1\x0f\xb7\xd1zಓY\xc9$g\x8fe8\xbf}\x06サ\xcct\xf2`\xcb\xdb\xfc\x80\x19\x12u\xccÙ\xb3\x19\x14y\xc6+\x1d>\u2007\xb3_3\xe4\xe6\xd3]`@\x8d\x9aڣ\x1d.\xe2`:Г\x879\xc1\xdcl\xdf\x03\xa8\x8b<\x03\x9c\x055\xd8\xcf\xe2/\x1b(1y\xed\xf6\x12\xa9u\xc4jb\xfe\x1a\x9a\xac\x8f\xb49\xdb\xd0\xfe\xd1S\xb7t0\xf4\x92\xdbk\x0er\x11\xa0`mkCŭ#\xf0\xf95\xb1d\x14\x98R\x97\x19\x1dn(l\xae|\x16\x83\xea\xe1\xf0\x06\x15\xb7\xd1z`\\\b\rg\xa5g\xd0\xff\x80\x95\x9e\xc1W\xad4Τ\xce`\xcfhRg\xd0g5\xa9\x01\xecs\x99\xd4\x19r\xb3I\x95\xb3\xe2x\xb4\xf54۠\x03\xeb\x1fN\x8dӛ\xd2yZ\xe6{\x97\x99<\xad\xf8%|\x1a$\xcc\xe6\x8fqv5\x1b\x8fT(\xfa`\xd3S\x02z\x9a=\xba\x7fT\xd6A\x87\xb9V\x00\xe8DO\xee\"\xe0`˹\x04;\x80`H+\x11\xa6\x1c&\x8e\xc1\xccFȭ\x88\xb2\x8bo\x8a\xe9\xfb5\x17\xaaK\xa6\xdaR\x17\xd5;\xba\x8f\xce.\x97A\xbf\xe6\x1e\xb3\x86J%\xf8\xe3\x95OK\xea\x7f\xf8\x98\xe7W\xfe=\x17\rAw\xb1\xeaI\xed̯#\xe1D\xc2Tk\x80c3\x99\xab\x86\x9a\xef&\x9f\u008ayU\xd7\x057Ƅ\xef\x9e\xe4q:\x83\x9b\xcfT\xd7d\x1c\xaf}\xaa\xda}\xccu\xe5[\xb6\xfe\x9b\xa9k\x1f|\xcd|}\xd3=\x9d?5\xb9\xfa\x91\xce\xf03=\x99oRF\x19\xf5\xb9O\xfa \xf2\x9d\xc37\xe2\x1c\xcelz4ʘ\xc4\xef\x80e\xc4؟\x7f\x03\x00\x00\xff\xff\x01\x00\x00\xff\xffo{V\x0e\x91\\\x00\x00")) } diff --git a/api/core/schema.yml b/api/core/schema.yml index 5901572ac..6da041769 100644 --- a/api/core/schema.yml +++ b/api/core/schema.yml @@ -105,6 +105,17 @@ kinds: type: ref doc: Reference to the initial ConfigVersion entity created before the first deploy + disabled_at: + type: time + doc: >- + When the app was disabled with `miren app disable`. While set, every + service is held at zero instances, nothing boots for a deploy, and + requests get a 503. The zero time means the app is not disabled. + + disabled_reason: + type: string + doc: Why the app was disabled, as given to `miren app disable` + workload_role: type: string doc: >- diff --git a/pkg/apphealth/apphealth.go b/pkg/apphealth/apphealth.go index f9e0fc5d2..cf99c316d 100644 --- a/pkg/apphealth/apphealth.go +++ b/pkg/apphealth/apphealth.go @@ -15,6 +15,9 @@ const ( Crashed = "crashed" // Idle means the app is deliberately scaled to zero (no desired instances). Idle = "idle" + // Disabled means the app was disabled with `miren app disable`: every + // service is held at zero until `miren app enable`, whatever its config. + Disabled = "disabled" // Ready means the app is deployed and available to invoke, but has no // long-running process to be healthy or idle. A task-only app is doing // exactly what it was configured to do; reporting it as idle would say it diff --git a/servers/app/app.go b/servers/app/app.go index e3e6a437b..61ba853b1 100644 --- a/servers/app/app.go +++ b/servers/app/app.go @@ -25,6 +25,7 @@ import ( "miren.dev/runtime/pkg/entity" "miren.dev/runtime/pkg/idgen" "miren.dev/runtime/pkg/rpc" + "miren.dev/runtime/pkg/rpc/standard" "miren.dev/runtime/pkg/secret" "miren.dev/runtime/pkg/ui" "miren.dev/runtime/pkg/workloadroles" @@ -239,6 +240,17 @@ func (r *AppInfo) collectAppHealth(ctx context.Context) (*appHealthSource, error // both arrive here, so they cannot drift into disagreeing about whether an app // is healthy. func (s *appHealthSource) healthOf(entry appEntry) apphealth.State { + out := s.poolHealthOf(entry) + // A disabled app is held at zero on purpose; whatever its pools read as + // on the way down, that is the state worth reporting. + if !entry.app.DisabledAt.IsZero() { + out.Health = apphealth.Disabled + } + return out +} + +// poolHealthOf classifies one app from its pools and config alone. +func (s *appHealthSource) poolHealthOf(entry appEntry) apphealth.State { out := apphealth.State{Name: entry.name} if ps, ok := s.poolState[entry.name]; ok { @@ -729,6 +741,11 @@ func (r *AppInfo) GetConfiguration(ctx context.Context, state *app_v1alpha.CrudG state.Results().SetMaintenanceRoutes(down) } + if !appRec.DisabledAt.IsZero() { + state.Results().SetDisabledAt(standard.ToTimestamp(appRec.DisabledAt)) + state.Results().SetDisabledReason(appRec.DisabledReason) + } + return nil } @@ -958,6 +975,12 @@ func (r *AppInfo) Restart(ctx context.Context, state *app_v1alpha.CrudRestart) e return fmt.Errorf("app %q not found: %w", name, err) } + // Restart restores fixed-mode counts, which would quietly undo a disable + // until the next launcher resync put it back. + if !appRec.DisabledAt.IsZero() { + return fmt.Errorf("app %q is disabled; run `miren app enable` to start it", name) + } + // Resolve the config to restore DesiredInstances for fixed-mode pools. // During crash cooldown the pool manager resets DesiredInstances to 1; // we need to restore the configured value so fixed-mode pools come back diff --git a/servers/app/disable.go b/servers/app/disable.go new file mode 100644 index 000000000..db736920d --- /dev/null +++ b/servers/app/disable.go @@ -0,0 +1,147 @@ +package app + +import ( + "context" + "fmt" + "slices" + "time" + + "miren.dev/runtime/api/app/app_v1alpha" + "miren.dev/runtime/api/compute/compute_v1alpha" + coreutil "miren.dev/runtime/api/core" + "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/pkg/entity" + "miren.dev/runtime/pkg/rpc" +) + +// Disable holds every service of an app at zero instances until Enable. The +// app keeps its versions, routes, env, disks and addons; nothing boots for it, +// including on a deploy, and requests get a 503. +// +// The flag on the App is what keeps it disabled: the launcher, activator and +// pool manager all read it, so the pools scaled down here stay down. +func (r *AppInfo) Disable(ctx context.Context, state *app_v1alpha.CrudDisable) error { + args := state.Args() + name := args.App() + + if !rpc.AllowApp(ctx, name) { + return rpc.AppAccessError(ctx, name) + } + + var appRec core_v1alpha.App + if err := r.EC.Get(ctx, name, &appRec); err != nil { + return fmt.Errorf("app %q not found: %w", name, err) + } + + // Set the flag before touching pools, so nothing that reads it scales a + // pool back up in between. Disabling again only updates the reason. + disabledAt := appRec.DisabledAt + if disabledAt.IsZero() { + disabledAt = time.Now() + } + if err := r.EC.Patch(ctx, appRec.ID, 0, + entity.Time(core_v1alpha.AppDisabledAtId, disabledAt), + entity.String(core_v1alpha.AppDisabledReasonId, args.Reason()), + ); err != nil { + return fmt.Errorf("marking app %q disabled: %w", name, err) + } + + poolList, err := r.EC.List(ctx, entity.Ref(compute_v1alpha.SandboxPoolAppId, appRec.ID)) + if err != nil { + return fmt.Errorf("listing pools: %w", err) + } + + var scaledPools int32 + for poolList.Next() { + var pool compute_v1alpha.SandboxPool + if err := poolList.Read(&pool); err != nil { + continue + } + if pool.DesiredInstances == 0 { + continue + } + if err := r.EC.Patch(ctx, pool.ID, 0, + entity.Int64(compute_v1alpha.SandboxPoolDesiredInstancesId, 0), + ); err != nil { + return fmt.Errorf("scaling pool %s to zero: %w", pool.ID, err) + } + scaledPools++ + } + + r.Log.Info("app disabled", "app", name, "reason", args.Reason(), "pools_scaled", scaledPools) + + state.Results().SetScaledPools(scaledPools) + return nil +} + +// Enable turns a disabled app back on. Fixed-mode services of the active +// version return to their configured count right away; autoscaled services +// stay at zero and start on the next request. +func (r *AppInfo) Enable(ctx context.Context, state *app_v1alpha.CrudEnable) error { + name := state.Args().App() + + if !rpc.AllowApp(ctx, name) { + return rpc.AppAccessError(ctx, name) + } + + var appRec core_v1alpha.App + if err := r.EC.Get(ctx, name, &appRec); err != nil { + return fmt.Errorf("app %q not found: %w", name, err) + } + if appRec.DisabledAt.IsZero() { + return fmt.Errorf("app %q is not disabled", name) + } + + if err := r.EC.Patch(ctx, appRec.ID, 0, + entity.Time(core_v1alpha.AppDisabledAtId, time.Time{}), + entity.String(core_v1alpha.AppDisabledReasonId, ""), + ); err != nil { + return fmt.Errorf("enabling app %q: %w", name, err) + } + + // The launcher's resync would restore fixed counts within a minute; doing + // it here makes enable take effect now. + var restoredPools int32 + if appRec.ActiveVersion != "" { + var ver core_v1alpha.AppVersion + if err := r.EC.GetById(ctx, appRec.ActiveVersion, &ver); err != nil { + return fmt.Errorf("getting active version: %w", err) + } + spec, err := coreutil.ResolveRuntimeConfig(ctx, r.EC.EAC(), &ver) + if err != nil { + return fmt.Errorf("resolving config: %w", err) + } + + poolList, err := r.EC.List(ctx, entity.Ref(compute_v1alpha.SandboxPoolAppId, appRec.ID)) + if err != nil { + return fmt.Errorf("listing pools: %w", err) + } + for poolList.Next() { + var pool compute_v1alpha.SandboxPool + if err := poolList.Read(&pool); err != nil { + continue + } + if !slices.Contains(pool.ReferencedByVersions, appRec.ActiveVersion) { + continue + } + svcConc, err := coreutil.GetServiceConcurrency(spec, pool.Service) + if err != nil || svcConc.Mode != "fixed" || svcConc.NumInstances == 0 { + continue + } + if pool.DesiredInstances == svcConc.NumInstances { + continue + } + if err := r.EC.Patch(ctx, pool.ID, 0, + entity.Int64(compute_v1alpha.SandboxPoolDesiredInstancesId, svcConc.NumInstances), + ); err != nil { + return fmt.Errorf("restoring pool %s: %w", pool.ID, err) + } + restoredPools++ + } + } + + r.Log.Info("app enabled", "app", name, "pools_restored", restoredPools) + + state.Results().SetRestoredPools(restoredPools) + return nil +} diff --git a/servers/app/disable_test.go b/servers/app/disable_test.go new file mode 100644 index 000000000..b4f128974 --- /dev/null +++ b/servers/app/disable_test.go @@ -0,0 +1,111 @@ +package app + +import ( + "context" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/api/app/app_v1alpha" + "miren.dev/runtime/api/compute/compute_v1alpha" + "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/api/entityserver" + "miren.dev/runtime/metrics" + "miren.dev/runtime/pkg/apphealth" + "miren.dev/runtime/pkg/entity" + "miren.dev/runtime/pkg/entity/testutils" + "miren.dev/runtime/pkg/rpc" +) + +// TestAppDisableEnable walks an app with a fixed worker and an autoscaled web +// service through disable, a refused restart, and enable. +func TestAppDisableEnable(t *testing.T) { + ctx := context.Background() + + inmem, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + ec := entityserver.NewClient(slog.Default(), inmem.EAC) + + appInfo := &AppInfo{ + Log: slog.Default(), + EC: ec, + CPU: &metrics.CPUUsage{}, + Mem: &metrics.MemoryUsage{}, + HTTP: &metrics.HTTPMetrics{}, + } + client := &app_v1alpha.CrudClient{ + Client: rpc.LocalClient(app_v1alpha.AdaptCrud(appInfo)), + } + + appName := "disable-app" + appID, err := ec.Create(ctx, appName, &core_v1alpha.App{}) + require.NoError(t, err) + + verID, err := ec.Create(ctx, "disable-app-v1", &core_v1alpha.AppVersion{ + App: appID, + Version: "v1", + Config: core_v1alpha.Config{ + Services: []core_v1alpha.Services{ + {Name: "web", ServiceConcurrency: core_v1alpha.ServiceConcurrency{Mode: "auto", RequestsPerInstance: 10}}, + {Name: "worker", ServiceConcurrency: core_v1alpha.ServiceConcurrency{Mode: "fixed", NumInstances: 2}}, + }, + }, + }) + require.NoError(t, err) + require.NoError(t, ec.Patch(ctx, appID, 0, entity.Ref(core_v1alpha.AppActiveVersionId, verID))) + + webPool, err := ec.Create(ctx, "disable-app-web", &compute_v1alpha.SandboxPool{ + App: appID, Service: "web", DesiredInstances: 1, ReferencedByVersions: []entity.Id{verID}, + }) + require.NoError(t, err) + workerPool, err := ec.Create(ctx, "disable-app-worker", &compute_v1alpha.SandboxPool{ + App: appID, Service: "worker", DesiredInstances: 2, ReferencedByVersions: []entity.Id{verID}, + }) + require.NoError(t, err) + + desired := func(id entity.Id) int64 { + t.Helper() + var pool compute_v1alpha.SandboxPool + require.NoError(t, ec.GetById(ctx, id, &pool)) + return pool.DesiredInstances + } + getApp := func() core_v1alpha.App { + t.Helper() + var app core_v1alpha.App + require.NoError(t, ec.GetById(ctx, appID, &app)) + return app + } + + _, err = client.Enable(ctx, appName) + require.ErrorContains(t, err, "is not disabled") + + disabled, err := client.Disable(ctx, appName, "moved to the new box") + require.NoError(t, err) + assert.Equal(t, int32(2), disabled.ScaledPools()) + assert.Equal(t, int64(0), desired(webPool)) + assert.Equal(t, int64(0), desired(workerPool)) + app := getApp() + assert.False(t, app.DisabledAt.IsZero()) + assert.Equal(t, "moved to the new box", app.DisabledReason) + + src, err := appInfo.collectAppHealth(ctx) + require.NoError(t, err) + for _, entry := range src.apps { + if entry.name == appName { + assert.Equal(t, apphealth.Disabled, src.healthOf(entry).Health) + } + } + + _, err = client.Restart(ctx, appName, "") + require.ErrorContains(t, err, "is disabled", "restart would restore fixed counts and undo the disable") + + enabled, err := client.Enable(ctx, appName) + require.NoError(t, err) + assert.Equal(t, int32(1), enabled.RestoredPools()) + assert.Equal(t, int64(2), desired(workerPool), "fixed service returns to its configured count") + assert.Equal(t, int64(0), desired(webPool), "autoscaled service starts on the next request") + app = getApp() + assert.True(t, app.DisabledAt.IsZero()) + assert.Empty(t, app.DisabledReason) +} diff --git a/servers/app/runtime.go b/servers/app/runtime.go index e691b1a6a..6449927f9 100644 --- a/servers/app/runtime.go +++ b/servers/app/runtime.go @@ -239,6 +239,12 @@ func (a *AppInfo) AppInfo(ctx context.Context, state *app_v1alpha.AppStatusAppIn } } rai.SetHealth(health.classify()) + if !appRec.DisabledAt.IsZero() { + rai.SetHealth(apphealth.Disabled) + for _, svc := range services { + svc.SetHealth(apphealth.Disabled) + } + } rai.SetReadyInstances(int32(health.ready)) rai.SetDesiredInstances(int32(health.desired)) if health.inCooldown { From d07852952a8345a65b7052c9b000f325db26213e Mon Sep 17 00:00:00 2001 From: Jeff Casimir Date: Wed, 7 Oct 2026 09:38:01 -0600 Subject: [PATCH 02/21] Hold a disabled app's services at zero The launcher keeps every pool of a disabled app at zero through resyncs and deploys, skipping the one-instance boot it does for deploy verification. The activator answers requests with ErrAppDisabled, served as 503, instead of waiting out the pool timeout for capacity that won't come. The pool manager clears crash state on a disabled app's pools, so cooldown can't hand them a replacement and revive the app. Co-Authored-By: Claude Opus 5.5 --- components/activator/activator.go | 23 ++++++++ .../activator/activator_fixed_mode_test.go | 50 +++++++++++++++++ controllers/deployment/launcher.go | 23 ++++++-- controllers/deployment/launcher_test.go | 54 +++++++++++++++++++ controllers/sandboxpool/manager.go | 30 ++++++++++- controllers/sandboxpool/manager_test.go | 32 +++++++++++ servers/httpingress/httpingress.go | 8 ++- 7 files changed, 211 insertions(+), 9 deletions(-) diff --git a/components/activator/activator.go b/components/activator/activator.go index f44170fc7..25dd68227 100644 --- a/components/activator/activator.go +++ b/components/activator/activator.go @@ -325,6 +325,11 @@ func (a *localActivator) AcquireLease(ctx context.Context, ver *core_v1alpha.App var ErrSandboxDiedEarly = fmt.Errorf("sandbox died while booting") var ErrPoolTimeout = fmt.Errorf("timeout waiting for sandbox from pool") +// ErrAppDisabled means the app was disabled with `miren app disable`. No +// capacity will arrive until it is enabled, so callers fail fast instead of +// waiting out the pool timeout. +var ErrAppDisabled = fmt.Errorf("app is disabled") + // waitForSandbox waits for a sandbox with capacity to become available. // If incrementPool is true, it will ensure the pool exists and increment DesiredInstances. // If incrementPool is false, it assumes PENDING sandboxes exist and just waits for them. @@ -599,6 +604,21 @@ func (a *localActivator) registerVersionPoolLocked( } } +// appDisabled reports whether the app is disabled. A lookup failure reads as +// not disabled, leaving the request to the normal capacity path. +func (a *localActivator) appDisabled(ctx context.Context, appID entity.Id) bool { + if appID == "" { + return false + } + resp, err := a.eac.Get(ctx, appID.String()) + if err != nil { + return false + } + var app core_v1alpha.App + app.Decode(resp.Entity().Entity()) + return !app.DisabledAt.IsZero() +} + // requestPoolCapacity finds the SandboxPool created by DeploymentLauncher and increments DesiredInstances. // It uses retry logic with exponential backoff to handle the race where Activator receives // a request before DeploymentLauncher has finished creating the pool. @@ -622,6 +642,9 @@ func (a *localActivator) requestPoolCapacity(ctx context.Context, ver *core_v1al sc := core_v1alpha.ServiceConcurrency(svcConcurrency) strategy := concurrency.NewStrategyForVersion(ver, service, &sc) maxInstances := int64(strategy.MaxInstances()) + if a.appDisabled(ctx, ver.App) { + return nil, ErrAppDisabled + } // poolLoop is labeled so the pool-creation retry path below can break all // the way back out to re-read the cache. The outer loop re-acquires a.mu diff --git a/components/activator/activator_fixed_mode_test.go b/components/activator/activator_fixed_mode_test.go index c647dc8bf..6ff7d141c 100644 --- a/components/activator/activator_fixed_mode_test.go +++ b/components/activator/activator_fixed_mode_test.go @@ -277,3 +277,53 @@ func TestActivatorFixedModeNoSlotExhaustion(t *testing.T) { require.NoError(t, err) } } + +// TestActivatorDisabledAppFailsFast verifies that a lease request for any +// service of a disabled app returns ErrAppDisabled at once. +func TestActivatorDisabledAppFailsFast(t *testing.T) { + ctx := context.Background() + log := testutils.TestLogger(t) + + server, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + appID, err := server.Client.Create(ctx, "test-disabled-app", &core_v1alpha.App{DisabledAt: time.Now()}) + require.NoError(t, err) + + appVer := &core_v1alpha.AppVersion{ + App: appID, + Version: "v1", + ImageUrl: "test:latest", + Config: core_v1alpha.Config{ + Port: 3000, + Services: []core_v1alpha.Services{ + { + Name: "web", + ServiceConcurrency: core_v1alpha.ServiceConcurrency{Mode: "auto", RequestsPerInstance: 10}, + }, + }, + }, + } + verID, err := server.Client.Create(ctx, "test-disabled-v1", appVer) + require.NoError(t, err) + appVer.ID = verID + + activator := &localActivator{ + log: log, + eac: server.EAC, + versions: make(map[verKey]*versionPoolRef), + poolSandboxes: make(map[entity.Id]*poolSandboxes), + pools: make(map[verKey]*poolState), + newSandboxChans: make(map[verKey][]chan struct{}), + invalidationCh: make(chan SandboxInvalidation, 1), + } + + reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + start := time.Now() + lease, err := activator.AcquireLease(reqCtx, appVer, "web") + assert.Nil(t, lease) + require.ErrorIs(t, err, ErrAppDisabled) + assert.Less(t, time.Since(start), time.Second, "a disabled app should fail fast") +} diff --git a/controllers/deployment/launcher.go b/controllers/deployment/launcher.go index ec3214ab3..61856eabe 100644 --- a/controllers/deployment/launcher.go +++ b/controllers/deployment/launcher.go @@ -482,6 +482,13 @@ func (l *Launcher) ensurePoolForService(ctx context.Context, app *core_v1alpha.A desiredInstances = min } + // A disabled app's pools are held at zero: they exist so the version is + // complete, but nothing boots, not even for deploy verification. + disabled := !app.DisabledAt.IsZero() + if disabled { + desiredInstances = 0 + } + // Ephemeral versions always get an isolated pool. Sharing a pool with a // non-ephemeral version (or another ephemeral with a different label) would // mean the pool's scaling behavior depends on which version is making the @@ -527,12 +534,13 @@ func (l *Launcher) ensurePoolForService(ctx context.Context, app *core_v1alpha.A needsUpdate = true } - // Fixed-mode strategies enforce their configured minimum. - if min := int64(strategy.MinInstances()); min > 0 && poolWithEntity.Pool.DesiredInstances != min { - poolWithEntity.Pool.DesiredInstances = min + // Fixed-mode strategies enforce their configured minimum; a disabled + // app is held at zero instead. + if min := int64(strategy.MinInstances()); (min > 0 || disabled) && poolWithEntity.Pool.DesiredInstances != desiredInstances { + poolWithEntity.Pool.DesiredInstances = desiredInstances l.Log.Info("strategy-enforced minimum, updating desired instances", "service", serviceName, - "desired_instances", min) + "desired_instances", desiredInstances) needsUpdate = true } @@ -542,7 +550,7 @@ func (l *Launcher) ensurePoolForService(ctx context.Context, app *core_v1alpha.A // the minutely resync repeatedly cold-starts every idle app and blocks the // launcher's worker while it waits for readiness. bootForVerification := false - if versionChanged && poolWithEntity.Pool.DesiredInstances < 1 { + if versionChanged && !disabled && poolWithEntity.Pool.DesiredInstances < 1 { poolWithEntity.Pool.DesiredInstances = 1 l.Log.Info("flooring drained pool to one instance for deploy verification", "service", serviceName, @@ -622,6 +630,11 @@ func (l *Launcher) ensurePoolForService(ctx context.Context, app *core_v1alpha.A "service", serviceName, "desired_instances", desiredInstances) + // A disabled app's pool has nothing to boot, so there's nothing to wait for. + if disabled { + return "", nil + } + // Return the new pool ID — caller should wait for it to become ready return id, nil } diff --git a/controllers/deployment/launcher_test.go b/controllers/deployment/launcher_test.go index e88e4e76e..5c4a9be40 100644 --- a/controllers/deployment/launcher_test.go +++ b/controllers/deployment/launcher_test.go @@ -4860,3 +4860,57 @@ func TestConcurrentCreateAndReconcileNoDeadlock(t *testing.T) { pools := listAllPools(t, ctx, server) assert.Len(t, pools, 1, "concurrent create + reconcile must converge on one pool") } + +// TestDisabledAppStaysAtZero verifies that the launcher holds every service +// of a disabled app at zero: the resync doesn't restore a fixed count and a +// deploy doesn't boot an instance for verification. +func TestDisabledAppStaysAtZero(t *testing.T) { + ctx := context.Background() + log := testutils.TestLogger(t) + + server, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + app := &core_v1alpha.App{Project: entity.Id("project-1")} + appID, err := server.Client.Create(ctx, "test-app", app) + require.NoError(t, err) + app.ID = appID + + newVersion := func(name string) *core_v1alpha.AppVersion { + t.Helper() + ver := &core_v1alpha.AppVersion{ + App: app.ID, + Version: name, + ImageUrl: "worker:latest", + Config: core_v1alpha.Config{ + Services: []core_v1alpha.Services{ + { + Name: "worker", + ServiceConcurrency: core_v1alpha.ServiceConcurrency{Mode: "fixed", NumInstances: 2}, + }, + }, + }, + } + verID, createErr := server.Client.Create(ctx, name, ver) + require.NoError(t, createErr) + ver.ID = verID + return ver + } + + app.ActiveVersion = newVersion("v1").ID + app.DisabledAt = time.Now() + require.NoError(t, server.Client.Update(ctx, app)) + require.NoError(t, newTestLauncher(log, server.EAC).Reconcile(ctx, app, nil)) + + pools := listAllPools(t, ctx, server) + require.Len(t, pools, 1) + assert.Equal(t, int64(0), pools[0].DesiredInstances, "a disabled app's fixed service must not be restored") + + app.ActiveVersion = newVersion("v2").ID + require.NoError(t, server.Client.Update(ctx, app)) + require.NoError(t, newTestLauncher(log, server.EAC).Reconcile(ctx, app, nil)) + + pools = listAllPools(t, ctx, server) + require.Len(t, pools, 1) + assert.Equal(t, int64(0), pools[0].DesiredInstances, "a deploy to a disabled app must not boot for verification") +} diff --git a/controllers/sandboxpool/manager.go b/controllers/sandboxpool/manager.go index d3eef4c4b..b52a4816c 100644 --- a/controllers/sandboxpool/manager.go +++ b/controllers/sandboxpool/manager.go @@ -85,8 +85,19 @@ func (m *Manager) Reconcile(ctx context.Context, pool *compute_v1alpha.SandboxPo // Skip crash detection for decommissioned pools (desired=0, no references). // Sandbox deaths during intentional scale-down are expected, not crashes. - if pool.DesiredInstances > 0 || len(pool.ReferencedByVersions) > 0 { - newCrashes, latestCrash := m.countStartupFailures(sandboxes, pool) + // + // The same holds for a disabled app's pools: they are held at zero on + // purpose, so sandboxes dying on the way down are not crashes, and cooldown + // must not hand a referenced pool a replacement. The app lookup only runs + // when there is crash state to clear, keeping it off the steady-state path. + newCrashes, latestCrash := m.countStartupFailures(sandboxes, pool) + hasCrashState := newCrashes > 0 || pool.ConsecutiveCrashCount > 0 || !pool.CooldownUntil.IsZero() + if pool.DesiredInstances == 0 && len(pool.ReferencedByVersions) > 0 && hasCrashState && m.appDisabled(ctx, pool) { + m.log.Info("clearing crash state on disabled app", "pool", pool.ID, "service", pool.Service) + pool.LastCrashTime = latestCrash + pool.ConsecutiveCrashCount = 0 + pool.CooldownUntil = time.Time{} + } else if pool.DesiredInstances > 0 || len(pool.ReferencedByVersions) > 0 { if newCrashes > 0 { pool.ConsecutiveCrashCount += int64(newCrashes) pool.LastCrashTime = latestCrash @@ -878,6 +889,21 @@ func (m *Manager) countStartupFailures(sandboxes []*sandboxWithMeta, pool *compu return count, latest } +// appDisabled reports whether the pool's app is disabled. A lookup failure +// reads as not disabled, which keeps the existing crash handling. +func (m *Manager) appDisabled(ctx context.Context, pool *compute_v1alpha.SandboxPool) bool { + if pool.App == "" { + return false + } + resp, err := m.eac.Get(ctx, pool.App.String()) + if err != nil { + return false + } + var app core_v1alpha.App + app.Decode(resp.Entity().Entity()) + return !app.DisabledAt.IsZero() +} + // backoffDuration calculates the exponential backoff duration based on consecutive crash count // Uses exponential backoff: 10s, 20s, 40s, 80s, 160s, 320s, 640s, ... // Capped at 15 minutes diff --git a/controllers/sandboxpool/manager_test.go b/controllers/sandboxpool/manager_test.go index 9eea432cd..4fdfc1af9 100644 --- a/controllers/sandboxpool/manager_test.go +++ b/controllers/sandboxpool/manager_test.go @@ -1441,6 +1441,38 @@ func TestManagerDecommissionedPool_NoCrashDetection(t *testing.T) { "decommissioned pool should not enter cooldown") } +// TestManagerDisabledApp_NoCrashCooldown verifies that a disabled app's pool, +// whose config still asks for instances, is held at zero rather than revived +// by crash cooldown. +func TestManagerDisabledApp_NoCrashCooldown(t *testing.T) { + ctx := context.Background() + server, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + appID, err := server.Client.Create(ctx, "disabled-app", &core_v1alpha.App{DisabledAt: time.Now()}) + require.NoError(t, err) + + pool := &compute_v1alpha.SandboxPool{ + App: appID, + Service: "worker", + DesiredInstances: 0, + ReferencedByVersions: []entity.Id{"ver-1"}, + ConsecutiveCrashCount: 2, + CooldownUntil: time.Now().Add(5 * time.Minute), + SandboxSpec: compute_v1alpha.SandboxSpec{Version: "ver-1"}, + } + poolID, err := server.Client.Create(ctx, "disabled-pool", pool) + require.NoError(t, err) + pool.ID = poolID + + reconcilePool(t, ctx, server, NewManager(testutils.TestLogger(t), server.EAC), pool) + + updated := getPool(t, ctx, server, poolID) + assert.Equal(t, int64(0), updated.DesiredInstances, "cooldown must not revive a disabled app") + assert.Equal(t, int64(0), updated.ConsecutiveCrashCount) + assert.True(t, updated.CooldownUntil.IsZero()) +} + // TestManagerCrashResetDoesNotRecount verifies that after crash state is reset // (e.g. by a deploy), old DEAD sandboxes are not re-counted as new crashes. // This is a regression test for MIR-956. diff --git a/servers/httpingress/httpingress.go b/servers/httpingress/httpingress.go index f7c0083a2..7c27cb1b5 100644 --- a/servers/httpingress/httpingress.go +++ b/servers/httpingress/httpingress.go @@ -1080,7 +1080,9 @@ func (h *Server) serveAuthenticatedRequest(w http.ResponseWriter, req *http.Requ actLease, err := h.aa.AcquireLease(actContext, &av, service) if err != nil { - if errors.Is(err, ephemeralx.ErrDisksUnsupported) { + if errors.Is(err, activator.ErrAppDisabled) { + h.serveIngressError(w, req, fmt.Sprintf("The application %s is disabled.\n", targetAppId), http.StatusServiceUnavailable) + } else if errors.Is(err, ephemeralx.ErrDisksUnsupported) { h.Log.Warn("disk-backed preview denied", "error", err, "app", targetAppId) h.serveIngressError(w, req, "This preview uses unsupported storage. Remove disk attachments and redeploy the preview.", http.StatusLocked) } else if errors.Is(err, activator.ErrSandboxDiedEarly) { @@ -1479,7 +1481,9 @@ func (h *Server) DoRequest(ctx context.Context, req *httpingress_v1alpha.Interna actLease, err := h.aa.AcquireLease(actContext, &av, service) if err != nil { - if errors.Is(err, ephemeralx.ErrDisksUnsupported) { + if errors.Is(err, activator.ErrAppDisabled) { + resp.SetError("app is disabled") + } else if errors.Is(err, ephemeralx.ErrDisksUnsupported) { resp.SetError("This preview uses unsupported storage. Remove disk attachments and redeploy the preview.") } else if errors.Is(err, activator.ErrSandboxDiedEarly) { resp.SetError("sandbox died early while acquiring lease") From b02e44bb70b31f3b85ee3913a2a56be0d0ffe2d5 Mon Sep 17 00:00:00 2001 From: Jeff Casimir Date: Wed, 7 Oct 2026 09:41:39 -0600 Subject: [PATCH 03/21] Add miren app disable and enable commands app list shows disabled apps, app status shows when and why, and a deploy to a disabled app finishes with a verdict instead of waiting for an instance. Co-Authored-By: Claude Opus 5.5 --- cli/commands/app_disable.go | 42 ++++++++++++++++++++++++ cli/commands/app_list.go | 2 ++ cli/commands/app_status.go | 17 ++++++++++ cli/commands/commands.go | 24 ++++++++++++++ cli/commands/deploy_events.go | 4 +++ cli/commands/health_wait.go | 10 ++++++ cli/commands/health_wait_test.go | 1 + docs/command-sidebar.json | 2 ++ docs/docs/command/app-disable.md | 55 ++++++++++++++++++++++++++++++++ docs/docs/command/app-enable.md | 45 ++++++++++++++++++++++++++ docs/docs/command/app.md | 2 ++ docs/docs/commands.md | 2 ++ docs/docs/scaling.md | 14 ++++++++ 13 files changed, 220 insertions(+) create mode 100644 cli/commands/app_disable.go create mode 100644 docs/docs/command/app-disable.md create mode 100644 docs/docs/command/app-enable.md diff --git a/cli/commands/app_disable.go b/cli/commands/app_disable.go new file mode 100644 index 000000000..2bbc3434a --- /dev/null +++ b/cli/commands/app_disable.go @@ -0,0 +1,42 @@ +package commands + +import ( + "miren.dev/runtime/api/app/app_v1alpha" +) + +func AppDisable(ctx *Context, opts struct { + Reason string `short:"r" long:"reason" description:"Why the app is disabled, shown by app status"` + AppCentric +}) error { + crudcl, err := ctx.RPCClient("dev.miren.runtime/app") + if err != nil { + return err + } + + result, err := app_v1alpha.NewCrudClient(crudcl).Disable(ctx, opts.App, opts.Reason) + if err != nil { + return err + } + + ctx.Printf("Disabled app %s (scaled %d pools to zero)\n", opts.App, result.ScaledPools()) + ctx.Printf("Run `miren app enable -a %s` to start it again.\n", opts.App) + return nil +} + +func AppEnable(ctx *Context, opts struct { + AppCentric +}) error { + crudcl, err := ctx.RPCClient("dev.miren.runtime/app") + if err != nil { + return err + } + + result, err := app_v1alpha.NewCrudClient(crudcl).Enable(ctx, opts.App) + if err != nil { + return err + } + + ctx.Printf("Enabled app %s (restored %d fixed pools; autoscaled services start on the next request)\n", + opts.App, result.RestoredPools()) + return nil +} diff --git a/cli/commands/app_list.go b/cli/commands/app_list.go index fb0e8fdf9..aa334f73a 100644 --- a/cli/commands/app_list.go +++ b/cli/commands/app_list.go @@ -149,6 +149,8 @@ func AppList(ctx *Context, opts struct { } else { status = infoYellow.Render("⚠️ 0 (fixed)") } + case "disabled": + status = infoYellow.Render("⏸ disabled") case "healthy": status = infoGreen.Render(fmt.Sprintf("%d%s", a.ReadyInstances(), modeSuffix)) case "degraded", "starting": diff --git a/cli/commands/app_status.go b/cli/commands/app_status.go index fd9ede183..3528051c5 100644 --- a/cli/commands/app_status.go +++ b/cli/commands/app_status.go @@ -9,6 +9,7 @@ import ( "github.com/charmbracelet/lipgloss" "miren.dev/runtime/api/app/app_v1alpha" "miren.dev/runtime/api/deployment/deployment_v1alpha" + "miren.dev/runtime/pkg/rpc/standard" "miren.dev/runtime/pkg/theme" "miren.dev/runtime/pkg/ui" ) @@ -104,6 +105,15 @@ func AppStatus(ctx *Context, opts struct { yellowStyle.Render(fmt.Sprintf("%s serving a holding page", maintenanceRouteList(down)))) } + if appResult.HasDisabledAt() { + since := standard.FromTimestamp(appResult.DisabledAt()) + line := fmt.Sprintf("since %s", since.Local().Format(time.RFC1123)) + if reason := appResult.DisabledReason(); reason != "" { + line += " — " + reason + } + ctx.Printf("%s %s\n", labelStyle.Render("Disabled:"), yellowStyle.Render(line)) + } + // Configuration if appConfig != nil { ctx.Printf("\n%s\n", labelStyle.Render("Configuration:")) @@ -385,6 +395,8 @@ func printAppStatusJSON( Source *sourceJSON `json:"source,omitempty"` WorkloadRole string `json:"workload_role,omitempty"` MaintenanceRoutes []string `json:"maintenance_routes,omitempty"` + DisabledAt *time.Time `json:"disabled_at,omitempty"` + DisabledReason string `json:"disabled_reason,omitempty"` Configuration *configuration `json:"configuration,omitempty"` ActiveDeployment *deploymentJSON `json:"active_deployment,omitempty"` RecentDeployments []deploymentJSON `json:"recent_deployments,omitempty"` @@ -395,8 +407,13 @@ func printAppStatusJSON( Cluster: cluster, WorkloadRole: appResult.WorkloadRole(), MaintenanceRoutes: appResult.MaintenanceRoutes(), + DisabledReason: appResult.DisabledReason(), Services: health, } + if appResult.HasDisabledAt() { + at := standard.FromTimestamp(appResult.DisabledAt()) + output.DisabledAt = &at + } if healthErr != nil { output.HealthError = healthErr.Error() } diff --git a/cli/commands/commands.go b/cli/commands/commands.go index 1314dbbab..9cba569a3 100644 --- a/cli/commands/commands.go +++ b/cli/commands/commands.go @@ -292,6 +292,20 @@ miren deploy --format jsonl | jq -c 'select(.event == "build_step")' Body: "miren app restart -s web", }), )) + d.Dispatch("app disable", Infer("app disable", "Hold every service of an app at zero instances", AppDisable, + WithDescription(appDisableDescription), + WithExample(mflags.Example{ + Name: "Disable an app", + Body: "miren app disable -a myapp --reason \"Moved to the new cluster\"", + }), + )) + d.Dispatch("app enable", Infer("app enable", "Start a disabled app again", AppEnable, + WithDescription(appEnableDescription), + WithExample(mflags.Example{ + Name: "Enable an app", + Body: "miren app enable -a myapp", + }), + )) d.Dispatch("app set-workload-role", Infer("app set-workload-role", "Set the API role for an app's sandbox identity tokens", AppSetWorkloadRole, WithExample(mflags.Example{ Name: "Let an app's workloads read and deploy their own app", @@ -1566,6 +1580,16 @@ Use restart to: You do not need to restart after ` + "`" + `miren env set` + "`" + `, ` + "`" + `miren env delete` + "`" + `, ` + "`" + `miren addon create` + "`" + `, or ` + "`" + `miren addon destroy` + "`" + `. Each of those already creates a new version and rolls out new sandboxes automatically. A manual restart on top only adds a redundant rollout. :::` +const appDisableDescription = `Disable turns an app off. Every service is held at zero instances: running sandboxes shut down, nothing boots on a deploy, and HTTP requests get a 503. The app keeps its versions, routes, environment, disks, and addons, so ` + "`" + `miren app enable` + "`" + ` brings it back as it was. + +Disabled is an explicit state, so ` + "`" + `miren app list` + "`" + ` and ` + "`" + `miren app status` + "`" + ` report it as such rather than as an app that failed to start. + +:::note[Tasks still run] +Disable stops services, not tasks. ` + "`" + `miren app run` + "`" + ` and scheduled tasks keep working, which leaves room for a backup or a migration while the app is off. +:::` + +const appEnableDescription = `Enable turns a disabled app back on. Fixed-mode services return to their configured count right away; autoscaled services stay at zero and start on the next request.` + const secretSetDescription = `Stores a value in the cluster's secret store. The value is encrypted at rest and is never echoed, logged, or written to disk by this command — it travels to the server, which holds the only key. Each write mints a new immutable version and prints its handle, e.g. ` + "`" + `payments/stripe-key@x1A` + "`" + `. Storing a value identical to the current one is reported as unchanged rather than minting a duplicate, so re-running the command is safe. diff --git a/cli/commands/deploy_events.go b/cli/commands/deploy_events.go index 001e721b5..deadcff7f 100644 --- a/cli/commands/deploy_events.go +++ b/cli/commands/deploy_events.go @@ -197,6 +197,10 @@ func outcomeEvent(o terminalOutcome) deployevents.Outcome { return deployevents.OutcomeHealthy case outcomeScaledToZero: return deployevents.OutcomeScaledToZero + case outcomeDisabled: + // Published as scaled to zero, which it is, rather than widening the + // event vocabulary for a variant of the same state. + return deployevents.OutcomeScaledToZero case outcomeNoService: // Keep the published value for compatibility. It predates static-only // apps and means the deployment has no service process to wait for. diff --git a/cli/commands/health_wait.go b/cli/commands/health_wait.go index 4bbd2d7d5..81dc5af89 100644 --- a/cli/commands/health_wait.go +++ b/cli/commands/health_wait.go @@ -96,6 +96,9 @@ const ( // decisionNoService: the app has no long-running process by design. Its // tasks or static content are available, which is as "up" as it gets. decisionNoService + // decisionDisabled: the app is disabled, so the version is deployed but + // held at zero until the app is enabled. + decisionDisabled ) // decideActivation maps a snapshot to a terminal decision (or "keep waiting"). @@ -125,6 +128,8 @@ func decideActivation(snap healthSnapshot) activationDecision { return decisionNoService case snap.health == apphealth.Idle: return decisionScaledToZero + case snap.health == apphealth.Disabled: + return decisionDisabled case snap.ready > 0: return decisionHealthy default: @@ -189,6 +194,7 @@ const ( outcomeHealthy terminalOutcome = iota outcomeScaledToZero outcomeNoService + outcomeDisabled outcomeCrashed outcomeTimeout outcomeCanceled @@ -210,6 +216,8 @@ func pollOutcome(ctx context.Context, getter appInfoGetter, appName, versionID s return outcomeScaledToZero, snap, true case decisionNoService: return outcomeNoService, snap, true + case decisionDisabled: + return outcomeDisabled, snap, true case decisionCrashed: return outcomeCrashed, snap, true default: @@ -240,6 +248,8 @@ func healthOutcomeText(versionDisplay string, outcome terminalOutcome, snap heal // Deliberately not the scaled-to-zero wording: nothing went to sleep, // this app never had a long-running process to begin with. return fmt.Sprintf("Version %s deployed — no long-running service required", versionDisplay), true + case outcomeDisabled: + return fmt.Sprintf("Version %s deployed — the app is disabled; run `miren app enable` to start it", versionDisplay), true case outcomeCrashed: detail := "" if snap.crashCount > 0 { diff --git a/cli/commands/health_wait_test.go b/cli/commands/health_wait_test.go index 1ebce6514..76a9ada26 100644 --- a/cli/commands/health_wait_test.go +++ b/cli/commands/health_wait_test.go @@ -104,6 +104,7 @@ func TestDecideActivation(t *testing.T) { // Without its own arm this falls to ready > 0, which an app without a // service never satisfies -- the deploy would time out despite succeeding. {"service-free app is deployed", healthSnapshot{versionActive: true, health: apphealth.Ready, ready: 0, desired: 0}, decisionNoService}, + {"disabled app is deployed", healthSnapshot{versionActive: true, health: apphealth.Disabled, ready: 0, desired: 0}, decisionDisabled}, {"one of many serving", healthSnapshot{versionActive: true, health: apphealth.Degraded, ready: 1, desired: 3}, decisionHealthy}, {"fully healthy", healthSnapshot{versionActive: true, health: apphealth.Healthy, ready: 2, desired: 2}, decisionHealthy}, {"active but nothing serving", healthSnapshot{versionActive: true, health: apphealth.Starting, ready: 0, desired: 1}, decisionWait}, diff --git a/docs/command-sidebar.json b/docs/command-sidebar.json index 5d6e3e83f..600e690bf 100644 --- a/docs/command-sidebar.json +++ b/docs/command-sidebar.json @@ -40,6 +40,8 @@ "items": [ "command/app-attach", "command/app-delete", + "command/app-disable", + "command/app-enable", "command/app-history", "command/app-list", "command/app-restart", diff --git a/docs/docs/command/app-disable.md b/docs/docs/command/app-disable.md new file mode 100644 index 000000000..a1b2e4d13 --- /dev/null +++ b/docs/docs/command/app-disable.md @@ -0,0 +1,55 @@ +--- +title: "miren app disable" +sidebar_label: "app disable" +description: "Hold every service of an app at zero instances" +--- + +# miren app disable + +Hold every service of an app at zero instances + +Disable turns an app off. Every service is held at zero instances: running sandboxes shut down, nothing boots on a deploy, and HTTP requests get a 503. The app keeps its versions, routes, environment, disks, and addons, so `miren app enable` brings it back as it was. + +Disabled is an explicit state, so `miren app list` and `miren app status` report it as such rather than as an app that failed to start. + +:::note[Tasks still run] +Disable stops services, not tasks. `miren app run` and scheduled tasks keep working, which leaves room for a backup or a migration while the app is off. +::: + +## Usage + +```bash +miren app disable [flags] +``` + +## Flags + +- `--reason, -r` — Why the app is disabled, shown by app status + +## Config Options + +- `--cluster, -C` — Cluster name +- `--config` — Path to the config file + +## App Options + +- `--app, -a` — Application name +- `--dir, -d` — Directory to run from (default: `.`) + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## Examples + +**Disable an app:** + +```bash +miren app disable -a myapp --reason "Moved to the new cluster" +``` + +## See also + +- [`miren app`](./app.md) diff --git a/docs/docs/command/app-enable.md b/docs/docs/command/app-enable.md new file mode 100644 index 000000000..c23f03082 --- /dev/null +++ b/docs/docs/command/app-enable.md @@ -0,0 +1,45 @@ +--- +title: "miren app enable" +sidebar_label: "app enable" +description: "Start a disabled app again" +--- + +# miren app enable + +Start a disabled app again + +Enable turns a disabled app back on. Fixed-mode services return to their configured count right away; autoscaled services stay at zero and start on the next request. + +## Usage + +```bash +miren app enable [flags] +``` + +## Config Options + +- `--cluster, -C` — Cluster name +- `--config` — Path to the config file + +## App Options + +- `--app, -a` — Application name +- `--dir, -d` — Directory to run from (default: `.`) + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## Examples + +**Enable an app:** + +```bash +miren app enable -a myapp +``` + +## See also + +- [`miren app`](./app.md) diff --git a/docs/docs/command/app.md b/docs/docs/command/app.md index 7bc1057a2..a62615233 100644 --- a/docs/docs/command/app.md +++ b/docs/docs/command/app.md @@ -61,6 +61,8 @@ miren app --watch - [`miren app attach`](./app-attach.md) — Attach to a running task - [`miren app delete`](./app-delete.md) — Delete an application and all its resources +- [`miren app disable`](./app-disable.md) — Hold every service of an app at zero instances +- [`miren app enable`](./app-enable.md) — Start a disabled app again - [`miren app history`](./app-history.md) — Show deployment history for an application - [`miren app list`](./app-list.md) — List all applications - [`miren app restart`](./app-restart.md) — Restart an application diff --git a/docs/docs/commands.md b/docs/docs/commands.md index c21bea5c0..acedd374c 100644 --- a/docs/docs/commands.md +++ b/docs/docs/commands.md @@ -40,6 +40,8 @@ Complete reference for all `miren` CLI commands. | [`miren app`](./command/app.md) | Get information about an application | | [`miren app attach`](./command/app-attach.md) | Attach to a running task | | [`miren app delete`](./command/app-delete.md) | Delete an application and all its resources | +| [`miren app disable`](./command/app-disable.md) | Hold every service of an app at zero instances | +| [`miren app enable`](./command/app-enable.md) | Start a disabled app again | | [`miren app history`](./command/app-history.md) | Show deployment history for an application | | [`miren app list`](./command/app-list.md) | List all applications | | [`miren app restart`](./command/app-restart.md) | Restart an application | diff --git a/docs/docs/scaling.md b/docs/docs/scaling.md index 297b76e0f..58cca7a9e 100644 --- a/docs/docs/scaling.md +++ b/docs/docs/scaling.md @@ -124,6 +124,20 @@ num_instances = 3 | `mode` | Must be `"fixed"` | `"fixed"` for non-web | | `num_instances` | Exact number of instances to run | 1 | +## Disabling an App + +To turn an app off without deleting it, disable it: + +```bash +miren app disable -a myapp --reason "Moved to the new cluster" +``` + +Every service, autoscaled or fixed, is held at zero until you run `miren app enable`. Running instances shut down, deploys record the new version without booting it, HTTP requests get a 503, and `miren app restart` refuses until the app is enabled. The app keeps its versions, routes, environment, disks, and addons. Tasks still run, so `miren app run` works for a backup or migration while the app is off. + +Unlike an autoscaled service that has scaled to zero, a disabled app doesn't wake on traffic. `miren app list` shows it as `disabled`, and `miren app status` shows when and why. + +`miren app enable` returns fixed services to their configured count right away; autoscaled services start on the next request. + ## Examples ### High-Traffic API From 24e90eb05ad61facb6a9a283cc942b84c9340b56 Mon Sep 17 00:00:00 2001 From: Jeff Casimir Date: Thu, 8 Oct 2026 11:03:10 -0600 Subject: [PATCH 04/21] Pause a disabled app's scheduled tasks The run scheduler skips a disabled app and forgets its tasks' frontiers, so enabling the app resumes the schedule from that moment instead of firing every tick missed while it was off. On a fallback copy left disabled on an old cluster, a nightly job would otherwise run on both clusters against shared state. Manual runs are unaffected. Co-Authored-By: Claude Opus 5.5 (1M context) --- controllers/run/scheduler.go | 22 ++++++++++++++++++++ controllers/run/scheduler_test.go | 34 +++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/controllers/run/scheduler.go b/controllers/run/scheduler.go index 458a0d544..3cb2061c3 100644 --- a/controllers/run/scheduler.go +++ b/controllers/run/scheduler.go @@ -137,6 +137,15 @@ func (s *Scheduler) Sweep(ctx context.Context, now time.Time) error { continue } + // A disabled app's schedule is paused, not queued up. Forgetting its + // frontier means the first sweep after enable starts at that moment, + // the same as after an outage, rather than firing every tick missed + // while the app was off. Manual runs aren't affected. + if !app.DisabledAt.IsZero() { + s.forgetApp(app.ID) + continue + } + if err := s.sweepApp(ctx, &app, now); err != nil { s.Log.Warn("failed to sweep app for scheduled tasks", "app", app.ID, "error", err) } @@ -145,6 +154,19 @@ func (s *Scheduler) Sweep(ctx context.Context, now time.Time) error { return nil } +// forgetApp drops the frontier of every task belonging to the app. +func (s *Scheduler) forgetApp(appID entity.Id) { + prefix := appID.String() + "/" + + s.mu.Lock() + defer s.mu.Unlock() + for key := range s.frontier { + if strings.HasPrefix(key, prefix) { + delete(s.frontier, key) + } + } +} + func (s *Scheduler) sweepApp(ctx context.Context, app *core_v1alpha.App, now time.Time) error { var ver core_v1alpha.AppVersion if err := s.EC.GetById(ctx, app.ActiveVersion, &ver); err != nil { diff --git a/controllers/run/scheduler_test.go b/controllers/run/scheduler_test.go index 3a7030436..dc715d3c9 100644 --- a/controllers/run/scheduler_test.go +++ b/controllers/run/scheduler_test.go @@ -226,6 +226,40 @@ func TestSchedulerIgnoresAppsWithoutAnActiveVersion(t *testing.T) { assert.Empty(t, h.runs(ctx)) } +// A disabled app's schedule pauses, and enabling it doesn't replay the ticks +// it missed. On a fallback copy left disabled on an old cluster, either one +// would run a job twice against state the two clusters share. +func TestSchedulerPausesWhileTheAppIsDisabled(t *testing.T) { + ctx := context.Background() + h := newSchedHarness(t, "*-*-* *:00/30:00") + appID := entity.Id("app/demo") + + setDisabled := func(at time.Time) { + t.Helper() + _, err := h.inm.EAC.Patch(ctx, entity.New( + entity.Ref(entity.DBId, appID), + entity.Time(core_v1alpha.AppDisabledAtId, at), + ).Attrs(), 0) + require.NoError(t, err) + } + + require.NoError(t, h.s.Sweep(ctx, atUTC("2026-08-04T12:00:00Z"))) + + setDisabled(atUTC("2026-08-04T12:10:00Z")) + require.NoError(t, h.s.Sweep(ctx, atUTC("2026-08-04T12:30:01Z"))) + require.NoError(t, h.s.Sweep(ctx, atUTC("2026-08-04T14:00:01Z"))) + assert.Empty(t, h.runs(ctx), "no tick fires while the app is disabled") + + setDisabled(time.Time{}) + require.NoError(t, h.s.Sweep(ctx, atUTC("2026-08-04T14:10:00Z"))) + assert.Empty(t, h.runs(ctx), "ticks missed while disabled are not backfilled on enable") + + require.NoError(t, h.s.Sweep(ctx, atUTC("2026-08-04T14:30:01Z"))) + runs := h.runs(ctx) + require.Len(t, runs, 1, "the schedule resumes from the moment of enable") + assert.Equal(t, "2026-08-04T14:30:00Z", runs[0].Tick) +} + // Manual and deploy tasks are not on a schedule and must never be fired by it. func TestSchedulerIgnoresUnscheduledTasks(t *testing.T) { ctx := context.Background() From 11c71bd281f298311018a6eb0ff8727f6e5202b2 Mon Sep 17 00:00:00 2001 From: Jeff Casimir Date: Thu, 8 Oct 2026 11:05:57 -0600 Subject: [PATCH 05/21] Serve a disabled app's 503 from ingress as a holding page The disabled check moves up into resolveIngressTarget, which already reads the App on every request. It now runs before auth, as maintenance does, and before the lease cache, so a warm app stops serving the moment it's disabled instead of when its sandboxes exit. DoRequest gets the same check ahead of its lease cache. The response goes through the maintenance holding page: no-store, the JSON shape with error "disabled", and the page worded as unavailable. Templates get a Disabled field alongside Maintenance, which stays true so a template written for maintenance still renders a holding page. The disable reason is left off the page; it's an operator note for app status. The activator check stays as a backstop for a request already past ingress when the app is disabled. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/docs/server-config.md | 7 +- servers/httpingress/disabled_test.go | 107 +++++++++++++++++++++++++++ servers/httpingress/error_page.go | 14 +++- servers/httpingress/httpingress.go | 24 +++++- servers/httpingress/maintenance.go | 53 +++++++++---- servers/httpingress/tls_check.go | 2 +- 6 files changed, 185 insertions(+), 22 deletions(-) create mode 100644 servers/httpingress/disabled_test.go diff --git a/docs/docs/server-config.md b/docs/docs/server-config.md index c35024ea6..c0adfb1b5 100644 --- a/docs/docs/server-config.md +++ b/docs/docs/server-config.md @@ -104,7 +104,7 @@ server after changing it. For an app-specific override, see Templates receive `.Status` (HTTP status number), `.Title`, `.Description`, `.Site` (visitor hostname on maintenance pages), `.Reason`, `.BackAt`, and -`.Maintenance` (boolean). `{{brandLogo}}` renders the built-in Miren logo. +`.Maintenance` and `.Disabled` (booleans). `{{brandLogo}}` renders the built-in Miren logo. Use self-contained markup and inline CSS if the page must work when the app is unavailable. Only HTML responses use these templates: `Accept` negotiation still selects JSON or plain text for API clients. If the app's template is @@ -115,7 +115,10 @@ For ordinary errors, `.Status`, `.Title`, and `.Description` are set; `.Site`, `.Reason`, and `.BackAt` are empty. During maintenance, `.Status` is 503, `.Maintenance` is true, `.Site` is the visitor's hostname, `.Reason` is the operator's message, and `.BackAt` is a formatted UTC time when provided. -`.Title` and `.Description` are empty on maintenance pages. These are the only +`.Title` and `.Description` are empty on maintenance pages. A disabled app +(see [`miren app disable`](./command/app-disable.md)) gets the same holding +page with `.Disabled` also true and `.Reason` and `.BackAt` empty, so a +template that only checks `.Maintenance` still renders a holding page. These are the only template data fields; app IDs, raw failure messages, and request details are not exposed. HTML escaping is automatic. Templates must be at most 128 KiB; rendered output is capped at 256 KiB and falls back if it exceeds that limit. diff --git a/servers/httpingress/disabled_test.go b/servers/httpingress/disabled_test.go new file mode 100644 index 000000000..e1696570c --- /dev/null +++ b/servers/httpingress/disabled_test.go @@ -0,0 +1,107 @@ +package httpingress + +import ( + "context" + "log/slog" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/api/entityserver/entityserver_v1alpha" + "miren.dev/runtime/api/ingress" + "miren.dev/runtime/api/ingress/ingress_v1alpha" + "miren.dev/runtime/components/activator" + "miren.dev/runtime/pkg/entity" + "miren.dev/runtime/pkg/entity/testutils" + "miren.dev/runtime/pkg/rpc" +) + +// A disabled app is checked before the lease cache. A warm app holds cached +// leases to sandboxes that keep running until the pool scales them down, so a +// check on the activator's scale-up path alone would leave it serving. +func TestDisabledAppStopsServingFromCachedLease(t *testing.T) { + var hits atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + _, _ = w.Write([]byte("from the app")) + })) + defer upstream.Close() + + inmem, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + ctx := context.Background() + + appID, err := inmem.Client.Create(ctx, "shop", &core_v1alpha.App{}) + require.NoError(t, err) + cvID, err := inmem.Client.Create(ctx, "cfg", &core_v1alpha.ConfigVersion{App: appID}) + require.NoError(t, err) + verID, err := inmem.Client.Create(ctx, "ver", &core_v1alpha.AppVersion{App: appID, ConfigVersion: cvID}) + require.NoError(t, err) + require.NoError(t, inmem.Client.Update(ctx, &core_v1alpha.App{ID: appID, ActiveVersion: verID})) + _, err = inmem.Client.Create(ctx, "shop.test", &ingress_v1alpha.HttpRoute{Host: "shop.test", App: appID}) + require.NoError(t, err) + + s := &Server{Log: slog.Default(), eac: inmem.EAC, aa: stubActivator{}, apps: make(map[string]*appUsage)} + s.ingressClient = ingress.NewClient(s.Log, rpc.LocalClient(entityserver_v1alpha.AdaptEntityAccess(inmem.Server))) + s.retainLease(ctx, leaseCacheKey(appID, "web", "", false), &activator.Lease{Size: 10, URL: upstream.URL}) + + do := func(accept string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + req := httptest.NewRequest("GET", "http://shop.test/", nil) + req.Header.Set("Accept", accept) + var appName string + s.serveHTTPWithMetrics(rec, req, &appName) + return rec + } + + rec := do("text/html") + require.Equal(t, http.StatusOK, rec.Code, "the cached lease serves while the app is enabled") + require.Equal(t, int32(1), hits.Load()) + + _, err = inmem.EAC.Patch(ctx, entity.New( + entity.Ref(entity.DBId, appID), + entity.Time(core_v1alpha.AppDisabledAtId, time.Now()), + entity.String(core_v1alpha.AppDisabledReasonId, "moved to the new cluster"), + ).Attrs(), 0) + require.NoError(t, err) + + rec = do("text/html") + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + assert.Equal(t, "no-store", rec.Header().Get("Cache-Control")) + assert.Contains(t, rec.Body.String(), "shop.test is unavailable") + assert.NotContains(t, rec.Body.String(), "moved to the new cluster", "the disable reason is an operator note, not visitor copy") + assert.NotContains(t, rec.Body.String(), "Try again") + + rec = do("application/json") + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + assert.JSONEq(t, `{"error":"disabled"}`, rec.Body.String()) + + rec = do("text/plain") + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + assert.Equal(t, "Unavailable\n", rec.Body.String()) + + assert.Equal(t, int32(1), hits.Load(), "no request reached the app after it was disabled") +} + +// A cluster template written for maintenance keys on .Maintenance. A disabled +// app sets it too, so that template still renders a holding page rather than +// falling through to its generic error branch. +func TestDisabledPageRendersThroughAMaintenanceTemplate(t *testing.T) { + page, err := parseErrorTemplate("{{if .Maintenance}}holding{{if .Disabled}} (disabled){{end}}{{else}}error{{end}}") + require.NoError(t, err) + s := newTestMaintenanceServer() + s.config.ErrorPageTemplate = page + + rec := httptest.NewRecorder() + req := httptest.NewRequest("GET", "http://shop.test/", nil) + req.Header.Set("Accept", "text/html") + s.serveDisabled(rec, req, &core_v1alpha.App{}) + + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + assert.Equal(t, "holding (disabled)", rec.Body.String()) +} diff --git a/servers/httpingress/error_page.go b/servers/httpingress/error_page.go index 5e479e8a3..9b52ce2fe 100644 --- a/servers/httpingress/error_page.go +++ b/servers/httpingress/error_page.go @@ -114,7 +114,11 @@ type errorPageData struct { Site string Reason string BackAt string + // Maintenance is true on every holding page; Disabled narrows it to an + // app turned off with `miren app disable`, so a template written for + // maintenance still renders a sensible page for a disabled app. Maintenance bool + Disabled bool } // A bundled copy of docs/static/img/logo-light.svg keeps the real wordmark @@ -231,7 +235,7 @@ var errorPage = template.Must(template.New("ingress-error").Funcs(template.FuncM -{{if .Maintenance}}Down for maintenance{{else}}{{.Status}} · {{.Title}}{{end}} — Miren +{{if .Disabled}}Unavailable{{else if .Maintenance}}Down for maintenance{{else}}{{.Status}} · {{.Title}}{{end}} — Miren