diff --git a/api/app/app_v1alpha/rpc.gen.go b/api/app/app_v1alpha/rpc.gen.go index a9d97a0f6..eafbe7d22 100644 --- a/api/app/app_v1alpha/rpc.gen.go +++ b/api/app/app_v1alpha/rpc.gen.go @@ -3090,13 +3090,15 @@ func (v *CrudGetConfigurationArgs) UnmarshalJSON(data []byte) error { } type crudGetConfigurationResultsData struct { - Configuration *Configuration `cbor:"0,keyasint,omitempty" json:"configuration,omitempty"` - VersionId *string `cbor:"1,keyasint,omitempty" json:"versionId,omitempty"` - VersionShortId *string `cbor:"2,keyasint,omitempty" json:"versionShortId,omitempty"` - WorkloadRole *string `cbor:"3,keyasint,omitempty" json:"workloadRole,omitempty"` - MaintenanceRoutes *[]string `cbor:"4,keyasint,omitempty" json:"maintenanceRoutes,omitempty"` - SourceKind *string `cbor:"5,keyasint,omitempty" json:"sourceKind,omitempty"` - SourceValue *string `cbor:"6,keyasint,omitempty" json:"sourceValue,omitempty"` + Configuration *Configuration `cbor:"0,keyasint,omitempty" json:"configuration,omitempty"` + VersionId *string `cbor:"1,keyasint,omitempty" json:"versionId,omitempty"` + VersionShortId *string `cbor:"2,keyasint,omitempty" json:"versionShortId,omitempty"` + WorkloadRole *string `cbor:"3,keyasint,omitempty" json:"workloadRole,omitempty"` + MaintenanceRoutes *[]string `cbor:"4,keyasint,omitempty" json:"maintenanceRoutes,omitempty"` + SourceKind *string `cbor:"5,keyasint,omitempty" json:"sourceKind,omitempty"` + SourceValue *string `cbor:"6,keyasint,omitempty" json:"sourceValue,omitempty"` + DisabledAt *standard.Timestamp `cbor:"7,keyasint,omitempty" json:"disabledAt,omitempty"` + DisabledReason *string `cbor:"8,keyasint,omitempty" json:"disabledReason,omitempty"` } type CrudGetConfigurationResults struct { @@ -3133,6 +3135,14 @@ func (v *CrudGetConfigurationResults) SetSourceValue(sourceValue string) { v.data.SourceValue = &sourceValue } +func (v *CrudGetConfigurationResults) SetDisabledAt(disabledAt *standard.Timestamp) { + v.data.DisabledAt = disabledAt +} + +func (v *CrudGetConfigurationResults) SetDisabledReason(disabledReason string) { + v.data.DisabledReason = &disabledReason +} + func (v *CrudGetConfigurationResults) MarshalCBOR() ([]byte, error) { return cbor.Marshal(v.data) } @@ -3814,6 +3824,148 @@ func (v *CrudRestartResults) UnmarshalJSON(data []byte) error { return json.Unmarshal(data, &v.data) } +type crudDisableArgsData struct { + App *string `cbor:"0,keyasint,omitempty" json:"app,omitempty"` + Reason *string `cbor:"1,keyasint,omitempty" json:"reason,omitempty"` +} + +type CrudDisableArgs struct { + call rpc.Call + data crudDisableArgsData +} + +func (v *CrudDisableArgs) HasApp() bool { + return v.data.App != nil +} + +func (v *CrudDisableArgs) App() string { + if v.data.App == nil { + return "" + } + return *v.data.App +} + +func (v *CrudDisableArgs) HasReason() bool { + return v.data.Reason != nil +} + +func (v *CrudDisableArgs) Reason() string { + if v.data.Reason == nil { + return "" + } + return *v.data.Reason +} + +func (v *CrudDisableArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *CrudDisableArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *CrudDisableArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *CrudDisableArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type crudDisableResultsData struct { + ScaledPools *int32 `cbor:"0,keyasint,omitempty" json:"scaled_pools,omitempty"` +} + +type CrudDisableResults struct { + call rpc.Call + data crudDisableResultsData +} + +func (v *CrudDisableResults) SetScaledPools(scaled_pools int32) { + v.data.ScaledPools = &scaled_pools +} + +func (v *CrudDisableResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *CrudDisableResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *CrudDisableResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *CrudDisableResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type crudEnableArgsData struct { + App *string `cbor:"0,keyasint,omitempty" json:"app,omitempty"` +} + +type CrudEnableArgs struct { + call rpc.Call + data crudEnableArgsData +} + +func (v *CrudEnableArgs) HasApp() bool { + return v.data.App != nil +} + +func (v *CrudEnableArgs) App() string { + if v.data.App == nil { + return "" + } + return *v.data.App +} + +func (v *CrudEnableArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *CrudEnableArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *CrudEnableArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *CrudEnableArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type crudEnableResultsData struct { + RestoredPools *int32 `cbor:"0,keyasint,omitempty" json:"restored_pools,omitempty"` +} + +type CrudEnableResults struct { + call rpc.Call + data crudEnableResultsData +} + +func (v *CrudEnableResults) SetRestoredPools(restored_pools int32) { + v.data.RestoredPools = &restored_pools +} + +func (v *CrudEnableResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *CrudEnableResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *CrudEnableResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *CrudEnableResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + type crudListVersionsArgsData struct { App *string `cbor:"0,keyasint,omitempty" json:"app,omitempty"` Ephemeral *bool `cbor:"1,keyasint,omitempty" json:"ephemeral,omitempty"` @@ -4190,6 +4342,58 @@ func (t *CrudRestart) Results() *CrudRestartResults { return results } +type CrudDisable struct { + rpc.Call + args CrudDisableArgs + results CrudDisableResults +} + +func (t *CrudDisable) Args() *CrudDisableArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *CrudDisable) Results() *CrudDisableResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + +type CrudEnable struct { + rpc.Call + args CrudEnableArgs + results CrudEnableResults +} + +func (t *CrudEnable) Args() *CrudEnableArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *CrudEnable) Results() *CrudEnableResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + type CrudListVersions struct { rpc.Call args CrudListVersionsArgs @@ -4228,6 +4432,8 @@ type Crud interface { SetInitialEnvVars(ctx context.Context, state *CrudSetInitialEnvVars) error DeleteEnvVar(ctx context.Context, state *CrudDeleteEnvVar) error Restart(ctx context.Context, state *CrudRestart) error + Disable(ctx context.Context, state *CrudDisable) error + Enable(ctx context.Context, state *CrudEnable) error ListVersions(ctx context.Context, state *CrudListVersions) error } @@ -4279,6 +4485,14 @@ func (reexportCrud) Restart(ctx context.Context, state *CrudRestart) error { panic("not implemented") } +func (reexportCrud) Disable(ctx context.Context, state *CrudDisable) error { + panic("not implemented") +} + +func (reexportCrud) Enable(ctx context.Context, state *CrudEnable) error { + panic("not implemented") +} + func (reexportCrud) ListVersions(ctx context.Context, state *CrudListVersions) error { panic("not implemented") } @@ -4462,6 +4676,38 @@ func AdaptCrud(t Crud) *rpc.Interface { return t.Restart(ctx, &CrudRestart{Call: call}) }, }, + { + Name: "disable", + InterfaceName: "Crud", + Index: 0, + Public: false, + Params: []string{"app", "reason"}, + HTTP: &rpc.HTTPBinding{ + Verb: "POST", + Path: "/api/v1/apps/{app}/disable", + Body: "*", + PathParams: []string{"app"}, + }, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.Disable(ctx, &CrudDisable{Call: call}) + }, + }, + { + Name: "enable", + InterfaceName: "Crud", + Index: 0, + Public: false, + Params: []string{"app"}, + HTTP: &rpc.HTTPBinding{ + Verb: "POST", + Path: "/api/v1/apps/{app}/enable", + Body: "*", + PathParams: []string{"app"}, + }, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.Enable(ctx, &CrudEnable{Call: call}) + }, + }, { Name: "listVersions", InterfaceName: "Crud", @@ -4650,6 +4896,25 @@ func (v *CrudClientGetConfigurationResults) SourceValue() string { return *v.data.SourceValue } +func (v *CrudClientGetConfigurationResults) HasDisabledAt() bool { + return v.data.DisabledAt != nil +} + +func (v *CrudClientGetConfigurationResults) DisabledAt() *standard.Timestamp { + return v.data.DisabledAt +} + +func (v *CrudClientGetConfigurationResults) HasDisabledReason() bool { + return v.data.DisabledReason != nil +} + +func (v *CrudClientGetConfigurationResults) DisabledReason() string { + if v.data.DisabledReason == nil { + return "" + } + return *v.data.DisabledReason +} + func (v CrudClient) GetConfiguration(ctx context.Context, app string) (*CrudClientGetConfigurationResults, error) { args := CrudGetConfigurationArgs{} args.data.App = &app @@ -4950,6 +5215,67 @@ func (v CrudClient) Restart(ctx context.Context, app string, service string) (*C return &CrudClientRestartResults{client: v.Client, data: ret}, nil } +type CrudClientDisableResults struct { + client rpc.Client + data crudDisableResultsData +} + +func (v *CrudClientDisableResults) HasScaledPools() bool { + return v.data.ScaledPools != nil +} + +func (v *CrudClientDisableResults) ScaledPools() int32 { + if v.data.ScaledPools == nil { + return 0 + } + return *v.data.ScaledPools +} + +func (v CrudClient) Disable(ctx context.Context, app string, reason string) (*CrudClientDisableResults, error) { + args := CrudDisableArgs{} + args.data.App = &app + args.data.Reason = &reason + + var ret crudDisableResultsData + + err := v.Call(ctx, "disable", &args, &ret) + if err != nil { + return nil, err + } + + return &CrudClientDisableResults{client: v.Client, data: ret}, nil +} + +type CrudClientEnableResults struct { + client rpc.Client + data crudEnableResultsData +} + +func (v *CrudClientEnableResults) HasRestoredPools() bool { + return v.data.RestoredPools != nil +} + +func (v *CrudClientEnableResults) RestoredPools() int32 { + if v.data.RestoredPools == nil { + return 0 + } + return *v.data.RestoredPools +} + +func (v CrudClient) Enable(ctx context.Context, app string) (*CrudClientEnableResults, error) { + args := CrudEnableArgs{} + args.data.App = &app + + var ret crudEnableResultsData + + err := v.Call(ctx, "enable", &args, &ret) + if err != nil { + return nil, err + } + + return &CrudClientEnableResults{client: v.Client, data: ret}, nil +} + type CrudClientListVersionsResults struct { client rpc.Client data crudListVersionsResultsData diff --git a/api/app/rpc.yml b/api/app/rpc.yml index 1a02da246..2fa742631 100644 --- a/api/app/rpc.yml +++ b/api/app/rpc.yml @@ -660,6 +660,12 @@ interfaces: - name: sourceValue type: string doc: "Normalized image reference or auto-detected stack name, depending on sourceKind" + - name: disabledAt + type: standard.Timestamp + doc: "When the app was disabled with app disable; unset when it is not disabled" + - name: disabledReason + type: string + doc: "Why the app was disabled, as given to app disable" - name: setWorkloadRole doc: | Set the authorization role that identity tokens minted for this app's @@ -780,6 +786,31 @@ interfaces: type: int32 - name: stopped_sandboxes type: int32 + - name: disable + http: + post: /apps/{app}/disable + body: "*" + parameters: + - name: app + type: string + - name: reason + type: string + doc: "Optional explanation, shown by app status" + results: + - name: scaled_pools + type: int32 + doc: "Pools scaled to zero by this call" + - name: enable + http: + post: /apps/{app}/enable + body: "*" + parameters: + - name: app + type: string + results: + - name: restored_pools + type: int32 + doc: "Fixed-mode pools returned to their configured count" - name: listVersions doc: List an app's version history newest first, including replaced and ephemeral versions. diff --git a/api/core/core_v1alpha/schema.gen.go b/api/core/core_v1alpha/schema.gen.go index 5eef20501..40f96aefd 100644 --- a/api/core/core_v1alpha/schema.gen.go +++ b/api/core/core_v1alpha/schema.gen.go @@ -1264,24 +1264,28 @@ func (o *ConfigSpecVariables) InitSchema(sb *schema.SchemaBuilder) { } const ( - AppActiveDeploymentId = entity.Id("dev.miren.core/app.active_deployment") - AppActiveVersionId = entity.Id("dev.miren.core/app.active_version") - AppDeploymentLockId = entity.Id("dev.miren.core/app.deployment_lock") - AppInitialConfigId = entity.Id("dev.miren.core/app.initial_config") - AppOwnedById = entity.Id("dev.miren.core/app.owned_by") - AppProjectId = entity.Id("dev.miren.core/app.project") - AppWorkloadRoleId = entity.Id("dev.miren.core/app.workload_role") + AppActiveDeploymentId = entity.Id("dev.miren.core/app.active_deployment") + AppActiveVersionId = entity.Id("dev.miren.core/app.active_version") + AppDeploymentLockId = entity.Id("dev.miren.core/app.deployment_lock") + AppDisabledId = entity.Id("dev.miren.core/app.disabled") + AppInitialConfigId = entity.Id("dev.miren.core/app.initial_config") + AppOwnedById = entity.Id("dev.miren.core/app.owned_by") + AppProjectId = entity.Id("dev.miren.core/app.project") + AppSchedulesResumedAtId = entity.Id("dev.miren.core/app.schedules_resumed_at") + AppWorkloadRoleId = entity.Id("dev.miren.core/app.workload_role") ) type App struct { - ID entity.Id `json:"id"` - ActiveDeployment entity.Id `cbor:"active_deployment,omitempty" json:"active_deployment,omitempty"` - ActiveVersion entity.Id `cbor:"active_version,omitempty" json:"active_version,omitempty"` - DeploymentLock DeploymentLock `cbor:"deployment_lock,omitempty" json:"deployment_lock"` - InitialConfig entity.Id `cbor:"initial_config,omitempty" json:"initial_config,omitempty"` - OwnedBy string `cbor:"owned_by,omitempty" json:"owned_by,omitempty"` - Project entity.Id `cbor:"project,omitempty" json:"project,omitempty"` - WorkloadRole string `cbor:"workload_role,omitempty" json:"workload_role,omitempty"` + ID entity.Id `json:"id"` + ActiveDeployment entity.Id `cbor:"active_deployment,omitempty" json:"active_deployment,omitempty"` + ActiveVersion entity.Id `cbor:"active_version,omitempty" json:"active_version,omitempty"` + DeploymentLock DeploymentLock `cbor:"deployment_lock,omitempty" json:"deployment_lock"` + Disabled Disabled `cbor:"disabled,omitempty" json:"disabled"` + InitialConfig entity.Id `cbor:"initial_config,omitempty" json:"initial_config,omitempty"` + OwnedBy string `cbor:"owned_by,omitempty" json:"owned_by,omitempty"` + Project entity.Id `cbor:"project,omitempty" json:"project,omitempty"` + SchedulesResumedAt time.Time `cbor:"schedules_resumed_at,omitempty" json:"schedules_resumed_at"` + WorkloadRole string `cbor:"workload_role,omitempty" json:"workload_role,omitempty"` } func (o *App) Decode(e entity.AttrGetter) { @@ -1295,6 +1299,9 @@ func (o *App) Decode(e entity.AttrGetter) { if a, ok := e.Get(AppDeploymentLockId); ok && a.Value.Kind() == entity.KindComponent { o.DeploymentLock.Decode(a.Value.Component()) } + if a, ok := e.Get(AppDisabledId); ok && a.Value.Kind() == entity.KindComponent { + o.Disabled.Decode(a.Value.Component()) + } if a, ok := e.Get(AppInitialConfigId); ok && a.Value.Kind() == entity.KindId { o.InitialConfig = a.Value.Id() } @@ -1304,6 +1311,9 @@ func (o *App) Decode(e entity.AttrGetter) { if a, ok := e.Get(AppProjectId); ok && a.Value.Kind() == entity.KindId { o.Project = a.Value.Id() } + if a, ok := e.Get(AppSchedulesResumedAtId); ok && a.Value.Kind() == entity.KindTime { + o.SchedulesResumedAt = a.Value.Time() + } if a, ok := e.Get(AppWorkloadRoleId); ok && a.Value.Kind() == entity.KindString { o.WorkloadRole = a.Value.String() } @@ -1335,6 +1345,9 @@ func (o *App) Encode() (attrs []entity.Attr) { if !o.DeploymentLock.Empty() { attrs = append(attrs, entity.Component(AppDeploymentLockId, o.DeploymentLock.Encode())) } + if !o.Disabled.Empty() { + attrs = append(attrs, entity.Component(AppDisabledId, o.Disabled.Encode())) + } if !entity.Empty(o.InitialConfig) { attrs = append(attrs, entity.Ref(AppInitialConfigId, o.InitialConfig)) } @@ -1344,6 +1357,9 @@ func (o *App) Encode() (attrs []entity.Attr) { if !entity.Empty(o.Project) { attrs = append(attrs, entity.Ref(AppProjectId, o.Project)) } + if !entity.Empty(o.SchedulesResumedAt) { + attrs = append(attrs, entity.Time(AppSchedulesResumedAtId, o.SchedulesResumedAt)) + } if !entity.Empty(o.WorkloadRole) { attrs = append(attrs, entity.String(AppWorkloadRoleId, o.WorkloadRole)) } @@ -1362,6 +1378,9 @@ func (o *App) Empty() bool { if !o.DeploymentLock.Empty() { return false } + if !o.Disabled.Empty() { + return false + } if !entity.Empty(o.InitialConfig) { return false } @@ -1371,6 +1390,9 @@ func (o *App) Empty() bool { if !entity.Empty(o.Project) { return false } + if !entity.Empty(o.SchedulesResumedAt) { + return false + } if !entity.Empty(o.WorkloadRole) { return false } @@ -1382,9 +1404,12 @@ func (o *App) InitSchema(sb *schema.SchemaBuilder) { sb.Ref("active_version", "dev.miren.core/app.active_version", schema.Doc("The version of the project that should be used")) sb.Component("deployment_lock", "dev.miren.core/app.deployment_lock", schema.Doc("The expiring lock held by the deployment currently mutating this app")) (&DeploymentLock{}).InitSchema(sb.Builder("app.deployment_lock")) + sb.Component("disabled", "dev.miren.core/app.disabled", schema.Doc("Operator-initiated disabled state, set by `miren app disable`. When present, every service is held at zero instances, anything that would change the active version is refused, and requests get a 503. Absent means the app runs normally.")) + (&Disabled{}).InitSchema(sb.Builder("app.disabled")) sb.Ref("initial_config", "dev.miren.core/app.initial_config", schema.Doc("Reference to the initial ConfigVersion entity created before the first deploy")) sb.String("owned_by", "dev.miren.core/app.owned_by", schema.Doc("Cloud principal subject that owns the app; empty for legacy apps and non-Cloud creators")) sb.Ref("project", "dev.miren.core/app.project", schema.Doc("The project that the app belongs to")) + sb.Time("schedules_resumed_at", "dev.miren.core/app.schedules_resumed_at", schema.Doc("When `miren app enable` last resumed the app's schedules. The run scheduler discards pending ticks up to this time, including ticks due before the disable, so a pause it never observed between sweeps still counts.")) sb.String("workload_role", "dev.miren.core/app.workload_role", schema.Doc("The authorization role that identity tokens minted for this app's sandboxes authenticate as (see pkg/workloadroles). Empty means the default (app-readonly). Cluster-scoped roles may only be set by an operator, never via app.toml.")) } @@ -1444,6 +1469,50 @@ func (o *DeploymentLock) InitSchema(sb *schema.SchemaBuilder) { sb.Time("expires_at", "dev.miren.core/deployment_lock.expires_at", schema.Doc("When another deployment may take over the lock")) } +const ( + DisabledAtId = entity.Id("dev.miren.core/disabled.at") + DisabledReasonId = entity.Id("dev.miren.core/disabled.reason") +) + +type Disabled struct { + At time.Time `cbor:"at,omitempty" json:"at"` + Reason string `cbor:"reason,omitempty" json:"reason,omitempty"` +} + +func (o *Disabled) Decode(e entity.AttrGetter) { + if a, ok := e.Get(DisabledAtId); ok && a.Value.Kind() == entity.KindTime { + o.At = a.Value.Time() + } + if a, ok := e.Get(DisabledReasonId); ok && a.Value.Kind() == entity.KindString { + o.Reason = a.Value.String() + } +} + +func (o *Disabled) Encode() (attrs []entity.Attr) { + if !entity.Empty(o.At) { + attrs = append(attrs, entity.Time(DisabledAtId, o.At)) + } + if !entity.Empty(o.Reason) { + attrs = append(attrs, entity.String(DisabledReasonId, o.Reason)) + } + return +} + +func (o *Disabled) Empty() bool { + if !entity.Empty(o.At) { + return false + } + if !entity.Empty(o.Reason) { + return false + } + return true +} + +func (o *Disabled) InitSchema(sb *schema.SchemaBuilder) { + sb.Time("at", "dev.miren.core/disabled.at", schema.Doc("When the app was disabled")) + sb.String("reason", "dev.miren.core/disabled.reason", schema.Doc("Why the app was disabled, shown by `miren app status`")) +} + const ( AppVersionAdminTokenId = entity.Id("dev.miren.core/app_version.admin_token") AppVersionAppId = entity.Id("dev.miren.core/app_version.app") @@ -3855,5 +3924,5 @@ func init() { (&Secret{}).InitSchema(sb) (&SecretVersion{}).InitSchema(sb) }) - schema.RegisterEncodedSchema("dev.miren.core", "v1alpha", []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xdc\\۲\xec6\xd1~\x8d??\t\x10\xce\x04\xe2$\x84p&\xc5\xe1\x82*nx\x04\x97\xc6\xd6x\xb4ƶ\xbce\xcd\xec5\xb9ۜ)\n\xaax\x05\xf6\u07b9\xe1\xf9\xe0\x9a\xb2N\x96ڲ\xd5\xf2,\xaa(nV\xa9e\xf5\xa7Sw\xab\xd5\xea5\xaf\xea\x9et\xb4\xaf\xe9\xb5蘠}QqA\xe9\x99\xf5\xf5\xf8\x8f\xe7a\xed{SmA\x86\xe1S\xc5#\xc0W2\f\x9a\xef_ǚw\x84\xf5\x00\xf4xd\xb4\xad\xc7?\xbc<\xb0\xfa\xf1KK\xe6\x82T\x92]iYӡ巎\xf6Ru\xf3lY-o\x03=\xb0Z\x01\xbd\xbd\x0et\xa5bd\xbc\xd7\x13\x04u\x06\xe2\xd5\x04\xf1\x85\b\xc4\xdc[\xd9\xf2\xea\xac08\xac\x9c@XŻ\x81\xf7\xb4\x97sI\xaf\x0f\xc4-\"\xb8\x98\x05\xfb\x8d\x9a\xe7;`\x90\x00\xa8 ճ\v\x13\xb4.\x89^\xb6\xb3_1\r\xb4\x96\xac\xa3\n\xea\x9b\t(\x8ff\xb5\x02\xebª\t\xee8J\xc1\xfaF\x01~=\x01H\x1f\a&\xe8h\x87\xf6\xe0\xd1nd\x8dٙ\xe6\xfa\x01i\x87\x13i\a\xc1:\"n\xe5\xb4B\xfd\x8c8\x01\xaen<\xeb\x99d\xa4-+\xde\x1fY\xa37\x1e\xd4\xf9\xb2\xf3\x99\b\x04\x7f\xdeӺ<\xdc\x14\xf3\xc9Qp\xca\xff\x1fa\x1d\x04\x7f\xa0\x95\x9ecc\t\xbf\xbf\xcfG\x98\x9esqn9\xa9K\xc1[\xaa\x17;\xac\xf2z\xde\\\xa3\x8a\f\xc3bXJcGZ\tj\x84\xf2\x02\x1a\xe8o\x181\xfc\xb5\x9a\u0097\xa3\xfcEu\x11b\xdan_\xe58\xac\xf4\x97\x02\x0e\xd4\xe0\fD\x9e\x14s\xadJع\x1f5\xfbbC\xd5\xf4\xcdN\xe8\xf9_A\v\xf3\x11\xb3\x00/\xd4\xc0ߌ\x03(\xb9\x11\xaa\v\xaa\x8bر[IY kk+$;\x12;zh\x90\xedW\xcc\xf0\x7f\x1b\x17y\x830\x19'\xd5\xc5$F\xdbBk9:ҳ#\x1d\xb5\xbc\x9f\x1c\x055\xe5\xab)\xfe\xb2f\x8d\x85\xe1\xb0\xd2CS\xb6\xfa\xb3kh\xa3$\xf22*\x90\xa3)+\xdbB\xfbKw\x9e\xfe\x94W\xd2^\xe8\xf8\xf7\xa3>\t\x16˭\x99\xcc\xd9q\"\xa2:\xb1+]vh\x9b\x99\xef\x9b[{\xb2\xa3\x8b\xefmG%\xa9\x89$\xf1\xbd\xb5_QGDtm,Bђ\x03mǺ#\xfd\xed\x9fz\x85LʹBT\x95\xa3\xb2\xed\x00\x94F\xce\x7f\xe0\x16\x7fn\x8do\xcb\"n\xaf\x9c\x85XLJ\xad\xdc|\x1c\x98\xa3\xf6\x8d\xd5\x13\b\xb3|\x7fS\xb3xk\x15c]9\xa0\xeb\x10\xf2\x94n\xc9N\x8e\x82k\a\r*@\xf0\r\xeaٯ\x808С\xf2p\xaa\xf62J*\xeca\xfe\xe0\xd1\x10\xe5+\x1b(\xbc\x1bZ*g\x0f\xa3\rj\xa0\xa2n\xccK\x17\xe7S\xf6\xecW$|*\x00[\xc4a1\x9b\xfe\xa7\x8d\t+\x90\x82\\\xe4\xa9\xec\xa8<\xf1ڬ\xbfW\x01W\x0e\x9aI\x1f\x88v\x84\xb5\xfal\xd0Ŕ\x02\xf9\xcc\t\u074b\xfb\x85\x9a\x95\x8b\x86\xf4\xec\x13\"\x19\xef\xed\xeesX\t\x01\xe3\"\xad\x01\xc7\xcba\xd6gK\xe0$Q\x03L\x9e\xde(I\xa7Չ\xcd$N-4\xc8e\xa4\xa2\x9c\xd7\xf4\xc1\xa3s&\xa3\xd8̪4\x96ș\x8c\xe2q;\xc3f\x12{\xf4?X\xb4\xc3-zTz\xc2M\x85\xe0\xa2\xec\xe88\x92Ƹ\x89a\x15Կ\r\xcb\xd40Y\xb2\xfeȵerT\xe6m&\x02\x88W;h\xd5-\x82\xd29\xae\xfd\xa8\xa3)\xc3-Y\xe5=\b\xd2W\xda}<\x9a2\xe4\xfd\xc6\x1aoŻ\x8e\xc9Rw\xe9\t\xd7\x18\xfb\x00Q\xbf\x96@\r\xa5~XԦl\x89\xc3ccY3!\xcd\xe5\xc4Q\xca\xd19p\xdeF\x8d\x89\xe3\xf6\xa5\xa7\x89\xc8MTc\x1c\xb7\xa0\x03\x1f\x99\xe4B\xf7\xfe\xe0\xd1\x10\x03:\x99\x0ec<\x11}\x8eN\x85\xd4-\xd2qM\xf7!\xd67\xa5\x14\x94\x96'2\xea-~\xb6\xacF\xbb\xdc\r\x93\x13r\xf4\x1e\xe9\xc95j\xc1\xbe\xb8\xce\xcf\a*\x94\x9d\xd5\x06b&!\xc6\xc6\x18\xf8EVܘ\x98\xc6\x12)\xb1\xf6\xf8\a\xa2\xee`0\xa8\xb2\xac\xf6ݚ\xf8y\xa4\xf1Ndԣ\xa1\xba\b\xc7R\xac\xf3\x8e\xfc\"*?\x92cͯ\x8c~\xc9pqFI\x84\xe7\x9aWw\xff\xf6Y(\x94\x03\xf5:vY\xb5\bE\xcdƳ?L\xaa+\x12c|\x1f?F\xdd\x03f\xa4\x7f\x8e\x06\x15\x14{Q\x1f\xca#3\x01\xcf\xc6\x12\xc9k\x91b\x9d\x9a\x8e\xb7Q\xd2N\v\x80G'\xdd\x7f\x05\xd0R2R\xe5~\xf1\x8b\x16\x84.\xac\u008d\xa3\xe3\x97^\x96.b\xf9\xe0\xd1\x10`\x11\x9eP\x00\x89\x9b\x1d\x94^ʹ\x15g\x8cJ\x86f\x1b\x04\xbf\xb2\x9a\n\x15\x0f[\x98ϩR\xedk\xe9\xda)\x11wT4\x94\xf6\x92\xb6\xbc\"\xed\xa2G\xcbU\xa8\xcfT}Yo\xa4\xea\x8e㳖I\xbaP?\xd7J\x7f\x8f\x9e7z\x82\x82\x92\xba\xe4}\xab\xbdS6\x93\xa1s\x1c\x97Ƒ}B\xcb\xe6`\xac\x96!\xac]\x88\xba\xb4\x86O\xb9-\xc6e\xd0\xe5\xd4\xe9h\x18\xd5t\xac\xfb\xc3f\x12k*\xb5Z\xbf\x8e\r\xcd)*\xed\xaf\x9e!\xa8&2a\x06\x8a\f3@\xfb+\xfe\xb9\x0f\xca2\xed\xafŁT\x93;\xa1\x17\xdd\x12\xa9\xe5\x9b\x18k:V\x82\rα>\xfb\x15\x00\x00\xc6\xfb'\xfe3\xd5BRM\x85\xd4\xddeb\x10T?\xa5i\xa5pԶdM\x8c#\xedG&\xd9\xd5D\bf2d\x85\xe6A\xb1\xa6E\xeb\xff\"lJ3\xb5y\xd0E\xf4\xf3\x11\xed\xaf\xd10\xa8\xdbp\xd6ٻ\x10\xd5E8\x9eED\xddr&\f\xdd*\xdf\xcaA\x1du\xc5\x03&/*3\x93)g>DP;4#\xe8[D\xc2\xd4\x06\b\xc1!\xac+\x9e\xf0\x10V\x80\x18\xfd\xfb]T\xc2\x14{j_\x16\xc6X3\xf1\x9a\x96ng\xd8L\x06\xdb\x13\xefp\xcb\xf3Z\x1c\x10\x9aCp\xc9+\xden\x9c[j\x7f\\;{ni*\xfe\x04T\xc9jX(\xba\xe5)d5T\x97z\xa3\xc1\xa5\x1e6\xe6\xe8\x04\xa7\x862\xb3m\xd0\x15\xf7\xab\xd8m\xdem\xba)\x94\x15\xef\xf5\xdbj\xa5\r\xd9\x18\xfb\x90\x10\xb6\x8f3\x84-\x02\x8f\x17=\x18Ԍ\x80\x15\x1d\xaf͚\xa9\x12\x14\xc4\xf7\x11\x10\xd3\xf6\xb2~\x94\xa4\x9f\\r\xe5ԅU\x81x~\x0f\x818\xd9y:ʱ\x1c\xa8p8\xfa\x11=\xfe)\xe8\xe1CD\x0f\xe3\xe4\x1e\x955\x7fޗ5m\x89\xde\xccaQ\v\x97\x03\x05}\xbaH\x05ỹâ\x16+\x9d\xc2\xf4\xe1u\xb1}\x93\xb3\xb2\x13\r\xe6[\xf9\x92DȲf\x82V.\xac\xc8a%\xb4\xb9+\x17\xb4+\x11\x8c\x1cZ\xea_\xd0\\\xdd\xfd\x174\v\x85\xf7x\xe0\x9d\xc1\"\xe0\xdc\x1e\x18\x1eq\xdc9\xbe\xcf\u00829\x94M\x0f\b\u07bb\x1d\x17\xca\r\x82&\xdaq#}!\xb8\xbf3\x7f\xda!\x82G\x88\xe3\xdd\xed\x159\x19\xdaN?\xd1R\x12\x8db/\x85)\xccK\x03u~\x8c\xec\xbd\r(:\x9chG\x05iK\x90\\%\xa3_\xc2 (|3\x8c\x03\xab\xbc\x00\xad\x94\xb02\xf5\x00\x12\a\x94\xb25/WAU*\xcc\xeb\x83)dz\xbc\b\r\xc4f2\xa5@>\b2qek\x95v\xe5\xael\x05-W\xa4{g\xd0R#\xe0\xcfh\xa8\xf2\x9a_\xbd\xeby\x8fj\x0f\x1e\x9dR=\x0f\xc1>\xfa4\x96H\x05:\f\xef4z\xed\x14\xa8R\xcaL\x19\xae;\x9e\xa9\f\xc2nca6.)<\xa3$\x92Ue\x10\xe6\xe6\xb02\x15\xca\xf2\x01W\xdf\x1e0\xa3\xf6\xb3J\x16\x8b\xaa\x1e \xce\xf4V\n.\xd5ӕ\x11\xbeE\xa2\x9d\xd7\x04/v\xd0V\xfa(Y/\xdeQ\xbb\x11\xa0\x1d\x05\xefJ{\xec\x9d\x1c\x95J;\b0\x04}.\xc80\x98\x03\x90ͤu\xfc\xa2\x0f\xef\x01\x046K\xedU]\xf3~\xf9\xd0c\x92Ϧo\xc7#a-\x8d\\\x9eu\x13\xfd\xf5$\xa8d\xd3\xe4\xd6\xd2\xd8\xec\xf7\a3\x99\xa9\xe9B\xb1lS\xdb\"*\x8f\xc14%wk}4e\xac8\xb6>P\\\x1e9\xab\xab\xf2\xc0\xfa\x9a\xf5͊<\xfaMPo7Q\x9f\xcdG\x89>e\xbd\x8e\xa5S\a\\UKX7y\xe65\x9b&\xe4\a\x05\x86ŷ\x84\xc5\a\x1d\x15\x9b\x1d\xe1_\x82\xe0Q\t\x91\xb6\xbdEx\xbd[p\x0fDJ*\x8cq\xb2\x04V\x1a\xb8\x82\x9bѢ]\x06\xeb\x90\xe5 C!\x0e\x90\xd88^L \xfch\xca)\x93\x13\xf0o\x84ҷ\xde\xee\x03\f\x93cU\xfa\x8b\xc8a%Z\xb5|\xe8\xa5\x02O\xaa\x15z\xa3F\xee\xe0\xd9\x1e6\xc2\v\x1a\xf4\x7fB\x9c\xa8\x82E-*\xe0\x1b\aZi?A\x95\x12J\x04\xb7\xcc}\xb7s\x9f@\xf0\xefp\xdf\xc7\xc0\x15U\xcb{Z\x92ɨ\x97\xeaF\xd1\a\xe1A\x19\xfd\x9e\x98\b\xe8\xb9\xc0\xf7\x8c߱\x8fw\xcf.\x15[\xfc\xe9~dS0ގ!\xb0:\xf0\xf8\xa6Ɵ\xe0\x15\xba\x02\xb7\xd8J\xdc>\xc8\x1d\x9a\xbf\x95mP\x0fg\rotqh\xdc\v\xb7\x12\xbfwQ\x80\xf7\xbc^\xbf\x8b\x12\xb3\x9c\a쿨U\xfeV\xd6\xc8\v\"\x1a\x7f\xf8\xb5\xa2\xe1\xf2~\x94\x87\x99\xcatx\x85Wp\x0f3\f͞3B\xb28\x8d\x8eu\x85w\xb9q\x1a\x1d\xeb#\x15\xa4\xfd\xc5~\xe4\xbc\xd8\xed\xaf\xf6wt_H\xf7\x97\xfb;\xde\x19齧ǧ\r\x00?\xbe5\xdbM۟\xd7\xdd\xebX\\:1\xda}\x99*\x1f\xe6)If\xb2\xca\x0fwL\x01\x97˒\xa9x٩.\xb8\x03\x15\xe0gg\xc2\xec\x9aEF\xa2\f|\x97A\xe1'\\\x8dL\v\x9eN\xb3\xf9tB\xfd\xd1\x1e\xd4\xff\x8d,\x1c\xa1zoiC\xaa[8\x88\x17\x8f?\xbbgY\xfc\xa1\xde\v\xe5O\xe8\xc5\xe3\xcf\xef\xc2\xf2\x92\x8f~\xbc\a\b\x99\x9b\xb4\xcb\xf8\xa4S\x97~\xb0\v6\xfdڲk)\xeeN|z\xbe<\x87\xe6T(\x9c\xeb~G\x82\xd4\ay\xa7OV\x8eT\xa6\x9dB\xa7Pe\xeeSn\x86U\xaeם\xcc\xc0ʔWt\x82V\xa6ze\xe4oe\x9eZ\xc8\xf4\xae\xef\xe6\xa3\xee~\xba\xb8,\xb5\xca\xe6\x83ezu[Yb\xafv\x88KG\xa5`\x95\xbe\x154\x96Hh\xe9GyZjP\xf1\x97\xa8L\x8d2\xf8\x05\xedU6\x82\x9e\x8a%BI\xca\xf4\xaf,2\xeb%\x15Wb\xb2\x9f\x1cu\xa7fY\xf4\xf5\x1fe\xb8\vu#\xb9/ӹr\x90\x97C\xcbt,\xf0h\xcany\xb7o7o,\xe5߀\xbe\xdc\x11\xacH8\xa4\x99h[\v\x95i#й\x91\x99{\xaaS$\xbdkD\x1b\xd4\xdci\xd9\xd0\xf9\x98\x99\xa6j_\x96f\xe6\x0543Q3\xf3(A\xe5qf\x9a\xab\x9c4\xcf]\xc3]\x11\xe8=ת\xff\xea$\xd1\xc8=I\xa1\xbex\xfc\xc9=\xb3\xb4}\xdf\vc\xd3Xw\xed\xe1\xce,\u05c8\xf7\xae\xf0\xb6\x93\x0f\x97Lj\xe0\xdf\xc6\r<3\xcb0#Por6j\xa6/\xea\x0f\x1e\r\x01\xbf\x93\x03\xa8\xf3\x1c\x06\xebH=[VC\xcb\a\xd3M\xe2𒀈\x9b\xae\xc8L\x8f\xdc\xc0\xc6ع?f\x9c\x81\n4\x19\xa7W+\x00\xff\xedx\v\xf0\xfe\x7f\x8cI\xa0\xe3/}\xb8\x83\xd6\x01\xe7\xdd\xccr\x169y-\xc3\xf9\xed3\x18\x17\xacaz\x90GS\xde\xe7\a̐\xa8k\x1eΜ͠\xc8;^\xee\xf4\x11\x17<\x9c\xfd\x9a!w\xdf\xee<\x03\xaa\xd5\xd4\\\xedp\x11\a=\x80\x8e<\xce\t\xe6\xfa\xf8\xeeA]\xe0\x19\xe0,\xa8\xc6~\x12\x7fYC\x89\xc9k7\x8fH\x8d%6\x13\xf3\xb7\xd0\xc6\xeaD\xeb\x8b\t\xed\x9f\x1cu\xcf\x00}/\xb9Ys\x90\xb3\x00\x05k\x1a\x13*n,\x81ϯ\t%#Ô\xda\xcch\xff@as\xe5\x93\x18T\a\x877\xa8\xb8\x83\xd6\x01\xe3Bh8+=\x83\xfe\a\xac\xf4\f\xbei\xa5q&u\x06{B\x93:\x83>\xa9I\xf5`\x9fʤΐ\xbbM\xea8+\x8eC\xdbN\xb3\xf5\x06\xb0\xfd[\xa7azS\xf5\x93>\x88|g\xbfE\x98ÙL\x8fF\x19\x93\xb0\r\xd8F\x8c\xfd\xf97\x00\x00\x00\xff\xff\x01\x00\x00\xff\xffW7\x93\xb1\xe5\xff\xbc\x0fkߕ\xb5\x15\x9a\xa6O\x15\x0f\x03O\xd14i\xbe\x7f\x1f[: 2\x02\xd0\xe3\x91\xe0\xbe\xe5\x7f\xfe\xe4@ڇ//\x99+\xd4\br\xc5u\x8b\xa7\x9e>\x0ex\x14\xaa\x99\x17\xcbj\xf18\xe1\x03i\x15\xd0[\xeb@W\xcc8\xa1\xa3\x1e \xa83\x10O\x12\xe2\x8b\x11\x88\xb9\xb5\xba\xa7\xcdYaPX)AHC\x87\x89\x8ex\x14sI\xcf\x0fĭ\"\xb89\x13\xf6[5ηA'\x01P\x85\x9a\x17\x17\xc2p[#=mg\xbfBv\xb4\x15d\xc0\n\xea[\t(\x8f&\xad\x02\x1b\xc2*\tw䂑\xb1S\x80\xdfH\x00⇉0\xccm\xd7\xee<\xda\xf5\xac3+\xd3]\xdfG\xfdtB\xfd\xc4Ȁ\xd8c-gh\x9c\x11%\xa0Z\xb5\xd7c\xabF8:\xf4Xw\xfa\xe4\xa8\xc4:\xbd\x1e['Û\xb3@\xbfQ\x93\xf0Y8\t\x06\xa12\xa3>,\xd6\xe1sk\x1c\f#n\xe4\xf6h\xcaޔoΔ\x1b\xf4\xaar\x90\x91\b\x82\xfa\xba\xa1\xe3\x91tZ9@\x9d\xaf_\xb1i\xa6\xf7#n\xebã\x9efGA\xb1\x803\"Y'F\xefp\xa3g\xa4\xb3\x84\xdf\xde\xd7\"L\xbc9\xe1\xf6\xd2c^3\xcc/\xc3,\xe4\"\xfa$\x9c\xe5/D\x00\xef);\xf7\x14\xb55\xa3=\xd6\x12\x1eV\xe5Nw\x83\xa6i1Ne&9n\x186\x12v\x01/\xe8g\xf9\xa2\xf5\x95(\x7f\xd5\\\x18\x93:\xe6\xdb9\n+\xfd\xb9\x85\x1d58\x13\x12'\xc5ܪR\xee؏\x9a}!!j\xf8fi\xf5\xf8\xaf\xe0\r\xf30g\x02^\xaa\x8e\xbf\x11\aP\x82\xc8T\x13X\x17s\xfbnEo\x81\xac\xb78&\xc8\x11\xd9\xde\xc3]\xd0>\xcd\xe9\xfe\xef\xe2:d\x10\xa4\xa5QMH1\nVj!\xb4\x96c@#9b\xae\xc5\xff\xe4(\xa8z\v-\x82\xfcuK:\vCa\xa5\x87\xf6\x143T\x0e\x8d\v$.\\\x1b*SVʇ\xc7\xcbp\x96?\xf5\x15\xf5\x17\xcc\xffq\xd4\xdb\xefb\xba5\x93ٰO\x885'r\xc5\xcb\x06\xedk\xe6\xf9\xb6\x05\xb4\xbd\x8b\xaf\xed\x80\x05j\x91@\xf1\xb5\xb5O\xb3\xf6\xe5\xe8\xdcX\x84\xaaG\a\xdc\xf3v@\xe3\xe3\xbf\xf4\f\x99\x1a9CX\x95\xa3\xb2\xed\x00\x94F\xce?p\x89?\xbfƷeb\xb7g\xceB,\x06\xa5fnރ;\xf9\xda궟3}\x7fW\xa3xs\x15c]9\xa0\xbf\x16\xf2\xd4n\xcaN\x8e\x82s\a\r*@\xf0\r\xeaٯ\x808Ћ\xf5p\x9a\xfe\xc2\x05fփ\xba\xf3h\x88\xf2\xd5\r\x14:L=\x16\xf3\x8e\xd7\a5PQ7ƥ\x8b\xf3\xb6}\xf6+\x12\x0e\x12\x80\xad\xe2\xb0\xf9\xce\x7f|\xc0\n\xa4B\x17q\xaa\a,N\xb45\xf3\xefU\xc0\x99\x83f\xd2\a\xc2\x03\"\xbd\xde\x1bt1\xa5@>sB\xf7\xe2θf\xa5\xacC#\xf9\x18\tBG\xbb\xfa\x14VB\xc0\xb8Hk@~9\xcc\xfal\x89jL\x1c\xb7/=]Dn\xa2\x1a\xe3\xb8\x19\x9e('\x822\xdd\xfa\x9dGC\f\xe8d:\f~Bz\x1f\x95\x85\xd4\xd1\xddq\xc9\xf3\x10\x19\xbbZ0\x8c\xeb\x13\xe2z\x89_,\xab\xb3]\xee\x8e\b\x89\x1c=\x98zr\x9d5a_Z\xe7\xa7\x13f\xca\xcej\x031\x93\x10c\xa3\x0f\xf4\"\x1ajLLg\x89\x94X{\xfc\x13Rg0\x18\xc9ZV\xfbnM|?\xd2x'\xc4uo\xb0.¾T뼜^X\xe3\x87Ϭ\xf9\x15\xd1'\x05.\x0e\x17\x88y\xaeɝGo\x1f\xc1C\x8c\xd8\t\"s\x9d|G\xad\x03'ޜM@\x82,\xe6\xddF9\xad\xd7gLq$\x9aa\xdf\xc81\xc1\x7f\x8bz>\x1eH\x85ځ\x8c\xb5\xa0gl=O\xaf\"e\x8f\x03\xa05\xc7\x19jM\xc0dNM\xc6s\xb6\x94\x1f'\x8dD\x93\x1c\xbb\x17M:zQ\xa4\x8d\xad\xed\xade\xd4\r\xa0\xe5L\xeb\x1f^\xc5fC\xf3\xab\x9d\x00\x8d\xad\x7f\b;\xb9\xbaD\xf7\xdeNv\xcf\xc1\xe7\ar\xa0\xa8Y\x04\v\xa5\x05\xd9\x12)\x87\xd5qs̮\xa41\xe6\xca\x12\xd9\x11C\v\x13UU3T<\n\xf68Q2\xda\b\xeeL\xa7\xa2~\x06a\xa2L\x98 \x93,I\xae\x86\x8cbk\xf9\xccH\x82\xe5su\xb7/\x9f\x85\xcar\xa0^\xc5\x0e\xab\x16\xa1j\t?\xfb\xddĺ\"\xd1\xc7\xf7\xf2\xfb\xa8[\xc8\xe9\xe9_\xa2A\x05\xc5^\xb5\x87\xfaHL\xc0\xb3\xb3D\xf2X\xa4X\xe5\xab\xfc\x91\v\xc6uw0V\xcb\x10\xd6.D]Zç\xdc\x16\xe32\xe8rjw4\x8cj8\xd6\xfd!3\x99k*\xb5Z\xbf\x8au\xcd)*\x1e\xaf\x9e!h$\x990\x03U\x81\x19\xc0\xe35\xc7\b\xfc1*\xcbx\xbcV\a\xd4HwBO\xba%R\xd3'\x19[\xcc\x1bF&\xe7X\x9f\xfd\n\x00\x00\xe3\xfd\x92\xff\x8c\xb5\x904\xb2\x90:\xbbH\x06\x86\xf5\xfd\xa5V\nGmK\x96d\xe4x\xe4D\x90\xab\x89\x10\xccd\xc8\n̓bM\x8b\xd6g\"lJ3\xb5y\xd0\xc5\xec\xeb#<^\xa3aP\xb7\xe0d\xb0g!\xac\x8b\xb0?\x8b\x88\xba\xe5L\x18\xbaU\xbe\x95\x8d:\xea\x8a\aL^Tf&S\xce|\x88\xa0VhFЧ\x88\x84\xa9\r\x10\x82MXW<\xe3&\xac\x00s\xf4\xef\xf7Q\tS\xec\xa9uY\x18c\xcdD[\\\xbb\x95!3\x19,O\xbc\xc1-\xcfk\xb1Ah\x0eF\x05mh\xbf\xb1o\xa9\xf5q\xef\xd9}KS\xf1+\xa0F4\xd3B\xd1-O%\x9a\xa9\xb9\xb4\x1b/\\\xdaic\x8cNpZ(3\xdb\x06]q?\xc5N\xf3n\xd1M\xa1n\xe8\xa8\xefV\x1bm\xc8x\xecAB\xd8>*\x10\xb6\b|\xbe\xe8\xc1\xa0f\x04\xac\x1ahk\xe6L\x95\xa0 \xbe\x97\x01!\x97\x97\x8c\\\xa0Q\xba\xe4ʩ\v\xab\x02\xf1\xfc~\x06\xa2\xb4\xf3\x98\v^O\x989\x1c}\x89\x1e\x7f\x14\xb4\xf0AF\v\\\xbaGuK\xefǺ\xc5=ҋ9-j\xe1tdA\x9f.BA\xf8n\ued28͕Nf\xda\xf0\x9a\xd8>\xc9Yى\x06\xf3\xad|\t\xc4D\xdd\x12\x86\x1b\x17V\xa4\xb0\x12\xdaܕ\x03\xda\x151\xa2\x92M\xbc\x03\x9a\xab\xbb\xfd\x80f\xa1\xf2=\x1exf\xb0\byn\x0f\f\x8f8\xee\x12\xdfga\xc1\x1cʦ\a\x04\xcfݎ+\xcb\r\x82&\xdaqg\xfaBp}g\xfe\xb4C\x04\xb7\x10ǻ\xdb+r2\xb4\x9d~\xa2\xa5$\x1a\xc5^\nS\x98\f\b\xea\xfc\x18ٻ\x1bPx:\xe1\x013\xd4\xd7 \xa3MD\x9f\x84APxg\x18\aVy\x01Z)ae\xea\x02$\x0e(Don\xae\x82\xaaT\x98\xd7\aS\x8eg}a\x1a\x88\xccdJ\x81|\x90\xccĕ\xadYڕ\xbb\xb2\x15\xb4\\\x91\xee\x9dAK\x8d\x90\xbfGC\x95\xd7\xfc\xea^ϻT\xbb\xf3\xe8\x94\xeay\b\xf6ҧ\xb3D*\xd0axe\xef\xb5S\xa0J)3e\xb8n\xb8\xa62\b\xbb\x8d\x85Y\xb8\xa4\xf0p\x81\x04i\xea \xccMae*\x94\xe5\x03\xae\xde=\xe4\xf4\xda\xcf*YL\xaa\xba\x808\xe3ǚQ\xa1\xae\xae\xe2y\xaa\xef\xfa\xaf\xe4\x8b\x1d\xb4\x95>Jэw\xd4n\x04hGF\x87\xdan{'G\xa5\xd2\x0e\x02\f\x86\xef\x19\x9a&\xb3\x01\x92\x99\xb4\x8e_\xf4\xe2=\x80\xc8\xcdR{j[:./zL\xf2\x99|v<\"\xd2\xe3\xc8\xe1Y\xbf\xa2\x9f\x9e\x18\x16D\x0en-\x8d\xcd>\xbf3\x83\x91\xaf.\x14˾j߈\xcac0LA\xdd\\\x1fM9W\x1c{\x1f(.\x8f\x94\xb4M} cK\xc6nE\x1e\xfdW\xb2\xeen\xa2>\x9b\x8f\x12\xbd\xcaR\x1e)\xcca\x0f\xb8\x9a\x1e\x91Az\xe6-\x91\x03\xf2\x83\x02\xd3\xe2Y\xc2\u20c6\xaa͆\xf2o\x82\xe0V\t\x91\xb6\xbdEx\xbc[pOH\b̌q\xb2D\xae4P\x057\xa3E\x9b\f\xe6\xa1\xc8A\x86B\x1c \x11\xce/&\x10~4\xe5\x94\xc9\t\xf87B\xe9[w\xf7\x01\x86ɱ\xaa\xfdI\xa4\xb02[\xb5|\xe8\xa5\x02K\xd5\n\xbdQ#wpo\x0f_\xca\x174\xe8\xff\x848Q\x05\x8bZT\xc0\xc7'\xdch?A\x95\x12J\x04\x97\xcc=\xb7c\x97 \xf9\xf7p?ȁ\xab\x9a\x9e\x8e\xb8FҨ\xd7\xeaD1\x06\xe1A\x11}\x9e\x18\bh\xb9\xcao9\x7f\xc5>\xda=\xbaTl\xf1g\xfb\x91M\xc1x;\x86\xc8Ձ\x8774\xbe\x84W\xe8\n\xdcb+q{\xbf\xb4k\xfeR\xf6A=\x1c5<\xd1š\xf3n\xb8\x95\xf8\xbd\x93\x05x\xcb\xed\xf5;YbVr\x81\xfdW5\xcb\xdf.\xeay\x85X\xe7w\xbfU4\x9c\xde\x0f\xcb0S\x99\x0eO\xf9\n\xeea\x86\xa1\xd9sAH6O\xa3cM\xe5\xbb\xdcy\x1a\x1dk#\x15\xa4\xfd\xe5~\xe4\xb2\xd8\xed\xaf\xf77t[H\xf7W\xfb\x1b\xde\x19齥\xc5\xe7\r\x00?\xbc9\xdbM۞\xd7ܫX\\:\xd1\xdb}\x99*\x1f\x94)Ia\xb2ʏv\f!/\x97\xa5P\xf1\x8aS]\xf26T\x80_\x9c\t\xb3k\x14\x05\x892\xf0^&\v?\xe1j\x14Z\xf0t\x9aͧ\x12\xf5\xc7{P\xff?\xb2p\x98j\xbd\xc7\x1dj\x1e\xc3N\xbc|\xf8\xf9-\xd3\xe2w\xf5V(\x7f@/\x1f~q\x13\x96\x97|\xf4\x93=@\x99\xb9I\xbb\x8cO:u釻`ӷ-\xbb\xa6\xe2\xe6ħ\xfb\xe5>4\xa7B\xe5\xb9\xee7$H\xbd_\xb6\xfb\x14\xe5H\x15ک\xec\x14\xaa\xc2u*Ͱ*\xf5\xba\x93\x19X\x85\U0009a760U\xa8^\x05\xf9[\x85\xbbVfz\xd7\xf7\xcaQw_]\\\x96Ze\xf3\xc1\n\xbd\xba\xad,\xb1\xa7\x1d\xe22`\xc1H\xa3O\x05\x9d%\x12Z\xfaa\x99\x96\x1a\xd4\xfcCT\xa1F\x19\xfc\n\x8f\xf3\x17A:K\x84\x92T\xe8_Yd2\n̮\xc8d?9\xeaFͲ\xe8\xeb\x1fe\xb8\tu#\xb9\xafйr\x90\x97COt,\xf0h\xcanz\xb7O7\xaf-\xe5߀~\xb2#X\x91pH\vѶ&\xaa\xd0Fd\xe7F\x16\xae\xa9N\x91\xf4\x8e\x11}Ps\xa3e\xcb\xce\xc7,4U\xfb\xb24\v\x0f\xa0\x85\x89\x9a\x85[IV\x1eg\xa1\xb9*I\xf3\xdc\xd5\xdd\x15\x81\xdes\xac\xfa\x9fN\x12\x8d\x9c\x93\x14\xeaˇ\x9f\xde2J\xdb\xf6\xad06\x8du\xd7\x1a\xee\xccr\x8dx\xef\no;\xf9pɤ:\xfe\x9d\xbc\x8e\x17f\x19\x16\x04\xeaM\xceFK\xf4A\xfdΣ!\xe0wK\x00u\x9e\xc3d\x1d\xa9\x17\xcbjh\xf9`\xbaI\x1c^ \x10q\xd3\x15\x85\xe9\x91\x1b\xd89v\xeeO\x05{\xa0\x02M\xc6\xe9\xd5\f\xc0\xbf\x1do\x01\xde\xfeǘ\x04z\xfe\xa1/o\xa3u\xc0e'\xb3\x92IN\x1e\xcb\xf2\xfc\xf6\x19\x8c2\xd2\x11\xf3\xe58S\xde\xe7\a̐YǼ\x8bAup\xf9\x065o\xa3u\xc0y!\xb4<+=\x83\xfe\x17\xac\xf4\f\xbei\xa5\xf3L\xea\f\xf6\x8c&u\x06}V\x93\xea\xc1>\x97I\x9d!w\x9bT>+\x8eC\xdbN\xb3\xf5:\xb0\xfd\x81\xd90\xbd)\x9e\xa7\xa5\xbfw\x99\xc8\xd3\n_\xcaO\x83\x84\xd9\xfc!NՐ鄙\xc0\x0f&=ţ\xe5\xec\xe1ã0\x0e:̵\x02@g|\xb6\x17\x01GSN%\xd8\x01\x04M\x1a\x89\xd0e?q\ff6Bn\x81\x84Y|]\x8c߯\xd9P]4\xd5\x16ۨ\x9e\xfb\xda\xedr\x19\xd4k\xf61i1\x17\x8c>\xae|ZR\xfd\xc3G?_\xf9\xf7\\0\x04\xd5\xc5z@\x8d5\xbf\x96\x84\x13\tS\xad\x01\x8e\xc9d\xae[\xac\xbf/}\xf6+\xe6U\xdd\x16\xdc\x10\x13\xbe{\xe6'y\x06ן\xf3n\xd04\xad}\xd2\xdb~\xccu\xe3[\xb6\ue6e9[\x1f|M|}\xd3>\x9d?5\xb9\xf9\x91N\xff3=\x89oR\x06\x19\xf5\xa9O\xfad\xe4;\xfbo\x849\x9c\xc9\xf4\xe8,c\x12\xbe\x03\x961\xc7\xfe\xfc\a\x00\x00\xff\xff\x01\x00\x00\xff\xffA\x1a9^\xb9]\x00\x00")) } diff --git a/api/core/schema.yml b/api/core/schema.yml index 9879ca95a..5dcfdbf6b 100644 --- a/api/core/schema.yml +++ b/api/core/schema.yml @@ -109,6 +109,29 @@ kinds: type: ref doc: Reference to the initial ConfigVersion entity created before the first deploy + disabled: + type: component + doc: >- + Operator-initiated disabled state, set by `miren app disable`. When + present, every service is held at zero instances, anything that would + change the active version is refused, and requests get a 503. Absent + means the app runs normally. + attrs: + at: + type: time + doc: When the app was disabled + reason: + type: string + doc: Why the app was disabled, shown by `miren app status` + + schedules_resumed_at: + type: time + doc: >- + When `miren app enable` last resumed the app's schedules. The run + scheduler discards pending ticks up to this time, including ticks due + before the disable, so a pause it never observed between sweeps still + counts. + workload_role: type: string doc: >- diff --git a/api/run/run_v1alpha/schema.gen.go b/api/run/run_v1alpha/schema.gen.go index cd9c34539..2293dd4d1 100644 --- a/api/run/run_v1alpha/schema.gen.go +++ b/api/run/run_v1alpha/schema.gen.go @@ -300,7 +300,7 @@ func (o *Run) InitSchema(sb *schema.SchemaBuilder) { sb.Singleton("dev.miren.run/status.timed_out") sb.Singleton("dev.miren.run/status.canceled") sb.Singleton("dev.miren.run/status.skipped") - sb.Ref("status", "dev.miren.run/run.status", schema.Doc("Indexed because the deadline sweep lists non-terminal runs on a short\ninterval; scanning every run of every app to find them would not hold up.\n\n\"skipped\" is a scheduled run that never executed because its predecessor\nwas still going. It exists so a skipped tick is visible rather than\nlooking like a gap, since \"my job stopped running\" and \"my job got\nslower than its interval\" are otherwise indistinguishable from outside.\n"), schema.Indexed, schema.Choices(RunStatusPendingId, RunStatusRunningId, RunStatusSucceededId, RunStatusFailedId, RunStatusTimedOutId, RunStatusCanceledId, RunStatusSkippedId)) + sb.Ref("status", "dev.miren.run/run.status", schema.Doc("Indexed because the deadline sweep lists non-terminal runs on a short\ninterval; scanning every run of every app to find them would not hold up.\n\n\"skipped\" is scheduled work discarded because its predecessor was still\ngoing or the app was disabled before the next attempt could start.\nIt exists so a skipped tick is visible rather than looking like a gap,\nsince \"my job stopped running\" and \"my job got slower than its interval\"\nare otherwise indistinguishable from outside.\n"), schema.Indexed, schema.Choices(RunStatusPendingId, RunStatusRunningId, RunStatusSucceededId, RunStatusFailedId, RunStatusTimedOutId, RunStatusCanceledId, RunStatusSkippedId)) sb.String("task", "dev.miren.run/run.task", schema.Doc("Task name from app.toml")) sb.String("tick", "dev.miren.run/run.tick", schema.Doc("Scheduled runs only: the RFC 3339 instant this run claims. Every replica\nderives the same ticks from the stored calendar expression, so creating\nthe run is a create-if-absent on an id derived from this, and etcd picks\nthe single winner.\n")) sb.String("timeout", "dev.miren.run/run.timeout", schema.Doc("Go duration bounding the run; empty means the platform default, \"0\" unbounded")) diff --git a/api/run/schema.yml b/api/run/schema.yml index 691b5ba20..d44d573a8 100644 --- a/api/run/schema.yml +++ b/api/run/schema.yml @@ -62,10 +62,11 @@ kinds: Indexed because the deadline sweep lists non-terminal runs on a short interval; scanning every run of every app to find them would not hold up. - "skipped" is a scheduled run that never executed because its predecessor - was still going. It exists so a skipped tick is visible rather than - looking like a gap, since "my job stopped running" and "my job got - slower than its interval" are otherwise indistinguishable from outside. + "skipped" is scheduled work discarded because its predecessor was still + going or the app was disabled before the next attempt could start. + It exists so a skipped tick is visible rather than looking like a gap, + since "my job stopped running" and "my job got slower than its interval" + are otherwise indistinguishable from outside. attempt: type: int diff --git a/cli/commands/app_disable.go b/cli/commands/app_disable.go new file mode 100644 index 000000000..2bbc3434a --- /dev/null +++ b/cli/commands/app_disable.go @@ -0,0 +1,42 @@ +package commands + +import ( + "miren.dev/runtime/api/app/app_v1alpha" +) + +func AppDisable(ctx *Context, opts struct { + Reason string `short:"r" long:"reason" description:"Why the app is disabled, shown by app status"` + AppCentric +}) error { + crudcl, err := ctx.RPCClient("dev.miren.runtime/app") + if err != nil { + return err + } + + result, err := app_v1alpha.NewCrudClient(crudcl).Disable(ctx, opts.App, opts.Reason) + if err != nil { + return err + } + + ctx.Printf("Disabled app %s (scaled %d pools to zero)\n", opts.App, result.ScaledPools()) + ctx.Printf("Run `miren app enable -a %s` to start it again.\n", opts.App) + return nil +} + +func AppEnable(ctx *Context, opts struct { + AppCentric +}) error { + crudcl, err := ctx.RPCClient("dev.miren.runtime/app") + if err != nil { + return err + } + + result, err := app_v1alpha.NewCrudClient(crudcl).Enable(ctx, opts.App) + if err != nil { + return err + } + + ctx.Printf("Enabled app %s (restored %d fixed pools; autoscaled services start on the next request)\n", + opts.App, result.RestoredPools()) + return nil +} diff --git a/cli/commands/app_list.go b/cli/commands/app_list.go index fb0e8fdf9..aa334f73a 100644 --- a/cli/commands/app_list.go +++ b/cli/commands/app_list.go @@ -149,6 +149,8 @@ func AppList(ctx *Context, opts struct { } else { status = infoYellow.Render("⚠️ 0 (fixed)") } + case "disabled": + status = infoYellow.Render("⏸ disabled") case "healthy": status = infoGreen.Render(fmt.Sprintf("%d%s", a.ReadyInstances(), modeSuffix)) case "degraded", "starting": diff --git a/cli/commands/app_status.go b/cli/commands/app_status.go index c6764ad6d..910d557e4 100644 --- a/cli/commands/app_status.go +++ b/cli/commands/app_status.go @@ -8,6 +8,7 @@ import ( "miren.dev/runtime/api/app/app_v1alpha" "miren.dev/runtime/api/deployment/deployment_v1alpha" + "miren.dev/runtime/pkg/rpc/standard" "miren.dev/runtime/pkg/ui" ) @@ -80,8 +81,13 @@ func AppStatus(ctx *Context, opts AppStatusOptions) error { if source := formatSource(appResult.SourceKind(), appResult.SourceValue()); source != "" { items = append(items, ui.NewNamedValue("Source", source)) } - if down := appResult.MaintenanceRoutes(); len(down) > 0 { - items = append(items, ui.NewNamedValue("Maintenance", fmt.Sprintf("%s serving a holding page", maintenanceRouteList(down)))) + var disabledAt *time.Time + if appResult.HasDisabledAt() { + at := standard.FromTimestamp(appResult.DisabledAt()) + disabledAt = &at + } + if label, lines := notServingLines(disabledAt, appResult.DisabledReason(), appResult.MaintenanceRoutes()); label != "" { + items = append(items, ui.NewNamedValue(label, strings.Join(lines, "; "))) } if activeDeployment != nil && activeDeployment.HasDeployment() { @@ -224,6 +230,8 @@ func printAppStatusJSON( Source *sourceJSON `json:"source,omitempty"` WorkloadRole string `json:"workload_role,omitempty"` MaintenanceRoutes []string `json:"maintenance_routes,omitempty"` + DisabledAt *time.Time `json:"disabled_at,omitempty"` + DisabledReason string `json:"disabled_reason,omitempty"` Configuration *configuration `json:"configuration,omitempty"` ActiveDeployment *deploymentJSON `json:"active_deployment,omitempty"` RecentDeployments []deploymentJSON `json:"recent_deployments,omitempty"` @@ -234,8 +242,13 @@ func printAppStatusJSON( Cluster: cluster, WorkloadRole: appResult.WorkloadRole(), MaintenanceRoutes: appResult.MaintenanceRoutes(), + DisabledReason: appResult.DisabledReason(), Services: health, } + if appResult.HasDisabledAt() { + at := standard.FromTimestamp(appResult.DisabledAt()) + output.DisabledAt = &at + } if healthErr != nil { output.HealthError = healthErr.Error() } @@ -290,3 +303,26 @@ func maintenanceRouteList(hosts []string) string { return strings.Join(named, ", ") } + +// notServingLines explains why an app, or some of its routes, isn't serving. +// A disabled app and a route in maintenance read as one section: the app is +// off, and those routes show their maintenance page instead of the disabled +// one. +func notServingLines(disabledAt *time.Time, reason string, maintenanceRoutes []string) (string, []string) { + if disabledAt == nil { + if len(maintenanceRoutes) == 0 { + return "", nil + } + return "Maintenance", []string{fmt.Sprintf("%s serving a holding page", maintenanceRouteList(maintenanceRoutes))} + } + + line := fmt.Sprintf("disabled since %s", disabledAt.Local().Format(time.RFC1123)) + if reason != "" { + line += " — " + reason + } + lines := []string{line} + if len(maintenanceRoutes) > 0 { + lines = append(lines, fmt.Sprintf("in maintenance instead: %s", maintenanceRouteList(maintenanceRoutes))) + } + return "Not serving", lines +} diff --git a/cli/commands/app_status_test.go b/cli/commands/app_status_test.go new file mode 100644 index 000000000..4429e441d --- /dev/null +++ b/cli/commands/app_status_test.go @@ -0,0 +1,55 @@ +package commands + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestNotServingLines(t *testing.T) { + at := time.Date(2026, 10, 8, 17, 0, 0, 0, time.UTC) + since := "disabled since " + at.Local().Format(time.RFC1123) + + tests := []struct { + name string + disabledAt *time.Time + reason string + routes []string + wantLabel string + wantLines []string + }{ + {name: "serving", wantLabel: "", wantLines: nil}, + { + name: "maintenance only", + routes: []string{"shop.example.com"}, + wantLabel: "Maintenance", + wantLines: []string{"shop.example.com serving a holding page"}, + }, + { + name: "disabled with a reason", + disabledAt: &at, + reason: "moved to the new cluster", + wantLabel: "Not serving", + wantLines: []string{since + " — moved to the new cluster"}, + }, + { + name: "disabled with routes in maintenance", + disabledAt: &at, + routes: []string{"shop.example.com", ""}, + wantLabel: "Not serving", + wantLines: []string{ + since, + "in maintenance instead: shop.example.com, the default route", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + label, lines := notServingLines(tt.disabledAt, tt.reason, tt.routes) + assert.Equal(t, tt.wantLabel, label) + assert.Equal(t, tt.wantLines, lines) + }) + } +} diff --git a/cli/commands/commands.go b/cli/commands/commands.go index 2d21eb0ec..361ecf7be 100644 --- a/cli/commands/commands.go +++ b/cli/commands/commands.go @@ -285,6 +285,20 @@ miren deployment create --format jsonl | jq -c 'select(.event == "build_step")' Body: "miren app restart -s web", }), )) + d.Dispatch("app disable", Infer("app disable", "Hold every service of an app at zero instances", AppDisable, + WithDescription(appDisableDescription), + WithExample(mflags.Example{ + Name: "Disable an app", + Body: "miren app disable -a myapp --reason \"Moved to the new cluster\"", + }), + )) + d.Dispatch("app enable", Infer("app enable", "Start a disabled app again", AppEnable, + WithDescription(appEnableDescription), + WithExample(mflags.Example{ + Name: "Enable an app", + Body: "miren app enable -a myapp", + }), + )) d.Dispatch("app set-workload-role", Infer("app set-workload-role", "Set the API role for an app's sandbox identity tokens", AppSetWorkloadRole, WithExample(mflags.Example{ Name: "Let an app's workloads read and deploy their own app", @@ -1498,6 +1512,20 @@ Use restart to: You do not need to restart after ` + "`" + `miren env set` + "`" + `, ` + "`" + `miren env delete` + "`" + `, ` + "`" + `miren addon create` + "`" + `, or ` + "`" + `miren addon delete` + "`" + `. Each of those already creates a new version and rolls out new sandboxes automatically. A manual restart on top only adds a redundant rollout. :::` +const appDisableDescription = `Disable turns an app off. Every service is held at zero instances: running sandboxes shut down, scheduled tasks pause, and visitors get a 503 holding page. The app is frozen until you enable it: deploys, rollbacks, and config changes are refused. The app keeps its versions, routes, environment, disks, and addons, so ` + "`" + `miren app enable` + "`" + ` brings it back as it was. Task runs already in progress finish, and addons keep running. + +Disabled is an explicit state, so ` + "`" + `miren app list` + "`" + ` and ` + "`" + `miren app status` + "`" + ` report it as such rather than as an app that failed to start. + +:::note[Manual tasks still run] +` + "`" + `miren app run` + "`" + ` keeps working, which leaves room for a backup or a migration while the app is off. Scheduled tasks don't fire while the app is disabled, and enabling it resumes the schedule from that moment without running the ticks it missed. +::: + +:::tip[Taking only traffic down?] +To show visitors a holding page while the app keeps running, for a migration for example, put its routes into maintenance with ` + "`" + `miren route down` + "`" + ` instead. +:::` + +const appEnableDescription = `Enable turns a disabled app back on. Fixed-mode services return to their configured count right away; autoscaled services stay at zero and start on the next request.` + const secretSetDescription = `Stores a value in the cluster's secret store. The value is encrypted at rest and is never echoed, logged, or written to disk by this command — it travels to the server, which holds the only key. Each write mints a new immutable version and prints its handle, e.g. ` + "`" + `payments/stripe-key@x1A` + "`" + `. Storing a value identical to the current one is reported as unchanged rather than minting a duplicate, so re-running the command is safe. diff --git a/cli/commands/deploy_events.go b/cli/commands/deploy_events.go index 001e721b5..deadcff7f 100644 --- a/cli/commands/deploy_events.go +++ b/cli/commands/deploy_events.go @@ -197,6 +197,10 @@ func outcomeEvent(o terminalOutcome) deployevents.Outcome { return deployevents.OutcomeHealthy case outcomeScaledToZero: return deployevents.OutcomeScaledToZero + case outcomeDisabled: + // Published as scaled to zero, which it is, rather than widening the + // event vocabulary for a variant of the same state. + return deployevents.OutcomeScaledToZero case outcomeNoService: // Keep the published value for compatibility. It predates static-only // apps and means the deployment has no service process to wait for. diff --git a/cli/commands/health_wait.go b/cli/commands/health_wait.go index 4bbd2d7d5..81dc5af89 100644 --- a/cli/commands/health_wait.go +++ b/cli/commands/health_wait.go @@ -96,6 +96,9 @@ const ( // decisionNoService: the app has no long-running process by design. Its // tasks or static content are available, which is as "up" as it gets. decisionNoService + // decisionDisabled: the app is disabled, so the version is deployed but + // held at zero until the app is enabled. + decisionDisabled ) // decideActivation maps a snapshot to a terminal decision (or "keep waiting"). @@ -125,6 +128,8 @@ func decideActivation(snap healthSnapshot) activationDecision { return decisionNoService case snap.health == apphealth.Idle: return decisionScaledToZero + case snap.health == apphealth.Disabled: + return decisionDisabled case snap.ready > 0: return decisionHealthy default: @@ -189,6 +194,7 @@ const ( outcomeHealthy terminalOutcome = iota outcomeScaledToZero outcomeNoService + outcomeDisabled outcomeCrashed outcomeTimeout outcomeCanceled @@ -210,6 +216,8 @@ func pollOutcome(ctx context.Context, getter appInfoGetter, appName, versionID s return outcomeScaledToZero, snap, true case decisionNoService: return outcomeNoService, snap, true + case decisionDisabled: + return outcomeDisabled, snap, true case decisionCrashed: return outcomeCrashed, snap, true default: @@ -240,6 +248,8 @@ func healthOutcomeText(versionDisplay string, outcome terminalOutcome, snap heal // Deliberately not the scaled-to-zero wording: nothing went to sleep, // this app never had a long-running process to begin with. return fmt.Sprintf("Version %s deployed — no long-running service required", versionDisplay), true + case outcomeDisabled: + return fmt.Sprintf("Version %s deployed — the app is disabled; run `miren app enable` to start it", versionDisplay), true case outcomeCrashed: detail := "" if snap.crashCount > 0 { diff --git a/cli/commands/health_wait_test.go b/cli/commands/health_wait_test.go index 1ebce6514..76a9ada26 100644 --- a/cli/commands/health_wait_test.go +++ b/cli/commands/health_wait_test.go @@ -104,6 +104,7 @@ func TestDecideActivation(t *testing.T) { // Without its own arm this falls to ready > 0, which an app without a // service never satisfies -- the deploy would time out despite succeeding. {"service-free app is deployed", healthSnapshot{versionActive: true, health: apphealth.Ready, ready: 0, desired: 0}, decisionNoService}, + {"disabled app is deployed", healthSnapshot{versionActive: true, health: apphealth.Disabled, ready: 0, desired: 0}, decisionDisabled}, {"one of many serving", healthSnapshot{versionActive: true, health: apphealth.Degraded, ready: 1, desired: 3}, decisionHealthy}, {"fully healthy", healthSnapshot{versionActive: true, health: apphealth.Healthy, ready: 2, desired: 2}, decisionHealthy}, {"active but nothing serving", healthSnapshot{versionActive: true, health: apphealth.Starting, ready: 0, desired: 1}, decisionWait}, diff --git a/components/activator/activator.go b/components/activator/activator.go index ce00e0a49..de655e752 100644 --- a/components/activator/activator.go +++ b/components/activator/activator.go @@ -321,6 +321,11 @@ var ErrCrashCooldown = fmt.Errorf("%w: application in crash cooldown", ErrSandbo var ErrPoolTimeout = fmt.Errorf("timeout waiting for sandbox from pool") +// ErrAppDisabled means the app was disabled with `miren app disable`. No +// capacity will arrive until it is enabled, so callers fail fast instead of +// waiting out the pool timeout. +var ErrAppDisabled = fmt.Errorf("app is disabled") + // waitForSandbox waits for a sandbox with capacity to become available. // If incrementPool is true, it will ensure the pool exists and increment DesiredInstances. // If incrementPool is false, it assumes PENDING sandboxes exist and just waits for them. @@ -851,6 +856,21 @@ func (a *localActivator) registerVersionPoolLocked( } } +// appDisabled reports whether the app is disabled. A lookup failure reads as +// not disabled, leaving the request to the normal capacity path. +func (a *localActivator) appDisabled(ctx context.Context, appID entity.Id) bool { + if appID == "" { + return false + } + resp, err := a.eac.Get(ctx, appID.String()) + if err != nil { + return false + } + var app core_v1alpha.App + app.Decode(resp.Entity().Entity()) + return !app.Disabled.Empty() +} + // requestPoolCapacity finds the SandboxPool created by DeploymentLauncher and increments DesiredInstances. // It uses retry logic with exponential backoff to handle the race where Activator receives // a request before DeploymentLauncher has finished creating the pool. @@ -883,6 +903,9 @@ func (a *localActivator) requestPoolCapacityTo(ctx context.Context, ver *core_v1 sc := core_v1alpha.ServiceConcurrency(svcConcurrency) strategy := concurrency.NewStrategyForVersion(ver, service, &sc) maxInstances := int64(strategy.MaxInstances()) + if a.appDisabled(ctx, ver.App) { + return nil, ErrAppDisabled + } // poolLoop is labeled so the pool-creation retry path below can break all // the way back out to re-read the cache. The outer loop re-acquires a.mu diff --git a/components/activator/activator_fixed_mode_test.go b/components/activator/activator_fixed_mode_test.go index c647dc8bf..12b2db2fb 100644 --- a/components/activator/activator_fixed_mode_test.go +++ b/components/activator/activator_fixed_mode_test.go @@ -277,3 +277,53 @@ func TestActivatorFixedModeNoSlotExhaustion(t *testing.T) { require.NoError(t, err) } } + +// TestActivatorDisabledAppFailsFast verifies that a lease request for any +// service of a disabled app returns ErrAppDisabled at once. +func TestActivatorDisabledAppFailsFast(t *testing.T) { + ctx := context.Background() + log := testutils.TestLogger(t) + + server, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + appID, err := server.Client.Create(ctx, "test-disabled-app", &core_v1alpha.App{Disabled: core_v1alpha.Disabled{At: time.Now()}}) + require.NoError(t, err) + + appVer := &core_v1alpha.AppVersion{ + App: appID, + Version: "v1", + ImageUrl: "test:latest", + Config: core_v1alpha.Config{ + Port: 3000, + Services: []core_v1alpha.Services{ + { + Name: "web", + ServiceConcurrency: core_v1alpha.ServiceConcurrency{Mode: "auto", RequestsPerInstance: 10}, + }, + }, + }, + } + verID, err := server.Client.Create(ctx, "test-disabled-v1", appVer) + require.NoError(t, err) + appVer.ID = verID + + activator := &localActivator{ + log: log, + eac: server.EAC, + versions: make(map[verKey]*versionPoolRef), + poolSandboxes: make(map[entity.Id]*poolSandboxes), + pools: make(map[verKey]*poolState), + newSandboxChans: make(map[verKey][]chan struct{}), + invalidationCh: make(chan SandboxInvalidation, 1), + } + + reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + start := time.Now() + lease, err := activator.AcquireLease(reqCtx, appVer, "web") + assert.Nil(t, lease) + require.ErrorIs(t, err, ErrAppDisabled) + assert.Less(t, time.Since(start), time.Second, "a disabled app should fail fast") +} diff --git a/controllers/deployment/launcher.go b/controllers/deployment/launcher.go index 6a3bd96af..fc5eec449 100644 --- a/controllers/deployment/launcher.go +++ b/controllers/deployment/launcher.go @@ -482,6 +482,13 @@ func (l *Launcher) ensurePoolForService(ctx context.Context, app *core_v1alpha.A desiredInstances = min } + // A disabled app's pools are held at zero: they exist so the version is + // complete, but nothing boots, not even for deploy verification. + disabled := !app.Disabled.Empty() + if disabled { + desiredInstances = 0 + } + // Ephemeral versions always get an isolated pool. Sharing a pool with a // non-ephemeral version (or another ephemeral with a different label) would // mean the pool's scaling behavior depends on which version is making the @@ -527,12 +534,13 @@ func (l *Launcher) ensurePoolForService(ctx context.Context, app *core_v1alpha.A needsUpdate = true } - // Fixed-mode strategies enforce their configured minimum. - if min := int64(strategy.MinInstances()); min > 0 && poolWithEntity.Pool.DesiredInstances != min { - poolWithEntity.Pool.DesiredInstances = min + // Fixed-mode strategies enforce their configured minimum; a disabled + // app is held at zero instead. + if min := int64(strategy.MinInstances()); (min > 0 || disabled) && poolWithEntity.Pool.DesiredInstances != desiredInstances { + poolWithEntity.Pool.DesiredInstances = desiredInstances l.Log.Info("strategy-enforced minimum, updating desired instances", "service", serviceName, - "desired_instances", min) + "desired_instances", desiredInstances) needsUpdate = true } @@ -542,7 +550,7 @@ func (l *Launcher) ensurePoolForService(ctx context.Context, app *core_v1alpha.A // the minutely resync repeatedly cold-starts every idle app and blocks the // launcher's worker while it waits for readiness. bootForVerification := false - if versionChanged && poolWithEntity.Pool.DesiredInstances < 1 { + if versionChanged && !disabled && poolWithEntity.Pool.DesiredInstances < 1 { poolWithEntity.Pool.DesiredInstances = 1 l.Log.Info("flooring drained pool to one instance for deploy verification", "service", serviceName, @@ -580,7 +588,7 @@ func (l *Launcher) ensurePoolForService(ctx context.Context, app *core_v1alpha.A l.Log.Info("strategy-enforced minimum, starting with desired instances", "service", serviceName, "desired_instances", desiredInstances) - } else if ver.EphemeralLabel == "" { + } else if !disabled && ver.EphemeralLabel == "" { // An autoscaled service's load doesn't change because its spec did. // Start the replacement at the size the current pool reached, so the // cutover doesn't hand that load to a single sandbox. @@ -637,6 +645,11 @@ func (l *Launcher) ensurePoolForService(ctx context.Context, app *core_v1alpha.A "service", serviceName, "desired_instances", desiredInstances) + // A disabled app's pool has nothing to boot, so there's nothing to wait for. + if disabled { + return "", nil + } + // Return the new pool ID — caller should wait for it to become ready return id, nil } diff --git a/controllers/deployment/launcher_test.go b/controllers/deployment/launcher_test.go index 899a9a5b9..523c5f84e 100644 --- a/controllers/deployment/launcher_test.go +++ b/controllers/deployment/launcher_test.go @@ -4862,6 +4862,60 @@ func TestConcurrentCreateAndReconcileNoDeadlock(t *testing.T) { assert.Len(t, pools, 1, "concurrent create + reconcile must converge on one pool") } +// TestDisabledAppStaysAtZero verifies that the launcher holds every service +// of a disabled app at zero: the resync doesn't restore a fixed count and a +// deploy doesn't boot an instance for verification. +func TestDisabledAppStaysAtZero(t *testing.T) { + ctx := context.Background() + log := testutils.TestLogger(t) + + server, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + app := &core_v1alpha.App{Project: entity.Id("project-1")} + appID, err := server.Client.Create(ctx, "test-app", app) + require.NoError(t, err) + app.ID = appID + + newVersion := func(name string) *core_v1alpha.AppVersion { + t.Helper() + ver := &core_v1alpha.AppVersion{ + App: app.ID, + Version: name, + ImageUrl: "worker:latest", + Config: core_v1alpha.Config{ + Services: []core_v1alpha.Services{ + { + Name: "worker", + ServiceConcurrency: core_v1alpha.ServiceConcurrency{Mode: "fixed", NumInstances: 2}, + }, + }, + }, + } + verID, createErr := server.Client.Create(ctx, name, ver) + require.NoError(t, createErr) + ver.ID = verID + return ver + } + + app.ActiveVersion = newVersion("v1").ID + app.Disabled.At = time.Now() + require.NoError(t, server.Client.Update(ctx, app)) + require.NoError(t, newTestLauncher(log, server.EAC).Reconcile(ctx, app, nil)) + + pools := listAllPools(t, ctx, server) + require.Len(t, pools, 1) + assert.Equal(t, int64(0), pools[0].DesiredInstances, "a disabled app's fixed service must not be restored") + + app.ActiveVersion = newVersion("v2").ID + require.NoError(t, server.Client.Update(ctx, app)) + require.NoError(t, newTestLauncher(log, server.EAC).Reconcile(ctx, app, nil)) + + pools = listAllPools(t, ctx, server) + require.Len(t, pools, 1) + assert.Equal(t, int64(0), pools[0].DesiredInstances, "a deploy to a disabled app must not boot for verification") +} + // A deploy that needs a new pool for an autoscaled service starts it at the // size the current pool reached, not at one. func TestNewAutoPoolSeededFromCurrentPoolSize(t *testing.T) { @@ -4925,6 +4979,65 @@ func TestNewAutoPoolSeededFromCurrentPoolSize(t *testing.T) { assert.Equal(t, int64(7), newPool.DesiredInstances, "the new pool should start at the current pool's size") } +// A sibling pool can still have instances while Disable is mid-way through +// scaling down. A new pool for the disabled app must not copy that size. +func TestNewPoolForDisabledAppIgnoresCurrentPoolSize(t *testing.T) { + ctx := context.Background() + server, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + app := &core_v1alpha.App{Project: entity.Id("project-1")} + appID, err := server.Client.Create(ctx, "test-app", app) + require.NoError(t, err) + app.ID = appID + + deploy := func(name, image string) { + ver := &core_v1alpha.AppVersion{ + App: app.ID, + Version: name, + ImageUrl: image, + Config: core_v1alpha.Config{ + Port: 3000, + Services: []core_v1alpha.Services{{ + Name: "web", + ServiceConcurrency: core_v1alpha.ServiceConcurrency{Mode: "auto", RequestsPerInstance: 10}, + }}, + }, + } + verID, err := server.Client.Create(ctx, "test-"+name, ver) + require.NoError(t, err) + app.ActiveVersion = verID + require.NoError(t, server.Client.Update(ctx, app)) + } + + launcher := newTestLauncher(testutils.TestLogger(t), server.EAC) + + deploy("v1", "oci.miren.cloud/web:1") + require.NoError(t, launcher.Reconcile(ctx, app, nil)) + pools := listAllPools(t, ctx, server) + require.Len(t, pools, 1) + oldPoolID := pools[0].ID + + _, err = server.EAC.Patch(ctx, []entity.Attr{ + entity.Ref(entity.DBId, oldPoolID), + entity.Int64(compute_v1alpha.SandboxPoolDesiredInstancesId, 7), + }, 0) + require.NoError(t, err) + + app.Disabled.At = time.Now() + deploy("v2", "oci.miren.cloud/web:2") + require.NoError(t, launcher.Reconcile(ctx, app, nil)) + + var newPool *compute_v1alpha.SandboxPool + for _, p := range listAllPools(t, ctx, server) { + if p.ID != oldPoolID { + newPool = &p + } + } + require.NotNil(t, newPool) + assert.Equal(t, int64(0), newPool.DesiredInstances) +} + // The old pool is only retired once the new one is fully up, so readiness // means every desired sandbox is running, not just the first. func TestWaitForPoolReadyWaitsForDesiredCount(t *testing.T) { diff --git a/controllers/run/controller.go b/controllers/run/controller.go index 4e273329e..1a97ce957 100644 --- a/controllers/run/controller.go +++ b/controllers/run/controller.go @@ -115,6 +115,18 @@ func (c *Controller) start(ctx context.Context, r *run_v1alpha.Run) error { return c.finish(ctx, r, run_v1alpha.TIMED_OUT, nil) } + // The scheduler may have created this run before disable. Make it terminal + // instead of holding it pending, so enabling cannot replay the skipped work. + if r.Trigger == run_v1alpha.SCHEDULE { + var app core_v1alpha.App + if err := c.EC.GetById(ctx, r.App, &app); err != nil { + return fmt.Errorf("checking scheduled run app: %w", err) + } + if !app.Disabled.Empty() { + return c.finish(ctx, r, run_v1alpha.SKIPPED, nil) + } + } + admitted, err := c.admit(ctx, r) if err != nil { return fmt.Errorf("admitting run: %w", err) diff --git a/controllers/run/controller_test.go b/controllers/run/controller_test.go index 1331215ba..161cb5a55 100644 --- a/controllers/run/controller_test.go +++ b/controllers/run/controller_test.go @@ -195,6 +195,47 @@ func TestRunReachesRunningAndCreatesASandbox(t *testing.T) { assert.Equal(t, id, runIDFor(&sb)) } +func TestPendingRunRespectsAppDisabledState(t *testing.T) { + for _, tc := range []struct { + name string + trigger run_v1alpha.RunTrigger + disabled bool + want run_v1alpha.RunStatus + }{ + {"enabled schedule", run_v1alpha.SCHEDULE, false, run_v1alpha.RUNNING}, + {"disabled schedule", run_v1alpha.SCHEDULE, true, run_v1alpha.SKIPPED}, + {"disabled manual", run_v1alpha.MANUAL, true, run_v1alpha.RUNNING}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + h := newHarness(t) + appID, verID := h.seedApp(ctx) + r := baseRun(appID, verID, "run/demo-migrate-1") + r.Trigger = tc.trigger + id := h.createRun(ctx, r) + if tc.disabled { + require.NoError(t, h.inm.Client.Patch(ctx, appID, 0, + entity.Component(core_v1alpha.AppDisabledId, (&core_v1alpha.Disabled{At: time.Now()}).Encode()))) + } + + require.NoError(t, h.reconcile(ctx, id)) + after := h.getRun(ctx, id) + assert.Equal(t, tc.want, after.Status) + if tc.want == run_v1alpha.SKIPPED { + assert.Empty(t, after.Sandbox) + assert.False(t, after.EndedAt.IsZero()) + require.NoError(t, h.inm.Client.Patch(ctx, appID, 0, + entity.Component(core_v1alpha.AppDisabledId, nil))) + require.NoError(t, h.reconcile(ctx, id)) + assert.Equal(t, run_v1alpha.SKIPPED, h.getRun(ctx, id).Status, + "enable must not replay the skipped run") + } else { + assert.NotEmpty(t, after.Sandbox) + } + }) + } +} + // Reconciling twice must not create a second sandbox. Framework retries can run // the same step again after a partial failure. func TestRunStartIsIdempotent(t *testing.T) { diff --git a/controllers/run/scheduler.go b/controllers/run/scheduler.go index 458a0d544..0aabaf63b 100644 --- a/controllers/run/scheduler.go +++ b/controllers/run/scheduler.go @@ -137,6 +137,10 @@ func (s *Scheduler) Sweep(ctx context.Context, now time.Time) error { continue } + if !app.Disabled.Empty() { + continue + } + if err := s.sweepApp(ctx, &app, now); err != nil { s.Log.Warn("failed to sweep app for scheduled tasks", "app", app.ID, "error", err) } @@ -204,6 +208,13 @@ func (s *Scheduler) sweepTask( from = now s.frontier[key] = now } + // Enable discards all pending ticks up to its timestamp, including ones + // due before disable. Only the latest enable matters, even when several + // disable/enable cycles happen between sweeps. + if app.SchedulesResumedAt.After(from) { + from = app.SchedulesResumedAt + s.frontier[key] = from + } s.mu.Unlock() // Only fire ticks this replica's own clock has passed. Skew therefore makes diff --git a/controllers/run/scheduler_test.go b/controllers/run/scheduler_test.go index 3a7030436..145cf684a 100644 --- a/controllers/run/scheduler_test.go +++ b/controllers/run/scheduler_test.go @@ -226,6 +226,82 @@ func TestSchedulerIgnoresAppsWithoutAnActiveVersion(t *testing.T) { assert.Empty(t, h.runs(ctx)) } +// A disabled app's schedule pauses, and enabling it doesn't replay the ticks +// it missed. On a fallback copy left disabled on an old cluster, either one +// would run a job twice against state the two clusters share. +func TestSchedulerPausesWhileTheAppIsDisabled(t *testing.T) { + ctx := context.Background() + h := newSchedHarness(t, "*-*-* *:00/30:00") + appID := entity.Id("app/demo") + + setDisabled := func(at time.Time) { + t.Helper() + _, err := h.inm.EAC.Patch(ctx, entity.New( + entity.Ref(entity.DBId, appID), + entity.Component(core_v1alpha.AppDisabledId, (&core_v1alpha.Disabled{At: at}).Encode()), + ).Attrs(), 0) + require.NoError(t, err) + } + + require.NoError(t, h.s.Sweep(ctx, atUTC("2026-08-04T12:00:00Z"))) + + setDisabled(atUTC("2026-08-04T12:10:00Z")) + require.NoError(t, h.s.Sweep(ctx, atUTC("2026-08-04T12:30:01Z"))) + require.NoError(t, h.s.Sweep(ctx, atUTC("2026-08-04T14:00:01Z"))) + assert.Empty(t, h.runs(ctx), "no tick fires while the app is disabled") + + _, err := h.inm.EAC.Patch(ctx, entity.New( + entity.Ref(entity.DBId, appID), + entity.Component(core_v1alpha.AppDisabledId, nil), + entity.Time(core_v1alpha.AppSchedulesResumedAtId, atUTC("2026-08-04T14:10:00Z")), + ).Attrs(), 0) + require.NoError(t, err) + + require.NoError(t, h.s.Sweep(ctx, atUTC("2026-08-04T14:30:01Z"))) + runs := h.runs(ctx) + require.Len(t, runs, 1, "the first sweep after enable skips missed ticks but fires ticks due after enable") + assert.Equal(t, "2026-08-04T14:30:00Z", runs[0].Tick) +} + +func TestSchedulerResumesFromLatestEnable(t *testing.T) { + for _, tc := range []struct { + name string + enables []string + want []string + }{ + {"single enable", []string{"2026-08-04T12:01:20Z"}, []string{"2026-08-04T12:02:00Z", "2026-08-04T12:03:00Z"}}, + {"repeated enables between sweeps", []string{"2026-08-04T12:01:20Z", "2026-08-04T12:02:20Z"}, []string{"2026-08-04T12:03:00Z"}}, + {"tick at enable is skipped", []string{"2026-08-04T12:02:00Z"}, []string{"2026-08-04T12:03:00Z"}}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + h := newSchedHarness(t, "*-*-* *:*:00") + require.NoError(t, h.s.Sweep(ctx, atUTC("2026-08-04T11:59:30Z"))) + + for _, enabledAt := range tc.enables { + _, err := h.inm.EAC.Patch(ctx, entity.New( + entity.Ref(entity.DBId, entity.Id("app/demo")), + entity.Time(core_v1alpha.AppSchedulesResumedAtId, atUTC(enabledAt)), + ).Attrs(), 0) + require.NoError(t, err) + } + + require.NoError(t, h.s.Sweep(ctx, atUTC("2026-08-04T12:03:01Z"))) + ticks := func() []string { + var ticks []string + for _, run := range h.runs(ctx) { + ticks = append(ticks, run.Tick) + } + return ticks + } + assert.ElementsMatch(t, tc.want, ticks(), "pending ticks before the latest enable are discarded") + + require.NoError(t, h.s.Sweep(ctx, atUTC("2026-08-04T12:04:01Z"))) + assert.ElementsMatch(t, append(tc.want, "2026-08-04T12:04:00Z"), ticks(), "later sweeps continue past the resume boundary") + }) + } +} + // Manual and deploy tasks are not on a schedule and must never be fired by it. func TestSchedulerIgnoresUnscheduledTasks(t *testing.T) { ctx := context.Background() diff --git a/controllers/sandboxpool/manager.go b/controllers/sandboxpool/manager.go index 02907e5a7..03b4ab719 100644 --- a/controllers/sandboxpool/manager.go +++ b/controllers/sandboxpool/manager.go @@ -81,8 +81,19 @@ func (m *Manager) Reconcile(ctx context.Context, pool *compute_v1alpha.SandboxPo // Skip crash detection for decommissioned pools (desired=0, no references). // Sandbox deaths during intentional scale-down are expected, not crashes. - if pool.DesiredInstances > 0 || len(pool.ReferencedByVersions) > 0 { - newCrashes, latestCrash := m.countStartupFailures(sandboxes, pool) + // + // The same holds for a disabled app's pools: they are held at zero on + // purpose, so sandboxes dying on the way down are not crashes, and cooldown + // must not hand a referenced pool a replacement. The app lookup only runs + // when there is crash state to clear, keeping it off the steady-state path. + newCrashes, latestCrash := m.countStartupFailures(sandboxes, pool) + hasCrashState := newCrashes > 0 || pool.ConsecutiveCrashCount > 0 || !pool.CooldownUntil.IsZero() + if pool.DesiredInstances == 0 && len(pool.ReferencedByVersions) > 0 && hasCrashState && m.appDisabled(ctx, pool) { + m.log.Info("clearing crash state on disabled app", "pool", pool.ID, "service", pool.Service) + pool.LastCrashTime = latestCrash + pool.ConsecutiveCrashCount = 0 + pool.CooldownUntil = time.Time{} + } else if pool.DesiredInstances > 0 || len(pool.ReferencedByVersions) > 0 { if newCrashes > 0 { pool.ConsecutiveCrashCount += int64(newCrashes) pool.LastCrashTime = latestCrash @@ -910,6 +921,21 @@ func (m *Manager) countStartupFailures(sandboxes []*sandboxWithMeta, pool *compu return count, latest } +// appDisabled reports whether the pool's app is disabled. A lookup failure +// reads as not disabled, which keeps the existing crash handling. +func (m *Manager) appDisabled(ctx context.Context, pool *compute_v1alpha.SandboxPool) bool { + if pool.App == "" { + return false + } + resp, err := m.eac.Get(ctx, pool.App.String()) + if err != nil { + return false + } + var app core_v1alpha.App + app.Decode(resp.Entity().Entity()) + return !app.Disabled.Empty() +} + // backoffDuration calculates the exponential backoff duration based on consecutive crash count // Uses exponential backoff: 10s, 20s, 40s, 80s, 160s, 320s, 640s, ... // Capped at 15 minutes diff --git a/controllers/sandboxpool/manager_test.go b/controllers/sandboxpool/manager_test.go index 3caba84d1..fb2c7b189 100644 --- a/controllers/sandboxpool/manager_test.go +++ b/controllers/sandboxpool/manager_test.go @@ -1554,6 +1554,38 @@ func TestManagerDecommissionedPool_NoCrashDetection(t *testing.T) { "decommissioned pool should not enter cooldown") } +// TestManagerDisabledApp_NoCrashCooldown verifies that a disabled app's pool, +// whose config still asks for instances, is held at zero rather than revived +// by crash cooldown. +func TestManagerDisabledApp_NoCrashCooldown(t *testing.T) { + ctx := context.Background() + server, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + appID, err := server.Client.Create(ctx, "disabled-app", &core_v1alpha.App{Disabled: core_v1alpha.Disabled{At: time.Now()}}) + require.NoError(t, err) + + pool := &compute_v1alpha.SandboxPool{ + App: appID, + Service: "worker", + DesiredInstances: 0, + ReferencedByVersions: []entity.Id{"ver-1"}, + ConsecutiveCrashCount: 2, + CooldownUntil: time.Now().Add(5 * time.Minute), + SandboxSpec: compute_v1alpha.SandboxSpec{Version: "ver-1"}, + } + poolID, err := server.Client.Create(ctx, "disabled-pool", pool) + require.NoError(t, err) + pool.ID = poolID + + reconcilePool(t, ctx, server, NewManager(testutils.TestLogger(t), server.EAC), pool) + + updated := getPool(t, ctx, server, poolID) + assert.Equal(t, int64(0), updated.DesiredInstances, "cooldown must not revive a disabled app") + assert.Equal(t, int64(0), updated.ConsecutiveCrashCount) + assert.True(t, updated.CooldownUntil.IsZero()) +} + // TestManagerCrashResetDoesNotRecount verifies that after crash state is reset // (e.g. by a deploy), old DEAD sandboxes are not re-counted as new crashes. // This is a regression test for MIR-956. diff --git a/docs/command-sidebar.json b/docs/command-sidebar.json index f257b088e..cc4fe7931 100644 --- a/docs/command-sidebar.json +++ b/docs/command-sidebar.json @@ -44,6 +44,8 @@ "command/app-admin-run", "command/app-attach", "command/app-delete", + "command/app-disable", + "command/app-enable", "command/app-history", "command/app-init", "command/app-list", diff --git a/docs/docs/command/app-disable.md b/docs/docs/command/app-disable.md new file mode 100644 index 000000000..4ee1cc541 --- /dev/null +++ b/docs/docs/command/app-disable.md @@ -0,0 +1,59 @@ +--- +title: "miren app disable" +sidebar_label: "app disable" +description: "Hold every service of an app at zero instances" +--- + +# miren app disable + +Hold every service of an app at zero instances + +Disable turns an app off. Every service is held at zero instances: running sandboxes shut down, scheduled tasks pause, and visitors get a 503 holding page. The app is frozen until you enable it: deploys, rollbacks, and config changes are refused. The app keeps its versions, routes, environment, disks, and addons, so `miren app enable` brings it back as it was. Task runs already in progress finish, and addons keep running. + +Disabled is an explicit state, so `miren app list` and `miren app status` report it as such rather than as an app that failed to start. + +:::note[Manual tasks still run] +`miren app run` keeps working, which leaves room for a backup or a migration while the app is off. Scheduled tasks don't fire while the app is disabled, and enabling it resumes the schedule from that moment without running the ticks it missed. +::: + +:::tip[Taking only traffic down?] +To show visitors a holding page while the app keeps running, for a migration for example, put its routes into maintenance with `miren route down` instead. +::: + +## Usage + +```bash +miren app disable [flags] +``` + +## Flags + +- `--reason, -r` — Why the app is disabled, shown by app status + +## Config Options + +- `--cluster, -C` — Cluster name +- `--config` — Path to the config file + +## App Options + +- `--app, -a` — Application name +- `--dir, -d` — Directory to run from (default: `.`) + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## Examples + +**Disable an app:** + +```bash +miren app disable -a myapp --reason "Moved to the new cluster" +``` + +## See also + +- [`miren app`](./app.md) diff --git a/docs/docs/command/app-enable.md b/docs/docs/command/app-enable.md new file mode 100644 index 000000000..c23f03082 --- /dev/null +++ b/docs/docs/command/app-enable.md @@ -0,0 +1,45 @@ +--- +title: "miren app enable" +sidebar_label: "app enable" +description: "Start a disabled app again" +--- + +# miren app enable + +Start a disabled app again + +Enable turns a disabled app back on. Fixed-mode services return to their configured count right away; autoscaled services stay at zero and start on the next request. + +## Usage + +```bash +miren app enable [flags] +``` + +## Config Options + +- `--cluster, -C` — Cluster name +- `--config` — Path to the config file + +## App Options + +- `--app, -a` — Application name +- `--dir, -d` — Directory to run from (default: `.`) + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## Examples + +**Enable an app:** + +```bash +miren app enable -a myapp +``` + +## See also + +- [`miren app`](./app.md) diff --git a/docs/docs/command/app.md b/docs/docs/command/app.md index 7f7f5ebbf..a6597a833 100644 --- a/docs/docs/command/app.md +++ b/docs/docs/command/app.md @@ -62,6 +62,8 @@ miren app --watch - [`miren app admin`](./app-admin.md) — Application-defined admin methods - [`miren app attach`](./app-attach.md) — Deprecated: use 'miren run attach' - [`miren app delete`](./app-delete.md) — Delete an application and all its resources +- [`miren app disable`](./app-disable.md) — Hold every service of an app at zero instances +- [`miren app enable`](./app-enable.md) — Start a disabled app again - [`miren app history`](./app-history.md) — Show deployment history for an application - [`miren app init`](./app-init.md) — Initialize a new application - [`miren app list`](./app-list.md) — List all applications diff --git a/docs/docs/commands.md b/docs/docs/commands.md index 64562378e..aa2643200 100644 --- a/docs/docs/commands.md +++ b/docs/docs/commands.md @@ -44,6 +44,8 @@ Complete reference for all `miren` CLI commands. | [`miren app admin run`](./command/app-admin-run.md) | Call an admin method on an application | | [`miren app attach`](./command/app-attach.md) | Deprecated: use 'miren run attach' | | [`miren app delete`](./command/app-delete.md) | Delete an application and all its resources | +| [`miren app disable`](./command/app-disable.md) | Hold every service of an app at zero instances | +| [`miren app enable`](./command/app-enable.md) | Start a disabled app again | | [`miren app history`](./command/app-history.md) | Show deployment history for an application | | [`miren app init`](./command/app-init.md) | Initialize a new application | | [`miren app list`](./command/app-list.md) | List all applications | diff --git a/docs/docs/in-cluster-api.md b/docs/docs/in-cluster-api.md index 22e91d351..82512975b 100644 --- a/docs/docs/in-cluster-api.md +++ b/docs/docs/in-cluster-api.md @@ -66,11 +66,11 @@ The default is **`app-readonly`** — a fresh app's workloads can read their own | --- | --- | --- | | `none` | — | Authenticates but authorizes nothing | | `app-readonly` *(default)* | own app | Read own app status, logs, and deployment history | -| `app-deployer` | own app | `app-readonly` + build and (re)deploy its own app | +| `app-deployer` | own app | `app-readonly` + build and (re)deploy its own app, and disable/enable it | | `app-debugger` | own app | `app-readonly` + open a shell / run commands in its own sandboxes | | `app-admin` | own app | `app-deployer` + edit config/env and exec — everything for its own app except deleting it | | `cluster-readonly` | cluster | Read status, logs, and infrastructure state across all apps | -| `cluster-deployer` | cluster | `cluster-readonly` + build, deploy, and configure any app (but not create or destroy apps) | +| `cluster-deployer` | cluster | `cluster-readonly` + build, deploy, configure, disable, and enable any app (but not create or destroy apps) | | `cluster-debugger` | cluster | `cluster-readonly` + exec and inspect any app | | `cluster-admin` | cluster | Broad control across the cluster (see the limits below) | diff --git a/docs/docs/maintenance-mode.md b/docs/docs/maintenance-mode.md index 58258ba2f..ecdce099b 100644 --- a/docs/docs/maintenance-mode.md +++ b/docs/docs/maintenance-mode.md @@ -117,7 +117,7 @@ There's no way to browse the real app from behind the holding page yet. Bring th ## What's not covered :::warning[Background work keeps running] -Maintenance mode only affects HTTP traffic. Background workers and scheduled jobs carry on, so a window opened for a migration does not stop them writing to the database you are migrating. Stop them the way you normally would. +Maintenance mode only affects HTTP traffic. Background workers and scheduled jobs carry on, so a window opened for a migration does not stop them writing to the database you are migrating. To stop them too, also [disable the app](./command/app-disable.md): every service goes to zero and scheduled tasks pause, the route keeps showing its maintenance page, and `miren app run` still works for the migration itself. ::: Custom holding-page HTML and scheduled windows (announce a future window, enter and leave automatically) aren't supported yet. diff --git a/docs/docs/scaling.md b/docs/docs/scaling.md index afafa7c1b..3c62f847a 100644 --- a/docs/docs/scaling.md +++ b/docs/docs/scaling.md @@ -128,6 +128,41 @@ num_instances = 3 | `mode` | Must be `"fixed"` | `"fixed"` for non-web | | `num_instances` | Exact number of instances to run | 1 | +## Disabling an App + +To turn an app off without deleting it, disable it: + +```bash +miren app disable -a myapp --reason "Moved to the new cluster" +``` + +Every service, autoscaled or fixed, is held at zero until you run `miren app enable`. Running instances shut down and visitors get a 503 holding page. A disabled app is frozen, so deploys, rollbacks, config changes such as `miren env set`, and `miren app restart` are all refused until it's enabled. That's what lets `miren app enable` bring it back as it was. The app keeps its versions, routes, environment, disks, and addons. Scheduled tasks pause, but `miren app run` still works for a backup or migration while the app is off. Task runs already in progress finish, and addons keep running. + +Enabling resumes scheduled tasks from the enable time. Any pending scheduled ticks at or before that time are discarded, including ticks that became due before the app was disabled but had not yet been picked up. + +To take traffic down while the app keeps running and can still be deployed, use [maintenance mode](./maintenance-mode.md) instead. + +Unlike an autoscaled service that has scaled to zero, a disabled app doesn't wake on traffic. `miren app list` shows it as `disabled`, and `miren app status` shows when and why. + +`miren app enable` returns fixed services to their configured count right away; autoscaled services start on the next request. + +### Updating a Disabled App + +To update an app that should stay off, such as a fallback copy on another cluster, turn it on behind maintenance mode, deploy, check it, and turn it off again: + +```bash +miren route down app.example.com --reason "Updating" +miren app enable -a myapp +miren deploy +miren app status -a myapp +miren app disable -a myapp --reason "Fallback copy" +miren route up app.example.com +``` + +Run `route down` for each hostname that points at the app, since anything you miss serves visitors while it's on. Bring the routes back up at the end, or they'll still be down the next time you enable the app. + +Enabling turns the whole app on, and maintenance mode only stops traffic on its routes. For that window workers run, scheduled tasks fire, and deploy tasks run during the deploy. If the copy shares a database or sends mail, give it a setting of its own that keeps those side effects quiet, such as `miren env set -e FALLBACK_MODE=1`, and set it while the app is on. + ## Examples ### High-Traffic API diff --git a/docs/docs/server-config.md b/docs/docs/server-config.md index 7cf55e818..18f08e9c6 100644 --- a/docs/docs/server-config.md +++ b/docs/docs/server-config.md @@ -133,7 +133,7 @@ server after changing it. For an app-specific override, see Templates receive `.Status` (HTTP status number), `.Title`, `.Description`, `.Site` (visitor hostname on maintenance pages), `.Reason`, `.BackAt`, and -`.Maintenance` (boolean). `{{brandLogo}}` renders the built-in Miren logo. +`.Maintenance` and `.Disabled` (booleans). `{{brandLogo}}` renders the built-in Miren logo. Use self-contained markup and inline CSS if the page must work when the app is unavailable. Only HTML responses use these templates: `Accept` negotiation still selects JSON or plain text for API clients. If the app's template is @@ -144,7 +144,10 @@ For ordinary errors, `.Status`, `.Title`, and `.Description` are set; `.Site`, `.Reason`, and `.BackAt` are empty. During maintenance, `.Status` is 503, `.Maintenance` is true, `.Site` is the visitor's hostname, `.Reason` is the operator's message, and `.BackAt` is a formatted UTC time when provided. -`.Title` and `.Description` are empty on maintenance pages. These are the only +`.Title` and `.Description` are empty on maintenance pages. A disabled app +(see [`miren app disable`](./command/app-disable.md)) gets the same holding +page with `.Disabled` also true and `.Reason` and `.BackAt` empty, so a +template that only checks `.Maintenance` still renders a holding page. These are the only template data fields; app IDs, raw failure messages, and request details are not exposed. HTML escaping is automatic. Templates must be at most 128 KiB; rendered output is capped at 256 KiB and falls back if it exceeds that limit. diff --git a/pkg/apphealth/apphealth.go b/pkg/apphealth/apphealth.go index f9e0fc5d2..cf99c316d 100644 --- a/pkg/apphealth/apphealth.go +++ b/pkg/apphealth/apphealth.go @@ -15,6 +15,9 @@ const ( Crashed = "crashed" // Idle means the app is deliberately scaled to zero (no desired instances). Idle = "idle" + // Disabled means the app was disabled with `miren app disable`: every + // service is held at zero until `miren app enable`, whatever its config. + Disabled = "disabled" // Ready means the app is deployed and available to invoke, but has no // long-running process to be healthy or idle. A task-only app is doing // exactly what it was configured to do; reporting it as idle would say it diff --git a/pkg/deploylifecycle/lock.go b/pkg/deploylifecycle/lock.go index 49fb384b4..8309b9c1a 100644 --- a/pkg/deploylifecycle/lock.go +++ b/pkg/deploylifecycle/lock.go @@ -481,6 +481,11 @@ func (l *Locks) commitActivation(ctx context.Context, appName string, appID, ver if app.ActiveVersion == versionID && app.ActiveDeployment == entity.Id(deploymentID) { return nil } + // Begin refuses a disabled app, but an attempt can outlive that check. + // Disable moves the app's revision, so this can't race the patch below. + if err := CheckEnabled(appName, app); err != nil { + return err + } if expectedRevision != 0 && current.Revision != expectedRevision { return cond.Conflict("app-revision", "app changed after deployment version was derived") } diff --git a/pkg/deploylifecycle/tracker.go b/pkg/deploylifecycle/tracker.go index 0ad71d516..39e429cb8 100644 --- a/pkg/deploylifecycle/tracker.go +++ b/pkg/deploylifecycle/tracker.go @@ -91,6 +91,64 @@ type BeginParams struct { ParentDeploymentID string } +// ErrAppDisabled means the app is frozen by `miren app disable`. Enable promises +// to bring the app back as it was, so nothing may change what it runs until +// then: not a build, a redeploy, a rollback, or a config change. An app that +// should keep taking changes while it's out of service belongs in maintenance +// mode instead. +var ErrAppDisabled = errors.New("app is disabled") + +// disabledError is what callers see for ErrAppDisabled. Its "conflict" code +// answers 409 over REST, but it deliberately doesn't match cond.ErrConflict: +// that means a lost race to config mutations, which would retry, and +// FailIfUnsettled reads it as already settled. +type disabledError struct { + name string +} + +func (e *disabledError) Error() string { + return fmt.Sprintf("%s: run `miren app enable -a %s` first", ErrAppDisabled, e.name) +} + +func (e *disabledError) Is(target error) bool { + return target == ErrAppDisabled +} + +func (e *disabledError) ErrorCode() string { + return "conflict" +} + +// ErrorCategory names the conflict for a client across the wire, which rebuilds +// this as a cond.ErrConflict and prints its category. +func (e *disabledError) ErrorCategory() string { + return "app-disabled" +} + +// CheckEnabled returns ErrAppDisabled, with what to do about it, if app is +// disabled. +func CheckEnabled(name string, app *core_v1alpha.App) error { + if !app.Disabled.Empty() { + return &disabledError{name: name} + } + return nil +} + +// refuseDisabled reads the app when the caller passed an AppID and Begin had no +// reason to look it up. +func (t *Tracker) refuseDisabled(ctx context.Context, name string, app *core_v1alpha.App) error { + if app == nil { + resolved, _, err := t.store.AppByName(ctx, name) + if errors.Is(err, cond.ErrNotFound{}) { + return nil + } + if err != nil { + return fmt.Errorf("failed to resolve deployment app: %w", err) + } + app = resolved + } + return CheckEnabled(name, app) +} + // Begin acquires the deploy lock and publishes a deployment record. // // If another live deployment holds the lock, Begin returns a *LockHeldError @@ -144,6 +202,9 @@ func (t *Tracker) Begin(ctx context.Context, params BeginParams) (*Record, error } app = resolved } + if err := t.refuseDisabled(ctx, params.AppName, app); err != nil { + return nil, err + } if params.ParentDeploymentID == "" && app != nil && app.ActiveDeployment != "" { params.ParentDeploymentID = string(app.ActiveDeployment) autoParent = true diff --git a/pkg/deploylifecycle/tracker_test.go b/pkg/deploylifecycle/tracker_test.go index 6130d9ef0..588326697 100644 --- a/pkg/deploylifecycle/tracker_test.go +++ b/pkg/deploylifecycle/tracker_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/pkg/cond" "miren.dev/runtime/pkg/entity" "miren.dev/runtime/pkg/entity/testutils" "miren.dev/runtime/pkg/rpc" @@ -627,3 +628,79 @@ func TestReconcileInterruptsAttemptAfterClaimIsStolen(t *testing.T) { assert.Equal(t, second, holder.DeploymentID, "reconciling the old attempt must preserve its successor's lock") } + +func disableTestApp(t *testing.T, tr *Tracker, app string) { + t.Helper() + disabled := core_v1alpha.Disabled{At: time.Now(), Reason: "parked"} + _, err := tr.store.eac.Patch(context.Background(), []entity.Attr{ + entity.Ref(entity.DBId, entity.Id("app/"+app)), + entity.Component(core_v1alpha.AppDisabledId, disabled.Encode()), + }, 0) + require.NoError(t, err) +} + +func TestBeginRefusesDisabledApp(t *testing.T) { + ctx := context.Background() + + for _, op := range []Operation{OperationBuild, OperationRedeploy, OperationRollback, OperationConfigChange} { + t.Run(string(op), func(t *testing.T) { + tr, _ := newTestTracker(t) + disableTestApp(t, tr, "web") + + _, err := tr.Begin(ctx, BeginParams{AppName: "web", ClusterID: "c1", Operation: op}) + require.ErrorIs(t, err, ErrAppDisabled) + require.ErrorContains(t, err, "miren app enable -a web") + + // A refused attempt leaves the lock free for after the app is enabled. + holder, err := tr.locks.Blocking(ctx, "web") + require.NoError(t, err) + require.Nil(t, holder) + }) + } +} + +func TestBeginRefusesDisabledAppWhenCallerResolvedTheID(t *testing.T) { + tr, _ := newTestTracker(t) + disableTestApp(t, tr, "web") + + _, err := tr.Begin(context.Background(), BeginParams{ + AppName: "web", AppID: entity.Id("app/web"), ClusterID: "c1", + }) + require.ErrorIs(t, err, ErrAppDisabled) +} + +func TestActivateRefusesAppDisabledMidAttempt(t *testing.T) { + ctx := context.Background() + tr, _ := newTestTracker(t) + + id := begin(t, tr, "web", "c1") + require.NoError(t, tr.SetAppVersion(ctx, id, "app_version/v1")) + disableTestApp(t, tr, "web") + + require.ErrorIs(t, tr.Activate(ctx, id), ErrAppDisabled) + + app, _, err := tr.store.AppByName(ctx, "web") + require.NoError(t, err) + require.Empty(t, app.ActiveVersion, "a frozen app must keep what it was running") +} + +// The refusal is expected, so REST must not report it as a 500, but it can't +// read as a lost race either: config mutations retry those, and FailIfUnsettled +// treats them as already settled. +func TestDisabledRefusalIsAConflictButNotALostRace(t *testing.T) { + err := CheckEnabled("web", &core_v1alpha.App{Disabled: core_v1alpha.Disabled{At: time.Now()}}) + + require.ErrorIs(t, err, ErrAppDisabled) + assert.NotErrorIs(t, err, cond.ErrConflict{}) + + var coded rpc.ErrorCode + require.ErrorAs(t, err, &coded) + assert.Equal(t, "conflict", coded.ErrorCode()) + + var category rpc.ErrorCategory + require.ErrorAs(t, err, &category) + assert.Equal(t, "app-disabled", category.ErrorCategory()) + + remote := cond.RemoteError(category.ErrorCategory(), coded.ErrorCode(), err.Error()) + assert.Equal(t, "conflict in app-disabled: app is disabled: run `miren app enable -a web` first", remote.Error()) +} diff --git a/pkg/rpc/authorization.go b/pkg/rpc/authorization.go index 195c2502e..3b5b454df 100644 --- a/pkg/rpc/authorization.go +++ b/pkg/rpc/authorization.go @@ -114,7 +114,7 @@ func AppAction(resource, action string) string { return "read" case "setconfiguration", "setenvvar", "setenvvars", "setinitialenvvars", "deleteenvvar": return "configure" - case "restart": + case "restart", "disable", "enable": return "deploy" } case "appstatus": diff --git a/pkg/workloadroles/roles.go b/pkg/workloadroles/roles.go index da85834ae..30ad2e1f5 100644 --- a/pkg/workloadroles/roles.go +++ b/pkg/workloadroles/roles.go @@ -82,6 +82,9 @@ var ( "updatedeploymentappversion", "canceldeployment", ), "builder": set("buildfromtar", "prepareupload", "buildfromprepared"), + // A deployer can already ship code that refuses every connection, so + // turning the app off and back on grants no new power. + "crud": set("disable", "enable"), } // appConfig: mutate one app's configuration. These handlers gain their diff --git a/pkg/workloadroles/roles_test.go b/pkg/workloadroles/roles_test.go index 54402d9e7..17ae65095 100644 --- a/pkg/workloadroles/roles_test.go +++ b/pkg/workloadroles/roles_test.go @@ -160,6 +160,15 @@ func TestRoleMembershipSpotChecks(t *testing.T) { {RoleClusterDeployer, [2]string{"deployment", "deployversion"}, [2]string{"crud", "destroy"}}, {RoleClusterDeployer, [2]string{"crud", "setenvvar"}, [2]string{"crud", "new"}}, {RoleClusterDebugger, [2]string{"internalhttp", "dorequest"}, [2]string{"crud", "destroy"}}, + // Disable and enable travel with deploy: a role that can ship an app + // can already take it down. + {RoleAppDeployer, [2]string{"crud", "disable"}, [2]string{"crud", "destroy"}}, + {RoleAppDeployer, [2]string{"crud", "enable"}, [2]string{"crud", "list"}}, + {RoleClusterDeployer, [2]string{"crud", "disable"}, [2]string{"crud", "destroy"}}, + {RoleClusterDeployer, [2]string{"crud", "enable"}, [2]string{"crud", "new"}}, + {RoleAppAdmin, [2]string{"crud", "disable"}, [2]string{"crud", "destroy"}}, + {RoleClusterAdmin, [2]string{"crud", "enable"}, [2]string{"crud", "setworkloadrole"}}, + {RoleAppDebugger, [2]string{"sandboxexec", "exec"}, [2]string{"crud", "disable"}}, } for _, c := range cases { diff --git a/servers/app/app.go b/servers/app/app.go index 5f292c7d7..961cfeaf1 100644 --- a/servers/app/app.go +++ b/servers/app/app.go @@ -25,6 +25,7 @@ import ( "miren.dev/runtime/pkg/entity" "miren.dev/runtime/pkg/idgen" "miren.dev/runtime/pkg/rpc" + "miren.dev/runtime/pkg/rpc/standard" "miren.dev/runtime/pkg/secret" "miren.dev/runtime/pkg/ui" "miren.dev/runtime/pkg/workloadroles" @@ -248,6 +249,17 @@ func (r *AppInfo) collectAppHealth(ctx context.Context) (*appHealthSource, error // both arrive here, so they cannot drift into disagreeing about whether an app // is healthy. func (s *appHealthSource) healthOf(entry appEntry) apphealth.State { + out := s.poolHealthOf(entry) + // A disabled app is held at zero on purpose; whatever its pools read as + // on the way down, that is the state worth reporting. + if !entry.app.Disabled.Empty() { + out.Health = apphealth.Disabled + } + return out +} + +// poolHealthOf classifies one app from its pools and config alone. +func (s *appHealthSource) poolHealthOf(entry appEntry) apphealth.State { out := apphealth.State{Name: entry.name} if ps, ok := s.poolState[entry.name]; ok { @@ -745,6 +757,11 @@ func (r *AppInfo) GetConfiguration(ctx context.Context, state *app_v1alpha.CrudG state.Results().SetMaintenanceRoutes(down) } + if !appRec.Disabled.Empty() { + state.Results().SetDisabledAt(standard.ToTimestamp(appRec.Disabled.At)) + state.Results().SetDisabledReason(appRec.Disabled.Reason) + } + return nil } @@ -957,6 +974,12 @@ func (r *AppInfo) Restart(ctx context.Context, state *app_v1alpha.CrudRestart) e return fmt.Errorf("app %q not found: %w", name, err) } + // Restart restores fixed-mode counts, which would quietly undo a disable + // until the next launcher resync put it back. + if err := deploylifecycle.CheckEnabled(name, &appRec); err != nil { + return err + } + // Resolve the config to restore DesiredInstances for fixed-mode pools. // During crash cooldown the pool manager resets DesiredInstances to 1; // we need to restore the configured value so fixed-mode pools come back diff --git a/servers/app/authorization_test.go b/servers/app/authorization_test.go index 73d3b0d88..e978dfa5d 100644 --- a/servers/app/authorization_test.go +++ b/servers/app/authorization_test.go @@ -66,6 +66,8 @@ func TestCrudResourceAuthorizationBeforeMutation(t *testing.T) { }}, {"deleteenvvar", "configure", func(ctx context.Context) error { _, err := client.DeleteEnvVar(ctx, "shop", "KEY", ""); return err }}, {"restart", "deploy", func(ctx context.Context) error { _, err := client.Restart(ctx, "shop", ""); return err }}, + {"disable", "deploy", func(ctx context.Context) error { _, err := client.Disable(ctx, "shop", ""); return err }}, + {"enable", "deploy", func(ctx context.Context) error { _, err := client.Enable(ctx, "shop"); return err }}, {"getconfiguration", "read", func(ctx context.Context) error { _, err := client.GetConfiguration(ctx, "shop"); return err }}, {"listversions", "read", func(ctx context.Context) error { _, err := client.ListVersions(ctx, appID.String(), false, 0) diff --git a/servers/app/disable.go b/servers/app/disable.go new file mode 100644 index 000000000..ad9e35d33 --- /dev/null +++ b/servers/app/disable.go @@ -0,0 +1,165 @@ +package app + +import ( + "context" + "fmt" + "slices" + "time" + + appclient "miren.dev/runtime/api/app" + "miren.dev/runtime/api/app/app_v1alpha" + "miren.dev/runtime/api/compute/compute_v1alpha" + coreutil "miren.dev/runtime/api/core" + "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/pkg/entity" +) + +// Disable holds every service of an app at zero instances until Enable. The +// app keeps its versions, routes, env, disks and addons; nothing boots for it, +// anything that would change the active version is refused, and requests get +// a 503. +// +// The flag on the App is what keeps it disabled: the launcher, activator and +// pool manager all read it, so the pools scaled down here stay down. +func (r *AppInfo) Disable(ctx context.Context, state *app_v1alpha.CrudDisable) error { + args := state.Args() + name := args.App() + + if err := appclient.Authorize(ctx, r.EC.EAC(), name, "deploy"); err != nil { + return err + } + + var appRec core_v1alpha.App + if err := r.EC.Get(ctx, name, &appRec); err != nil { + return fmt.Errorf("app %q not found: %w", name, err) + } + + // Set the flag before touching pools, so nothing that reads it scales a + // pool back up in between. Disabling again only updates the reason. + disabled := core_v1alpha.Disabled{At: appRec.Disabled.At, Reason: args.Reason()} + if disabled.At.IsZero() { + disabled.At = time.Now() + } + if err := r.EC.Patch(ctx, appRec.ID, 0, + entity.Component(core_v1alpha.AppDisabledId, disabled.Encode()), + ); err != nil { + return fmt.Errorf("marking app %q disabled: %w", name, err) + } + + poolList, err := r.EC.List(ctx, entity.Ref(compute_v1alpha.SandboxPoolAppId, appRec.ID)) + if err != nil { + return fmt.Errorf("listing pools: %w", err) + } + + var scaledPools int32 + for poolList.Next() { + var pool compute_v1alpha.SandboxPool + if err := poolList.Read(&pool); err != nil { + continue + } + if pool.DesiredInstances == 0 { + continue + } + if err := r.EC.Patch(ctx, pool.ID, 0, + entity.Int64(compute_v1alpha.SandboxPoolDesiredInstancesId, 0), + ); err != nil { + return fmt.Errorf("scaling pool %s to zero: %w", pool.ID, err) + } + scaledPools++ + } + + r.Log.Info("app disabled", "app", name, "reason", args.Reason(), "pools_scaled", scaledPools) + + state.Results().SetScaledPools(scaledPools) + return nil +} + +// Enable turns a disabled app back on. Fixed-mode services of the active +// version return to their configured count right away; autoscaled services +// stay at zero and start on the next request. +func (r *AppInfo) Enable(ctx context.Context, state *app_v1alpha.CrudEnable) error { + name := state.Args().App() + + if err := appclient.Authorize(ctx, r.EC.EAC(), name, "deploy"); err != nil { + return err + } + + var appRec core_v1alpha.App + if err := r.EC.Get(ctx, name, &appRec); err != nil { + return fmt.Errorf("app %q not found: %w", name, err) + } + if appRec.Disabled.Empty() { + return fmt.Errorf("app %q is not disabled", name) + } + + // The scheduler resumes from this timestamp rather than replaying pending + // ticks, even if it never observed the app disabled between sweeps. + if err := r.EC.Patch(ctx, appRec.ID, 0, + entity.Component(core_v1alpha.AppDisabledId, nil), + entity.Time(core_v1alpha.AppSchedulesResumedAtId, time.Now()), + ); err != nil { + return fmt.Errorf("enabling app %q: %w", name, err) + } + + // The app is enabled from here on, so a failure below is not a failed + // enable: retrying would only report that the app isn't disabled, and the + // launcher's resync restores fixed counts within a minute anyway. + restoredPools, err := r.restoreFixedPools(ctx, &appRec) + if err != nil { + r.Log.Warn("app enabled, but restoring its fixed pools failed; the launcher will restore them", + "app", name, "error", err) + } + + r.Log.Info("app enabled", "app", name, "pools_restored", restoredPools) + + state.Results().SetRestoredPools(restoredPools) + return nil +} + +// restoreFixedPools returns the active version's fixed services to their +// configured counts, so enable takes effect now rather than at the next resync. +// It reports how many pools it restored, including when it stops on an error. +func (r *AppInfo) restoreFixedPools(ctx context.Context, appRec *core_v1alpha.App) (int32, error) { + if appRec.ActiveVersion == "" { + return 0, nil + } + + var ver core_v1alpha.AppVersion + if err := r.EC.GetById(ctx, appRec.ActiveVersion, &ver); err != nil { + return 0, fmt.Errorf("getting active version: %w", err) + } + spec, err := coreutil.ResolveRuntimeConfig(ctx, r.EC.EAC(), &ver) + if err != nil { + return 0, fmt.Errorf("resolving config: %w", err) + } + + poolList, err := r.EC.List(ctx, entity.Ref(compute_v1alpha.SandboxPoolAppId, appRec.ID)) + if err != nil { + return 0, fmt.Errorf("listing pools: %w", err) + } + + var restored int32 + for poolList.Next() { + var pool compute_v1alpha.SandboxPool + if err := poolList.Read(&pool); err != nil { + continue + } + if !slices.Contains(pool.ReferencedByVersions, appRec.ActiveVersion) { + continue + } + svcConc, err := coreutil.GetServiceConcurrency(spec, pool.Service) + if err != nil || svcConc.Mode != "fixed" || svcConc.NumInstances == 0 { + continue + } + if pool.DesiredInstances == svcConc.NumInstances { + continue + } + if err := r.EC.Patch(ctx, pool.ID, 0, + entity.Int64(compute_v1alpha.SandboxPoolDesiredInstancesId, svcConc.NumInstances), + ); err != nil { + return restored, fmt.Errorf("restoring pool %s: %w", pool.ID, err) + } + restored++ + } + return restored, nil +} diff --git a/servers/app/disable_test.go b/servers/app/disable_test.go new file mode 100644 index 000000000..d5230cce0 --- /dev/null +++ b/servers/app/disable_test.go @@ -0,0 +1,186 @@ +package app + +import ( + "context" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/api/app/app_v1alpha" + "miren.dev/runtime/api/compute/compute_v1alpha" + "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/api/entityserver" + "miren.dev/runtime/metrics" + "miren.dev/runtime/pkg/apphealth" + "miren.dev/runtime/pkg/entity" + "miren.dev/runtime/pkg/entity/testutils" + "miren.dev/runtime/pkg/rpc" +) + +// Once the flag is clear the app is enabled, whatever happens next, so a failed +// pool restore must not report a failure that a retry can only contradict. +func TestEnableSucceedsWhenRestoringPoolsFails(t *testing.T) { + ctx := context.Background() + + inmem, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + ec := entityserver.NewClient(slog.Default(), inmem.EAC) + + client := &app_v1alpha.CrudClient{ + Client: rpc.LocalClient(app_v1alpha.AdaptCrud(&AppInfo{Log: slog.Default(), EC: ec})), + } + + appID, err := ec.Create(ctx, "restore-app", &core_v1alpha.App{ + ActiveVersion: entity.Id("app_version/missing"), + Disabled: core_v1alpha.Disabled{At: time.Now()}, + }) + require.NoError(t, err) + + res, err := client.Enable(ctx, "restore-app") + require.NoError(t, err) + assert.Zero(t, res.RestoredPools()) + + var app core_v1alpha.App + require.NoError(t, ec.GetById(ctx, appID, &app)) + assert.True(t, app.Disabled.Empty()) +} + +// TestAppDisableEnable walks an app with a fixed worker and an autoscaled web +// service through disable, a refused restart, and enable. +func TestAppDisableEnable(t *testing.T) { + ctx := context.Background() + + inmem, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + ec := entityserver.NewClient(slog.Default(), inmem.EAC) + + appInfo := &AppInfo{ + Log: slog.Default(), + EC: ec, + CPU: &metrics.CPUUsage{}, + Mem: &metrics.MemoryUsage{}, + HTTP: &metrics.HTTPMetrics{}, + } + client := &app_v1alpha.CrudClient{ + Client: rpc.LocalClient(app_v1alpha.AdaptCrud(appInfo)), + } + + appName := "disable-app" + appID, err := ec.Create(ctx, appName, &core_v1alpha.App{}) + require.NoError(t, err) + + verID, err := ec.Create(ctx, "disable-app-v1", &core_v1alpha.AppVersion{ + App: appID, + Version: "v1", + Config: core_v1alpha.Config{ + Services: []core_v1alpha.Services{ + {Name: "web", ServiceConcurrency: core_v1alpha.ServiceConcurrency{Mode: "auto", RequestsPerInstance: 10}}, + {Name: "worker", ServiceConcurrency: core_v1alpha.ServiceConcurrency{Mode: "fixed", NumInstances: 2}}, + }, + }, + }) + require.NoError(t, err) + require.NoError(t, ec.Patch(ctx, appID, 0, entity.Ref(core_v1alpha.AppActiveVersionId, verID))) + + webPool, err := ec.Create(ctx, "disable-app-web", &compute_v1alpha.SandboxPool{ + App: appID, Service: "web", DesiredInstances: 1, ReferencedByVersions: []entity.Id{verID}, + }) + require.NoError(t, err) + workerPool, err := ec.Create(ctx, "disable-app-worker", &compute_v1alpha.SandboxPool{ + App: appID, Service: "worker", DesiredInstances: 2, ReferencedByVersions: []entity.Id{verID}, + }) + require.NoError(t, err) + + desired := func(id entity.Id) int64 { + t.Helper() + var pool compute_v1alpha.SandboxPool + require.NoError(t, ec.GetById(ctx, id, &pool)) + return pool.DesiredInstances + } + getApp := func() core_v1alpha.App { + t.Helper() + var app core_v1alpha.App + require.NoError(t, ec.GetById(ctx, appID, &app)) + return app + } + + _, err = client.Enable(ctx, appName) + require.ErrorContains(t, err, "is not disabled") + + disabled, err := client.Disable(ctx, appName, "moved to the new box") + require.NoError(t, err) + assert.Equal(t, int32(2), disabled.ScaledPools()) + assert.Equal(t, int64(0), desired(webPool)) + assert.Equal(t, int64(0), desired(workerPool)) + app := getApp() + assert.False(t, app.Disabled.Empty()) + assert.Equal(t, "moved to the new box", app.Disabled.Reason) + disabledAt := app.Disabled.At + + src, err := appInfo.collectAppHealth(ctx) + require.NoError(t, err) + for _, entry := range src.apps { + if entry.name == appName { + assert.Equal(t, apphealth.Disabled, src.healthOf(entry).Health) + } + } + + _, err = client.Restart(ctx, appName, "") + require.ErrorContains(t, err, "is disabled", "restart would restore fixed counts and undo the disable") + + enabled, err := client.Enable(ctx, appName) + require.NoError(t, err) + assert.Equal(t, int32(1), enabled.RestoredPools()) + assert.Equal(t, int64(2), desired(workerPool), "fixed service returns to its configured count") + assert.Equal(t, int64(0), desired(webPool), "autoscaled service starts on the next request") + app = getApp() + assert.True(t, app.Disabled.Empty()) + assert.Empty(t, app.Disabled.Reason) + assert.False(t, app.SchedulesResumedAt.Before(disabledAt)) +} + +// An app can be disabled before it's ever deployed. Status has to say so +// rather than falling back to unknown for want of an active version, the same +// way list does. +func TestAppStatusReportsDisabledWithoutAnActiveVersion(t *testing.T) { + ctx := context.Background() + + inmem, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + ec := entityserver.NewClient(slog.Default(), inmem.EAC) + + // AppInfo reads the last hour of CPU and memory; an empty answer is enough. + vm := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[]}}`)) + })) + defer vm.Close() + reader := metrics.NewVictoriaMetricsReader(slog.Default(), vm.URL, 0) + + appInfo := &AppInfo{ + Log: slog.Default(), + EC: ec, + CPU: &metrics.CPUUsage{Reader: reader}, + Mem: &metrics.MemoryUsage{Reader: reader}, + HTTP: &metrics.HTTPMetrics{}, + } + crud := &app_v1alpha.CrudClient{Client: rpc.LocalClient(app_v1alpha.AdaptCrud(appInfo))} + status := app_v1alpha.NewAppStatusClient(rpc.LocalClient(app_v1alpha.AdaptAppStatus(appInfo))) + + _, err := ec.Create(ctx, "never-deployed", &core_v1alpha.App{}) + require.NoError(t, err) + + res, err := status.AppInfo(ctx, "never-deployed") + require.NoError(t, err) + assert.Equal(t, apphealth.Unknown, res.Status().Health(), "an enabled app with no version is unknown") + + _, err = crud.Disable(ctx, "never-deployed", "not needed yet") + require.NoError(t, err) + + res, err = status.AppInfo(ctx, "never-deployed") + require.NoError(t, err) + assert.Equal(t, apphealth.Disabled, res.Status().Health()) +} diff --git a/servers/app/runtime.go b/servers/app/runtime.go index 400be0f7e..8e32a3353 100644 --- a/servers/app/runtime.go +++ b/servers/app/runtime.go @@ -256,6 +256,16 @@ func (a *AppInfo) AppInfo(ctx context.Context, state *app_v1alpha.AppStatusAppIn rai.SetHealth(apphealth.Unknown) } + // Disabled is an explicit operator decision, so it wins over whatever the + // pools say, including when there's no active version or the pool lookup + // failed. List and the cloud health feed report it the same way. + if !appRec.Disabled.Empty() { + rai.SetHealth(apphealth.Disabled) + for _, svc := range rai.Services() { + svc.SetHealth(apphealth.Disabled) + } + } + // Get addon instances from entity store addonResults, err := a.EC.List(ctx, entity.Ref(addon_v1alpha.AddonAssociationAppId, appRec.ID)) if err != nil { diff --git a/servers/build/authorization.go b/servers/build/authorization.go index c26eaba7e..a9c8ef75a 100644 --- a/servers/build/authorization.go +++ b/servers/build/authorization.go @@ -5,7 +5,9 @@ import ( "errors" "miren.dev/runtime/api/app" + "miren.dev/runtime/api/core/core_v1alpha" "miren.dev/runtime/pkg/cond" + "miren.dev/runtime/pkg/deploylifecycle" "miren.dev/runtime/pkg/rpc" ) @@ -32,3 +34,19 @@ func (b *Builder) authorizeBuild(ctx context.Context, name string) error { Resource: "app", Action: "deploy", Instance: name, Owner: app.CreationSubject(ctx), }) } + +// refuseDisabled stops a build of a disabled app before any work happens. A +// deploy to an app that's off has no coherent behavior: its deploy tasks would +// boot sandboxes against whatever the app shares, and nothing could verify the +// new version without booting the app. Maintenance mode is for apps that should +// keep taking deploys. +func (b *Builder) refuseDisabled(ctx context.Context, name string) error { + var rec core_v1alpha.App + if err := b.ec.Get(ctx, name, &rec); err != nil { + if errors.Is(err, cond.ErrNotFound{}) { + return nil + } + return err + } + return deploylifecycle.CheckEnabled(name, &rec) +} diff --git a/servers/build/authorization_test.go b/servers/build/authorization_test.go index 6d428bad5..c78785601 100644 --- a/servers/build/authorization_test.go +++ b/servers/build/authorization_test.go @@ -5,10 +5,13 @@ import ( "errors" "strings" "testing" + "time" "github.com/stretchr/testify/require" "miren.dev/runtime/api/app" "miren.dev/runtime/api/build/build_v1alpha" + "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/pkg/entity" "miren.dev/runtime/pkg/oidcauth" "miren.dev/runtime/pkg/rpc" "miren.dev/runtime/pkg/rpc/stream" @@ -67,6 +70,26 @@ func TestPrepareUploadAuthorizesBeforeWriting(t *testing.T) { } } +func TestPrepareUploadRefusesDisabledApp(t *testing.T) { + b := newDeployTestBuilder(t) + ctx := context.Background() + rec, _, err := app.EnsureApp(ctx, b.EAS, "off", "owner") + require.NoError(t, err) + client := &build_v1alpha.BuilderClient{Client: rpc.LocalClient(build_v1alpha.AdaptBuilder(&SagaBuilder{inner: b}))} + + require.NoError(t, b.ec.Patch(ctx, rec.ID, 0, entity.Component(core_v1alpha.AppDisabledId, + (&core_v1alpha.Disabled{At: time.Now()}).Encode()))) + _, err = client.PrepareUpload(ctx, "off", nil) + require.ErrorContains(t, err, "is disabled") + + require.NoError(t, b.ec.Patch(ctx, rec.ID, 0, entity.Component(core_v1alpha.AppDisabledId, nil))) + _, err = client.PrepareUpload(ctx, "off", nil) + require.ErrorContains(t, err, "data path not configured", "an enabled app reaches the upload") + + _, err = client.PrepareUpload(ctx, "brand-new", nil) + require.ErrorContains(t, err, "data path not configured", "a first deploy has nothing to refuse") +} + func TestPreparedUploadDenialDoesNotConsumeSession(t *testing.T) { b := newDeployTestBuilder(t) sess := &buildSession{appName: "existing", subject: "owner", cancelFunc: func() { t.Fatal("denied caller canceled session") }} diff --git a/servers/build/build.go b/servers/build/build.go index 75dd9ec6d..2d9724df5 100644 --- a/servers/build/build.go +++ b/servers/build/build.go @@ -1385,6 +1385,9 @@ func (b *Builder) PrepareUpload(ctx context.Context, state *build_v1alpha.Builde if err := b.authorizeBuild(ctx, name); err != nil { return err } + if err := b.refuseDisabled(ctx, name); err != nil { + return err + } if b.DataPath == "" { return fmt.Errorf("data path not configured") diff --git a/servers/build/build_saga.go b/servers/build/build_saga.go index 4d0ce45ae..c20f9fff9 100644 --- a/servers/build/build_saga.go +++ b/servers/build/build_saga.go @@ -719,6 +719,13 @@ func runDeployTasks(ctx context.Context, in runDeployTasksIn) (runDeployTasksOut return runDeployTasksOut{}, fmt.Errorf("reading app: %w", err) } + // The build started before this app was disabled. Its tasks must not run. + if !appRec.Disabled.Empty() { + err := fmt.Errorf("app %s was disabled during this deploy; deploy tasks did not run", in.AppName) + status.SendError("%s", err) + return runDeployTasksOut{}, err + } + err = b.runDeployTasks(ctx, in.AppName, appRec.ID, entity.Id(in.AppVersionID), &spec, status) if err != nil { status.SendError("%s", err) diff --git a/servers/build/saga_builder.go b/servers/build/saga_builder.go index ef9702622..941542321 100644 --- a/servers/build/saga_builder.go +++ b/servers/build/saga_builder.go @@ -136,6 +136,9 @@ func (s *SagaBuilder) BuildFromTar(ctx context.Context, state *build_v1alpha.Bui if err := validateDeploymentMessage(args.EphemeralLabel(), args.Deployment()); err != nil { return err } + if err := s.inner.refuseDisabled(ctx, name); err != nil { + return err + } work := newDeploymentContext(ctx) defer work.Close() defer func() { retErr = work.result(retErr) }() @@ -198,6 +201,9 @@ func (s *SagaBuilder) BuildFromPrepared(ctx context.Context, state *build_v1alph if err := s.inner.authorizeBuild(ctx, name); err != nil { return err } + if err := s.inner.refuseDisabled(ctx, name); err != nil { + return err + } if !s.inner.sessions.CompareAndDelete(sessionID, val) { return fmt.Errorf("unknown or expired upload session: %s", sessionID) } diff --git a/servers/deployment/lock_integration_test.go b/servers/deployment/lock_integration_test.go index bd8475eb0..3c865f0f4 100644 --- a/servers/deployment/lock_integration_test.go +++ b/servers/deployment/lock_integration_test.go @@ -4,13 +4,16 @@ import ( "context" "log/slog" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "miren.dev/runtime/api/core/core_v1alpha" deployment_v1alpha "miren.dev/runtime/api/deployment/deployment_v1alpha" "miren.dev/runtime/pkg/deploylifecycle" + "miren.dev/runtime/pkg/entity" "miren.dev/runtime/pkg/entity/testutils" + ephemeralx "miren.dev/runtime/pkg/ephemeral" "miren.dev/runtime/pkg/rpc" ) @@ -219,3 +222,76 @@ func TestFailedSentinelNormalizedInHistory(t *testing.T) { assert.Equal(t, "", got.Deployment().AppVersionId(), "a legacy failed- sentinel must render as empty") } + +// A disabled app is frozen. These paths skip the build, so the build's refusal +// doesn't cover them. +func TestDeployVersionRefusesDisabledApp(t *testing.T) { + cases := []struct { + name string + rollback bool + label string + }{ + {name: "redeploy"}, + {name: "rollback", rollback: true}, + {name: "preview", label: "feat-x"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + client, inmem := newLockTestClient(t) + + appID, err := inmem.Client.Create(ctx, "web", &core_v1alpha.App{}) + require.NoError(t, err) + _, err = inmem.Client.Create(ctx, "web-v1", &core_v1alpha.AppVersion{Version: "web-v1"}) + require.NoError(t, err) + + disabled := core_v1alpha.Disabled{At: time.Now(), Reason: "parked"} + require.NoError(t, inmem.Client.Patch(ctx, appID, 0, + entity.Component(core_v1alpha.AppDisabledId, disabled.Encode()))) + + ttl := "" + if tc.label != "" { + ttl = "48h" + } + res, err := client.DeployVersion(ctx, "web", "prod", "web-v1", tc.rollback, nil, tc.label, ttl, "") + require.NoError(t, err) + require.True(t, res.HasError(), "a disabled app must refuse this") + assert.Contains(t, res.Error(), "miren app enable -a web") + + lock, err := client.GetDeployLock(ctx, "web", "prod") + require.NoError(t, err) + assert.False(t, lock.Held(), "a refused deploy must not acquire the lock") + + var app core_v1alpha.App + require.NoError(t, inmem.Client.Get(ctx, "web", &app)) + assert.Empty(t, app.ActiveVersion, "a refused deploy must not change what the app runs") + + if tc.label != "" { + preview, err := ephemeralx.LookupByLabel(ctx, inmem.EAC, appID, tc.label) + require.NoError(t, err) + assert.Nil(t, preview, "a refused preview must not leave a version behind") + } + }) + } +} + +func TestEnvChangesRefuseDisabledApp(t *testing.T) { + ctx := context.Background() + client, inmem := newLockTestClient(t) + + appID, err := inmem.Client.Create(ctx, "web", &core_v1alpha.App{}) + require.NoError(t, err) + disabled := core_v1alpha.Disabled{At: time.Now(), Reason: "parked"} + require.NoError(t, inmem.Client.Patch(ctx, appID, 0, + entity.Component(core_v1alpha.AppDisabledId, disabled.Encode()))) + + var v deployment_v1alpha.EnvironmentVariable + v.SetKey("FALLBACK_MODE") + v.SetValue("1") + _, err = client.SetEnvVars(ctx, "web", "prod", []*deployment_v1alpha.EnvironmentVariable{&v}, "") + require.ErrorContains(t, err, "app is disabled") + + _, err = client.DeleteEnvVars(ctx, "web", "prod", []string{"FALLBACK_MODE"}, "") + require.ErrorContains(t, err, "app is disabled") +} diff --git a/servers/deployment/server.go b/servers/deployment/server.go index cb36e5c28..3f8fce922 100644 --- a/servers/deployment/server.go +++ b/servers/deployment/server.go @@ -676,7 +676,8 @@ func (d *DeploymentServer) DeployVersion(ctx context.Context, req *deployment_v1 if err := d.verifyVersionOwnedByApp(ctx, &appVersion, appName); err != nil { return err } - if _, _, err := d.tracker.Store().AppByName(ctx, appName); err != nil { + targetApp, _, err := d.tracker.Store().AppByName(ctx, appName) + if err != nil { if errors.Is(err, cond.ErrNotFound{}) { results.SetError(fmt.Sprintf("app %q not found", appName)) return nil @@ -685,6 +686,12 @@ func (d *DeploymentServer) DeployVersion(ctx context.Context, req *deployment_v1 results.SetError("failed to look up app") return nil } + // Begin refuses a disabled app too, but a preview never gets that far and + // creates its version first. + if err := deploylifecycle.CheckEnabled(appName, targetApp); err != nil { + results.SetError(err.Error()) + return nil + } isEphemeral := args.HasEphemeralLabel() && args.EphemeralLabel() != "" if isEphemeral { diff --git a/servers/httpingress/disabled_test.go b/servers/httpingress/disabled_test.go new file mode 100644 index 000000000..6f37c43cb --- /dev/null +++ b/servers/httpingress/disabled_test.go @@ -0,0 +1,106 @@ +package httpingress + +import ( + "context" + "log/slog" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/api/entityserver/entityserver_v1alpha" + "miren.dev/runtime/api/ingress" + "miren.dev/runtime/api/ingress/ingress_v1alpha" + "miren.dev/runtime/components/activator" + "miren.dev/runtime/pkg/entity" + "miren.dev/runtime/pkg/entity/testutils" + "miren.dev/runtime/pkg/rpc" +) + +// A disabled app is checked before the lease cache. A warm app holds cached +// leases to sandboxes that keep running until the pool scales them down, so a +// check on the activator's scale-up path alone would leave it serving. +func TestDisabledAppStopsServingFromCachedLease(t *testing.T) { + var hits atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + _, _ = w.Write([]byte("from the app")) + })) + defer upstream.Close() + + inmem, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + ctx := context.Background() + + appID, err := inmem.Client.Create(ctx, "shop", &core_v1alpha.App{}) + require.NoError(t, err) + cvID, err := inmem.Client.Create(ctx, "cfg", &core_v1alpha.ConfigVersion{App: appID}) + require.NoError(t, err) + verID, err := inmem.Client.Create(ctx, "ver", &core_v1alpha.AppVersion{App: appID, ConfigVersion: cvID}) + require.NoError(t, err) + require.NoError(t, inmem.Client.Update(ctx, &core_v1alpha.App{ID: appID, ActiveVersion: verID})) + _, err = inmem.Client.Create(ctx, "shop.test", &ingress_v1alpha.HttpRoute{Host: "shop.test", App: appID}) + require.NoError(t, err) + + s := &Server{Log: slog.Default(), eac: inmem.EAC, aa: stubActivator{}, apps: make(map[string]*appUsage)} + s.ingressClient = ingress.NewClient(s.Log, rpc.LocalClient(entityserver_v1alpha.AdaptEntityAccess(inmem.Server))) + s.retainLease(ctx, leaseCacheKey(appID, "web", "", false), &activator.Lease{Size: 10, URL: upstream.URL}) + + do := func(accept string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + req := httptest.NewRequest("GET", "http://shop.test/", nil) + req.Header.Set("Accept", accept) + var appName string + s.serveHTTPWithMetrics(rec, req, &appName) + return rec + } + + rec := do("text/html") + require.Equal(t, http.StatusOK, rec.Code, "the cached lease serves while the app is enabled") + require.Equal(t, int32(1), hits.Load()) + + _, err = inmem.EAC.Patch(ctx, entity.New( + entity.Ref(entity.DBId, appID), + entity.Component(core_v1alpha.AppDisabledId, (&core_v1alpha.Disabled{At: time.Now(), Reason: "moved to the new cluster"}).Encode()), + ).Attrs(), 0) + require.NoError(t, err) + + rec = do("text/html") + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + assert.Equal(t, "no-store", rec.Header().Get("Cache-Control")) + assert.Contains(t, rec.Body.String(), "shop.test is unavailable") + assert.NotContains(t, rec.Body.String(), "moved to the new cluster", "the disable reason is an operator note, not visitor copy") + assert.NotContains(t, rec.Body.String(), "Try again") + + rec = do("application/json") + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + assert.JSONEq(t, `{"error":"disabled"}`, rec.Body.String()) + + rec = do("text/plain") + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + assert.Equal(t, "Unavailable\n", rec.Body.String()) + + assert.Equal(t, int32(1), hits.Load(), "no request reached the app after it was disabled") +} + +// A cluster template written for maintenance keys on .Maintenance. A disabled +// app sets it too, so that template still renders a holding page rather than +// falling through to its generic error branch. +func TestDisabledPageRendersThroughAMaintenanceTemplate(t *testing.T) { + page, err := parseErrorTemplate("{{if .Maintenance}}holding{{if .Disabled}} (disabled){{end}}{{else}}error{{end}}") + require.NoError(t, err) + s := newTestMaintenanceServer() + s.config.ErrorPageTemplate = page + + rec := httptest.NewRecorder() + req := httptest.NewRequest("GET", "http://shop.test/", nil) + req.Header.Set("Accept", "text/html") + s.serveDisabled(rec, req, &core_v1alpha.App{}) + + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + assert.Equal(t, "holding (disabled)", rec.Body.String()) +} diff --git a/servers/httpingress/error_page.go b/servers/httpingress/error_page.go index 42024c8f6..778d4c3a0 100644 --- a/servers/httpingress/error_page.go +++ b/servers/httpingress/error_page.go @@ -114,8 +114,12 @@ type errorPageData struct { Site string Reason string BackAt string + // Maintenance is true on every holding page; Disabled narrows it to an + // app turned off with `miren app disable`, so a template written for + // maintenance still renders a sensible page for a disabled app. Maintenance bool Loading bool + Disabled bool } // A bundled copy of docs/static/img/logo-light.svg keeps the real wordmark @@ -270,7 +274,7 @@ var errorPage = template.Must(template.New("ingress-error").Funcs(template.FuncM {{end}} -{{if .Loading}}Loading…{{else if .Maintenance}}Down for maintenance{{else}}{{.Status}} · {{.Title}}{{end}} — Miren +{{if .Loading}}Loading…{{else if .Disabled}}Unavailable{{else if .Maintenance}}Down for maintenance{{else}}{{.Status}} · {{.Title}}{{end}} — Miren