From b1aa2eb9622cb4f9e2eb1e5878775adc0cc5f6aa Mon Sep 17 00:00:00 2001 From: Minura Kariyawasam Date: Mon, 17 Aug 2026 10:41:31 +0300 Subject: [PATCH] Fix startup xDS snapshot using legacy cluster names after controller restart The initial Envoy xDS snapshot was generated before the transformer registry was wired into the translator. With t.transformers still nil, the translator silently fell back to the legacy translation path, which names clusters "cluster__", while the policy engine's routes (built later via the transformer path) reference "upstream___". After any controller restart, every request to a previously deployed API failed with 503 cluster_not_found until the API was manually redeployed, since control-plane reconciliation skips APIs whose deployed_at is unchanged. Fix: build the transformer registry and call translator.SetTransformers before the initial snapshot is generated, so the very first snapshot already uses transformer-path naming. This also avoids the transient mismatch window a regenerate-after-wiring approach would leave for already-running gateway runtimes that reconnect as soon as the xDS server starts. The event-gateway controller has the same initialization order, so the same reorder is applied there. Also log a warning when the translator falls back to the legacy path for a non-WebSubApi kind, so this class of regression is visible in logs instead of failing silently. Resolves wso2/api-platform#3197 --- .../gateway-controller/cmd/controller/main.go | 33 ++++++++----- .../gateway-controller/cmd/controller/main.go | 49 +++++++++++-------- .../gateway-controller/pkg/xds/translator.go | 12 ++++- 3 files changed, 62 insertions(+), 32 deletions(-) diff --git a/event-gateway/gateway-controller/cmd/controller/main.go b/event-gateway/gateway-controller/cmd/controller/main.go index da78d64a99..05b20f4bb9 100644 --- a/event-gateway/gateway-controller/cmd/controller/main.go +++ b/event-gateway/gateway-controller/cmd/controller/main.go @@ -389,6 +389,26 @@ func main() { } } + // Build the transformer registry and wire it into the Envoy translator BEFORE + // the initial xDS snapshot below, so the first snapshot already uses the + // transformer-path cluster/route names ("upstream___") that the + // policy engine's resources reference. Wiring it later would leave the startup + // snapshot on the legacy naming path ("cluster__"), breaking every + // previously deployed non-WebSub API with 503 cluster_not_found after a controller + // restart (issue #3197). WebSubApi is intentionally excluded so it keeps using the + // async-specific legacy translation path. + policyVersionResolver := utils.NewLoadedPolicyVersionResolver(policyDefinitions) + restTransformer := transform.NewRestAPITransformer(&cfg.Router, cfg, policyDefinitions) + llmTransformer := transform.NewLLMTransformer(configStore, db, &cfg.Router, cfg, policyDefinitions, policyVersionResolver) + transformerRegistry := transform.NewRegistry(restTransformer, llmTransformer) + + xdsTranslator.SetTransformers(map[string]models.ConfigTransformer{ + "RestApi": transformerRegistry, + "Mcp": transformerRegistry, + "LlmProvider": transformerRegistry, + "LlmProxy": transformerRegistry, + }) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) if err := snapshotManager.UpdateSnapshot(ctx, ""); err != nil { log.Warn("Failed to generate initial xDS snapshot", slog.Any("error", err)) @@ -427,19 +447,10 @@ func main() { policyManager := policyxds.NewPolicyManager(policySnapshotManager, log) policyManager.SetRuntimeStore(runtimeStore) - policyVersionResolver := utils.NewLoadedPolicyVersionResolver(policyDefinitions) - restTransformer := transform.NewRestAPITransformer(&cfg.Router, cfg, policyDefinitions) - llmTransformer := transform.NewLLMTransformer(configStore, db, &cfg.Router, cfg, policyDefinitions, policyVersionResolver) - transformerRegistry := transform.NewRegistry(restTransformer, llmTransformer) + // Share the transformer registry (built before the initial xDS snapshot above) + // with the policy manager so both snapshot paths key resources identically. policyManager.SetTransformers(transformerRegistry) - xdsTranslator.SetTransformers(map[string]models.ConfigTransformer{ - "RestApi": transformerRegistry, - "Mcp": transformerRegistry, - "LlmProvider": transformerRegistry, - "LlmProxy": transformerRegistry, - }) - loadedAPIs := configStore.GetAll() if _, err := loadRuntimeConfigsFromExistingAPIConfigurations(loadedAPIs, runtimeStore, secretsService, transformerRegistry, log, cfg.Controller.Server.SkipInvalidDeploymentsOnStartup); err != nil { log.Error("Failed to load runtime configs from API configurations", slog.Any("error", err)) diff --git a/gateway/gateway-controller/cmd/controller/main.go b/gateway/gateway-controller/cmd/controller/main.go index ec2549f673..5f74ba89f5 100644 --- a/gateway/gateway-controller/cmd/controller/main.go +++ b/gateway/gateway-controller/cmd/controller/main.go @@ -376,6 +376,33 @@ func main() { } } + // Build transformer registry for StoredConfig → RuntimeDeployConfig conversion. + // This MUST happen before the initial xDS snapshot below: the Envoy translator + // and the policy engine must agree on cluster names ("upstream___" + // from the transformer path). With no transformers wired the translator silently + // falls back to the legacy path, which names clusters "cluster__" — + // the policy engine then routes to upstream_* clusters that don't exist in Envoy, + // and every API returns 503 cluster_not_found until it is redeployed (issue #3197). + policyVersionResolver := utils.NewLoadedPolicyVersionResolver(policyDefinitions) + restTransformer := transform.NewRestAPITransformer(&cfg.Router, cfg, policyDefinitions) + llmTransformer := transform.NewLLMTransformer(configStore, db, &cfg.Router, cfg, policyDefinitions, policyVersionResolver) + transformerRegistry := transform.NewRegistry(restTransformer, llmTransformer) + + // Wire the transformer into the Envoy xDS translator so Envoy routes are built from the + // RuntimeDeployConfig (RDC) path — identical to how the policy engine's RouteConfig/PolicyChain + // resources are keyed. Without this the Envoy translator falls back to the legacy per-operation + // path, which (a) does not render header matchers and (b) names routes "method|path|vhost" + // (3 segments), while the policy resources are keyed "method|path|vhost|". The + // policy engine resolves the chain by the Envoy route name, so the mismatch makes every + // header-matched route fail with 500 ("policy chain not found"). WebSubApi is intentionally + // excluded so it keeps using the async-specific legacy translation path. + translator.SetTransformers(map[string]models.ConfigTransformer{ + "RestApi": transformerRegistry, + "Mcp": transformerRegistry, + "LlmProvider": transformerRegistry, + "LlmProxy": transformerRegistry, + }) + // Generate initial xDS snapshot log.Info("Generating initial xDS snapshot") ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) @@ -426,28 +453,10 @@ func main() { policyManager := policyxds.NewPolicyManager(policySnapshotManager, log) policyManager.SetRuntimeStore(runtimeStore) - // Build transformer registry for StoredConfig → RuntimeDeployConfig conversion - policyVersionResolver := utils.NewLoadedPolicyVersionResolver(policyDefinitions) - restTransformer := transform.NewRestAPITransformer(&cfg.Router, cfg, policyDefinitions) - llmTransformer := transform.NewLLMTransformer(configStore, db, &cfg.Router, cfg, policyDefinitions, policyVersionResolver) - transformerRegistry := transform.NewRegistry(restTransformer, llmTransformer) + // Share the transformer registry (built before the initial xDS snapshot above) + // with the policy manager so both snapshot paths key resources identically. policyManager.SetTransformers(transformerRegistry) - // Wire the same transformer into the Envoy xDS translator so Envoy routes are built from the - // RuntimeDeployConfig (RDC) path — identical to how the policy engine's RouteConfig/PolicyChain - // resources are keyed. Without this the Envoy translator falls back to the legacy per-operation - // path, which (a) does not render header matchers and (b) names routes "method|path|vhost" - // (3 segments), while the policy resources are keyed "method|path|vhost|". The - // policy engine resolves the chain by the Envoy route name, so the mismatch makes every - // header-matched route fail with 500 ("policy chain not found"). WebSubApi is intentionally - // excluded so it keeps using the async-specific legacy translation path. - translator.SetTransformers(map[string]models.ConfigTransformer{ - "RestApi": transformerRegistry, - "Mcp": transformerRegistry, - "LlmProvider": transformerRegistry, - "LlmProxy": transformerRegistry, - }) - // Load runtime configs from existing API configurations on startup. // We write directly to runtimeStore to avoid triggering N separate snapshot updates; // the single UpdateSnapshot call below covers all of them. diff --git a/gateway/gateway-controller/pkg/xds/translator.go b/gateway/gateway-controller/pkg/xds/translator.go index ae27e6832d..b4d55a2365 100644 --- a/gateway/gateway-controller/pkg/xds/translator.go +++ b/gateway/gateway-controller/pkg/xds/translator.go @@ -719,7 +719,17 @@ func (t *Translator) TranslateConfigs( var err error // Try RuntimeDeployConfig transformer path first (produces minimal metadata routes) - if transformer, ok := t.transformers[cfg.Kind]; ok { + transformer, ok := t.transformers[cfg.Kind] + if !ok && cfg.Kind != "WebSubApi" { + // Transformers should be registered for every non-WebSub kind before the + // first snapshot is generated. Falling back here means Envoy cluster/route + // names will not match the policy engine's resources (503 cluster_not_found / + // 500 policy chain not found) — see issue #3197. + log.Warn("No transformer registered for config kind, using legacy translation path", + slog.String("id", cfg.UUID), + slog.String("kind", cfg.Kind)) + } + if ok { rdc, transformErr := transformer.Transform(cfg) if transformErr != nil { log.Error("Failed to transform config via RuntimeDeployConfig, falling back to legacy path",