From a9f167d36ab7db6bb4a47cc032616e5d6c814e96 Mon Sep 17 00:00:00 2001 From: mchen-sentry Date: Thu, 10 Sep 2026 15:44:07 -0700 Subject: [PATCH 1/2] fix(pipedream): keep shared pipeline-level env vars at pipeline level Grouped pipelines demoted every service env var to stage or job level, so variables like SKIP_CANARY_CHECKS disappeared from GoCD's Trigger with options dialog and could no longer be overridden per run. Pipeline-level vars identical across all regions in a group now stay at pipeline level. --- README.md | 8 ++-- libs/pipedream.libsonnet | 44 +++++++++++++------ .../pipedream/env-vars-precedence.jsonnet | 1 + ...ars-precedence.jsonnet_output-files.golden | 12 +++-- ...vars-precedence.jsonnet_single-file.golden | 12 +++-- 5 files changed, 55 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 125cf6c..2076bdb 100644 --- a/README.md +++ b/README.md @@ -99,9 +99,11 @@ group (e.g. `deploy-example-s4s2`, `deploy-example-st`). opt in. Environment variables set at the pipeline or stage level in `pipeline_fn` are -handled automatically: variables identical across all regions in a group stay at -the stage level, while region-specific variables are cascaded to the job level. -GoCD resolves precedence as job > stage > pipeline. +handled automatically: pipeline-level variables identical across all regions in +a group stay at the pipeline level (so they can be overridden from GoCD's +"Trigger with options" dialog), stage-level variables identical across all +regions stay at the stage level, and region-specific variables are cascaded to +the job level. GoCD resolves precedence as job > stage > pipeline. ### Targeting a subset of regions diff --git a/libs/pipedream.libsonnet b/libs/pipedream.libsonnet index d9a3afb..17fbc5f 100644 --- a/libs/pipedream.libsonnet +++ b/libs/pipedream.libsonnet @@ -315,6 +315,25 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde true ); + // Returns the env vars whose value is identical across every region. + local common_env_vars(per_region_envs) = + local first = per_region_envs[regions[0]]; + { + [k]: first[k] + for k in std.objectFields(first) + if std.length(std.filter( + function(r) std.objectHas(per_region_envs[r], k) && per_region_envs[r][k] == first[k], + regions + )) == std.length(regions) + }; + + // Pipeline-level env vars shared by all regions stay at pipeline level so they + // show up in GoCD's "Trigger with options" dialog and can be overridden per run. + local common_pipeline_env = common_env_vars({ + [region]: get_pipeline_env_vars(region_pipelines[region]) + for region in regions + }); + // Collect all unique stages across all regions in the group. // If a region doesn't define a stage that another region has, // it simply contributes no jobs to that stage. @@ -339,8 +358,8 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde if std.length(matching) > 0 then matching[0] else null; // Transforms a stage by aggregating jobs from all regions. - // Env vars identical across all regions are kept at stage level; - // region-specific env vars are cascaded down to the job level. + // Env vars identical across all regions are kept at stage level (unless already + // set at pipeline level); region-specific env vars are cascaded down to the job level. local transform_stage(stage) = local stage_name = get_stage_name(stage); local stage_props = get_stage_props(stage); @@ -375,24 +394,23 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde for region in regions }; - // Env vars identical across ALL regions stay at stage level - local first_env = per_region_parent_envs[regions[0]]; + // Env vars identical across ALL regions stay at stage level. Ones already at + // pipeline level with the same value are dropped so a trigger-time override isn't shadowed. + local stage_common_env = common_env_vars(per_region_parent_envs); local common_env = { - [k]: first_env[k] - for k in std.objectFields(first_env) - if std.length(std.filter( - function(r) std.objectHas(per_region_parent_envs[r], k) && per_region_parent_envs[r][k] == first_env[k], - regions - )) == std.length(regions) + [k]: stage_common_env[k] + for k in std.objectFields(stage_common_env) + if !(std.objectHas(common_pipeline_env, k) && common_pipeline_env[k] == stage_common_env[k]) }; local all_jobs = std.foldl( function(acc, region) local parent_env = per_region_parent_envs[region]; + local inherited_env = common_pipeline_env + common_env; local region_specific_env = { [k]: parent_env[k] for k in std.objectFields(parent_env) - if !std.objectHas(common_env, k) || common_env[k] != parent_env[k] + if !std.objectHas(inherited_env, k) || inherited_env[k] != parent_env[k] }; local p = region_pipelines[region]; local region_stage = get_matching_stage(p, stage_name); @@ -464,7 +482,7 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde for stage in all_stages ]; - // Strip pipeline and stage level environment variables + // Strip pipeline-level environment variables; shared ones are re-added below. local filtered_template = { [k]: template_pipeline[k] for k in std.objectFields(template_pipeline) @@ -477,7 +495,7 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde filtered_template { group: service_name, display_order: display_order, - environment_variables: { + environment_variables: common_pipeline_env { PIPEDREAM_GROUP_REGIONS: std.join(',', regions), }, stages: prepend_stages + transformed_stages + [ diff --git a/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet b/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet index c7b5a0c..77fd410 100644 --- a/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet +++ b/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet @@ -13,6 +13,7 @@ local pipeline_fn(region) = { // Pipeline-level env vars environment_variables: { PIPELINE_VAR: 'pipeline-' + region, // Should cascade down to becoming a job level var + PIPELINE_SHARED_VAR: 'from-pipeline', // Identical across regions, stays at pipeline level SHARED_VAR_JOB: 'from-pipeline', // Should be overwritten by stage, then job SHARED_VAR_STAGE: 'from-pipeline', // Overridden by stage }, diff --git a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden index 57c8d9d..99383ed 100644 --- a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden @@ -5,7 +5,11 @@ "deploy-example-s4s2": { "display_order": 2, "environment_variables": { - "PIPEDREAM_GROUP_REGIONS": "s4s2" + "PIPEDREAM_GROUP_REGIONS": "s4s2", + "PIPELINE_SHARED_VAR": "from-pipeline", + "PIPELINE_VAR": "pipeline-s4s2", + "SHARED_VAR_JOB": "from-pipeline", + "SHARED_VAR_STAGE": "from-pipeline" }, "group": "example", "materials": { @@ -18,7 +22,6 @@ { "deploy": { "environment_variables": { - "PIPELINE_VAR": "pipeline-s4s2", "SHARED_VAR_JOB": "from-stage", "SHARED_VAR_STAGE": "from-stage", "STAGE_VAR": "stage-s4s2" @@ -64,7 +67,10 @@ "deploy-example-st": { "display_order": 3, "environment_variables": { - "PIPEDREAM_GROUP_REGIONS": "customer-1,customer-2,customer-7" + "PIPEDREAM_GROUP_REGIONS": "customer-1,customer-2,customer-7", + "PIPELINE_SHARED_VAR": "from-pipeline", + "SHARED_VAR_JOB": "from-pipeline", + "SHARED_VAR_STAGE": "from-pipeline" }, "group": "example", "materials": { diff --git a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden index b513388..6058916 100644 --- a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden @@ -4,7 +4,11 @@ "deploy-example-s4s2": { "display_order": 2, "environment_variables": { - "PIPEDREAM_GROUP_REGIONS": "s4s2" + "PIPEDREAM_GROUP_REGIONS": "s4s2", + "PIPELINE_SHARED_VAR": "from-pipeline", + "PIPELINE_VAR": "pipeline-s4s2", + "SHARED_VAR_JOB": "from-pipeline", + "SHARED_VAR_STAGE": "from-pipeline" }, "group": "example", "materials": { @@ -17,7 +21,6 @@ { "deploy": { "environment_variables": { - "PIPELINE_VAR": "pipeline-s4s2", "SHARED_VAR_JOB": "from-stage", "SHARED_VAR_STAGE": "from-stage", "STAGE_VAR": "stage-s4s2" @@ -58,7 +61,10 @@ "deploy-example-st": { "display_order": 3, "environment_variables": { - "PIPEDREAM_GROUP_REGIONS": "customer-1,customer-2,customer-7" + "PIPEDREAM_GROUP_REGIONS": "customer-1,customer-2,customer-7", + "PIPELINE_SHARED_VAR": "from-pipeline", + "SHARED_VAR_JOB": "from-pipeline", + "SHARED_VAR_STAGE": "from-pipeline" }, "group": "example", "materials": { From 00e7a14e66b660abbd1a8644b17e0d59b9aa9ece Mon Sep 17 00:00:00 2001 From: mchen-sentry Date: Thu, 10 Sep 2026 15:48:00 -0700 Subject: [PATCH 2/2] docs: trim README wording --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2076bdb..0c483b8 100644 --- a/README.md +++ b/README.md @@ -99,11 +99,9 @@ group (e.g. `deploy-example-s4s2`, `deploy-example-st`). opt in. Environment variables set at the pipeline or stage level in `pipeline_fn` are -handled automatically: pipeline-level variables identical across all regions in -a group stay at the pipeline level (so they can be overridden from GoCD's -"Trigger with options" dialog), stage-level variables identical across all -regions stay at the stage level, and region-specific variables are cascaded to -the job level. GoCD resolves precedence as job > stage > pipeline. +handled automatically: variables identical across all regions in a group stay at +their original level, while region-specific variables are cascaded to the job level. +GoCD resolves precedence as job > stage > pipeline. ### Targeting a subset of regions