diff --git a/README.md b/README.md index 125cf6c..0c483b8 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ opt in. Environment variables set at the pipeline or stage level in `pipeline_fn` are handled automatically: variables identical across all regions in a group stay at -the stage level, while region-specific variables are cascaded to the job level. +their original level, while region-specific variables are cascaded to the job level. GoCD resolves precedence as job > stage > pipeline. ### Targeting a subset of regions diff --git a/libs/pipedream.libsonnet b/libs/pipedream.libsonnet index d9a3afb..17fbc5f 100644 --- a/libs/pipedream.libsonnet +++ b/libs/pipedream.libsonnet @@ -315,6 +315,25 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde true ); + // Returns the env vars whose value is identical across every region. + local common_env_vars(per_region_envs) = + local first = per_region_envs[regions[0]]; + { + [k]: first[k] + for k in std.objectFields(first) + if std.length(std.filter( + function(r) std.objectHas(per_region_envs[r], k) && per_region_envs[r][k] == first[k], + regions + )) == std.length(regions) + }; + + // Pipeline-level env vars shared by all regions stay at pipeline level so they + // show up in GoCD's "Trigger with options" dialog and can be overridden per run. + local common_pipeline_env = common_env_vars({ + [region]: get_pipeline_env_vars(region_pipelines[region]) + for region in regions + }); + // Collect all unique stages across all regions in the group. // If a region doesn't define a stage that another region has, // it simply contributes no jobs to that stage. @@ -339,8 +358,8 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde if std.length(matching) > 0 then matching[0] else null; // Transforms a stage by aggregating jobs from all regions. - // Env vars identical across all regions are kept at stage level; - // region-specific env vars are cascaded down to the job level. + // Env vars identical across all regions are kept at stage level (unless already + // set at pipeline level); region-specific env vars are cascaded down to the job level. local transform_stage(stage) = local stage_name = get_stage_name(stage); local stage_props = get_stage_props(stage); @@ -375,24 +394,23 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde for region in regions }; - // Env vars identical across ALL regions stay at stage level - local first_env = per_region_parent_envs[regions[0]]; + // Env vars identical across ALL regions stay at stage level. Ones already at + // pipeline level with the same value are dropped so a trigger-time override isn't shadowed. + local stage_common_env = common_env_vars(per_region_parent_envs); local common_env = { - [k]: first_env[k] - for k in std.objectFields(first_env) - if std.length(std.filter( - function(r) std.objectHas(per_region_parent_envs[r], k) && per_region_parent_envs[r][k] == first_env[k], - regions - )) == std.length(regions) + [k]: stage_common_env[k] + for k in std.objectFields(stage_common_env) + if !(std.objectHas(common_pipeline_env, k) && common_pipeline_env[k] == stage_common_env[k]) }; local all_jobs = std.foldl( function(acc, region) local parent_env = per_region_parent_envs[region]; + local inherited_env = common_pipeline_env + common_env; local region_specific_env = { [k]: parent_env[k] for k in std.objectFields(parent_env) - if !std.objectHas(common_env, k) || common_env[k] != parent_env[k] + if !std.objectHas(inherited_env, k) || inherited_env[k] != parent_env[k] }; local p = region_pipelines[region]; local region_stage = get_matching_stage(p, stage_name); @@ -464,7 +482,7 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde for stage in all_stages ]; - // Strip pipeline and stage level environment variables + // Strip pipeline-level environment variables; shared ones are re-added below. local filtered_template = { [k]: template_pipeline[k] for k in std.objectFields(template_pipeline) @@ -477,7 +495,7 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde filtered_template { group: service_name, display_order: display_order, - environment_variables: { + environment_variables: common_pipeline_env { PIPEDREAM_GROUP_REGIONS: std.join(',', regions), }, stages: prepend_stages + transformed_stages + [ diff --git a/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet b/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet index c7b5a0c..77fd410 100644 --- a/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet +++ b/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet @@ -13,6 +13,7 @@ local pipeline_fn(region) = { // Pipeline-level env vars environment_variables: { PIPELINE_VAR: 'pipeline-' + region, // Should cascade down to becoming a job level var + PIPELINE_SHARED_VAR: 'from-pipeline', // Identical across regions, stays at pipeline level SHARED_VAR_JOB: 'from-pipeline', // Should be overwritten by stage, then job SHARED_VAR_STAGE: 'from-pipeline', // Overridden by stage }, diff --git a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden index 57c8d9d..99383ed 100644 --- a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden @@ -5,7 +5,11 @@ "deploy-example-s4s2": { "display_order": 2, "environment_variables": { - "PIPEDREAM_GROUP_REGIONS": "s4s2" + "PIPEDREAM_GROUP_REGIONS": "s4s2", + "PIPELINE_SHARED_VAR": "from-pipeline", + "PIPELINE_VAR": "pipeline-s4s2", + "SHARED_VAR_JOB": "from-pipeline", + "SHARED_VAR_STAGE": "from-pipeline" }, "group": "example", "materials": { @@ -18,7 +22,6 @@ { "deploy": { "environment_variables": { - "PIPELINE_VAR": "pipeline-s4s2", "SHARED_VAR_JOB": "from-stage", "SHARED_VAR_STAGE": "from-stage", "STAGE_VAR": "stage-s4s2" @@ -64,7 +67,10 @@ "deploy-example-st": { "display_order": 3, "environment_variables": { - "PIPEDREAM_GROUP_REGIONS": "customer-1,customer-2,customer-7" + "PIPEDREAM_GROUP_REGIONS": "customer-1,customer-2,customer-7", + "PIPELINE_SHARED_VAR": "from-pipeline", + "SHARED_VAR_JOB": "from-pipeline", + "SHARED_VAR_STAGE": "from-pipeline" }, "group": "example", "materials": { diff --git a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden index b513388..6058916 100644 --- a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden @@ -4,7 +4,11 @@ "deploy-example-s4s2": { "display_order": 2, "environment_variables": { - "PIPEDREAM_GROUP_REGIONS": "s4s2" + "PIPEDREAM_GROUP_REGIONS": "s4s2", + "PIPELINE_SHARED_VAR": "from-pipeline", + "PIPELINE_VAR": "pipeline-s4s2", + "SHARED_VAR_JOB": "from-pipeline", + "SHARED_VAR_STAGE": "from-pipeline" }, "group": "example", "materials": { @@ -17,7 +21,6 @@ { "deploy": { "environment_variables": { - "PIPELINE_VAR": "pipeline-s4s2", "SHARED_VAR_JOB": "from-stage", "SHARED_VAR_STAGE": "from-stage", "STAGE_VAR": "stage-s4s2" @@ -58,7 +61,10 @@ "deploy-example-st": { "display_order": 3, "environment_variables": { - "PIPEDREAM_GROUP_REGIONS": "customer-1,customer-2,customer-7" + "PIPEDREAM_GROUP_REGIONS": "customer-1,customer-2,customer-7", + "PIPELINE_SHARED_VAR": "from-pipeline", + "SHARED_VAR_JOB": "from-pipeline", + "SHARED_VAR_STAGE": "from-pipeline" }, "group": "example", "materials": {