diff --git a/.github/workflows/10_feature_dbt_checks.yml b/.github/workflows/10_feature_dbt_checks.yml index 90e040068..f25d28e51 100644 --- a/.github/workflows/10_feature_dbt_checks.yml +++ b/.github/workflows/10_feature_dbt_checks.yml @@ -124,12 +124,12 @@ jobs: # There is an issue with --empty and dynamic tables so need to exclude them - name: Governance run of dbt with EMPTY models using slim mode if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' }} - run: "dbt build --fail-fast --defer --state logs --select state:modified+ --empty --exclude config.materialized:dynamic_table ${{ env.FULL_REFRESH_FLAG }}" + run: "dbt build --fail-fast --defer --state logs --select state:modified+ --empty --exclude config.materialized:dynamic_table tag:requires_fixture_data ${{ env.FULL_REFRESH_FLAG }}" # There is an issue with --empty and dynamic tables so need to exclude - name: Governance run of dbt with EMPTY models using full run if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }} - run: "dbt build --fail-fast --empty --exclude config.materialized:dynamic_table ${{ env.FULL_REFRESH_FLAG }}" + run: "dbt build --fail-fast --empty --exclude config.materialized:dynamic_table tag:requires_fixture_data ${{ env.FULL_REFRESH_FLAG }}" - name: Generate Docs Combining Prod and branch catalog.json if: ${{ steps.prod_manifest.outputs.catalog_found == 'true' }} diff --git a/orchestrate/dags/other_examples/test_oom.py b/orchestrate/dags/other_examples/test_oom.py index 42b2eb299..d81ee546c 100644 --- a/orchestrate/dags/other_examples/test_oom.py +++ b/orchestrate/dags/other_examples/test_oom.py @@ -21,7 +21,7 @@ "start_date": datetime(2024, 1, 1), "owner": "Fernando Mercado", "email_on_failure": False, - "retries": 0, + "retries": 3, } @@ -30,7 +30,7 @@ catchup=False, schedule=None, default_args=default_args, - tags=["test", "oom"], + tags=["sample", "maintenance"], dag_id="test_oom", ) def test_oom(): diff --git a/secure/snowcap/apply.sh b/secure/snowcap/apply.sh index 0a6ddd8be..1b37e4530 100755 --- a/secure/snowcap/apply.sh +++ b/secure/snowcap/apply.sh @@ -75,7 +75,7 @@ if $USE_PII; then CONFIG_PATHS="--config $COMBINED_CONFIG" else ACCOUNT_TO_USE="$SNOWFLAKE_ACCOUNT" - EXCLUDE_RESOURCES="--exclude masking_policy,tag,tag_reference,tag_masking_policy_reference,row_access_policy" + EXCLUDE_RESOURCES="--exclude masking_policy,tag,tag_reference,tag_masking_policy_reference,row_access_policy,stream" USE_ACCOUNT_USAGE="" # Standard account - only include base resources CONFIG_PATHS="--config resources/" diff --git a/secure/snowcap/plan.sh b/secure/snowcap/plan.sh index 31ca6e95c..2dbafcacf 100755 --- a/secure/snowcap/plan.sh +++ b/secure/snowcap/plan.sh @@ -70,7 +70,7 @@ if $USE_PII; then CONFIG_PATHS="--config $COMBINED_CONFIG" else ACCOUNT_TO_USE="$SNOWFLAKE_ACCOUNT" - EXCLUDE_RESOURCES="--exclude masking_policy,tag,tag_reference,tag_masking_policy_reference,row_access_policy" + EXCLUDE_RESOURCES="--exclude masking_policy,tag,tag_reference,tag_masking_policy_reference,row_access_policy,stream" USE_ACCOUNT_USAGE="" # Standard account - only include base resources CONFIG_PATHS="--config resources/" diff --git a/secure/snowcap/resources/object_templates/schema.yml b/secure/snowcap/resources/object_templates/schema.yml index 30023f34b..edb9f4ccc 100644 --- a/secure/snowcap/resources/object_templates/schema.yml +++ b/secure/snowcap/resources/object_templates/schema.yml @@ -6,6 +6,14 @@ schemas: owner: "{{ each.value.get('owner', parent.owner) }}" managed_access: true + # BALBOA_QA companion schemas share the BALBOA schema role below. + - for_each: var.schemas + where: "each.value.name.split('.')[0] == 'BALBOA'" + name: "{{ each.value.name.split('.')[1] }}" + database: BALBOA_QA + owner: "{{ each.value.get('owner', parent.owner) }}" + managed_access: true + # Schema roles roles: - for_each: var.schemas @@ -17,3 +25,10 @@ grants: priv: USAGE on: "schema {{ each.value.name }}" to: "z_schema__{{ each.value.name.split('.')[1] }}" + + # QA schema grants + - for_each: var.schemas + where: "each.value.name.split('.')[0] == 'BALBOA'" + priv: USAGE + on: "schema BALBOA_QA.{{ each.value.name.split('.')[1] }}" + to: "z_schema__{{ each.value.name.split('.')[1] }}" diff --git a/secure/snowcap/resources/roles__functional.yml b/secure/snowcap/resources/roles__functional.yml index d1bb48588..46986d512 100644 --- a/secure/snowcap/resources/roles__functional.yml +++ b/secure/snowcap/resources/roles__functional.yml @@ -44,6 +44,9 @@ role_grants: - to_role: finance_team roles: - z_db__balboa + - z_db__balboa_qa + - z_tables_views__select + - z_schema__l3_accounts_payable - z_schema__l3_loan_analytics - z_wh__wh_finance @@ -63,13 +66,18 @@ role_grants: - z_schema__l1_google_analytics_4 - z_schema__l1_loans - z_schema__l1_observe + - z_schema__l1_erp - z_schema__l1_usgs__earthquake_data - z_schema__l1_us_population - z_schema__l2_country_demographics - z_schema__l2_covid_observations + - z_schema__l2_invoices + - z_schema__l2_purchase_orders - z_schema__l2_snowflake_usage + - z_schema__l2_vendors + - z_schema__l3_accounts_payable - z_schema__l3_covid_analytics - z_schema__l3_earthquake_analytics - z_schema__l3_loan_analytics diff --git a/secure/snowcap/resources/schemas.yml b/secure/snowcap/resources/schemas.yml index e5ad0ff45..e24c509f3 100644 --- a/secure/snowcap/resources/schemas.yml +++ b/secure/snowcap/resources/schemas.yml @@ -23,13 +23,18 @@ vars: - name: BALBOA.L1_GOOGLE_ANALYTICS_4 - name: BALBOA.L1_LOANS - name: BALBOA.L1_OBSERVE + - name: BALBOA.L1_ERP - name: BALBOA.L1_USGS__EARTHQUAKE_DATA - name: BALBOA.L1_US_POPULATION - name: BALBOA.L2_COUNTRY_DEMOGRAPHICS - name: BALBOA.L2_COVID_OBSERVATIONS + - name: BALBOA.L2_INVOICES + - name: BALBOA.L2_PURCHASE_ORDERS - name: BALBOA.L2_SNOWFLAKE_USAGE + - name: BALBOA.L2_VENDORS + - name: BALBOA.L3_ACCOUNTS_PAYABLE - name: BALBOA.L3_COVID_ANALYTICS - name: BALBOA.L3_EARTHQUAKE_ANALYTICS - name: BALBOA.L3_LOAN_ANALYTICS @@ -42,23 +47,3 @@ vars: # GREAT_BAY DB - name: GREAT_BAY.COVE_MARKETING -grants: - # BALBOA_QA is a clone of BALBOA (transform/macros/tooling/blue-green). - # CREATE DATABASE ... CLONE copies grants on child objects, so every - # z_schema__ role already reaches the QA copy of its schema -- these - # grants exist in Snowflake whether or not we declare them. Declaring them - # keeps sync from revoking them. - # - # Deliberately per-schema rather than "all schemas in database balboa_qa": - # roles are scoped by layer (finance_team sees only L3_LOAN_ANALYTICS), and a - # database-wide grant would hand every such role the whole QA database. - # - # The QA schemas themselves stay undeclared -- the clone creates them, and - # the `where` keeps this off schemas in RAW, GOVERNANCE and the rest, which - # have no QA counterpart. - - for_each: var.schemas - where: "each.value.name.split('.')[0] == 'BALBOA'" - priv: "USAGE" - on: "schema BALBOA_QA.{{ each.value.name.split('.')[1] }}" - to: "z_schema__{{ each.value.name.split('.')[1] }}" - diff --git a/transform/dbt_project.yml b/transform/dbt_project.yml index 7f1b2044a..2394bc732 100644 --- a/transform/dbt_project.yml +++ b/transform/dbt_project.yml @@ -74,6 +74,8 @@ models: +schema: L1_COVID19_EPIDEMIOLOGICAL_DATA loans: +schema: L1_LOANS + erp: + +schema: L1_ERP observe: +schema: L1_OBSERVE us_population: @@ -91,6 +93,12 @@ models: +schema: L2_COVID_OBSERVATIONS snowflake_usage: +schema: L2_SNOWFLAKE_USAGE + vendors: + +schema: L2_VENDORS + invoices: + +schema: L2_INVOICES + purchase_orders: + +schema: L2_PURCHASE_ORDERS L3_coves: +group: marketing @@ -103,6 +111,8 @@ models: +schema: L3_EARTHQUAKE_ANALYTICS loan_analytics: +schema: L3_LOAN_ANALYTICS + accounts_payable: + +schema: L3_ACCOUNTS_PAYABLE # cannot persist docs on dynamic tables # +persist_docs: # relation: false @@ -153,6 +163,8 @@ data_tests: vars: 'dbt_date:time_zone': 'America/Los_Angeles' + authored_history_through: '2025-12-01' + as_of: '2026-09-08' # Snowcap governance macros snowcap_tag_database: "GOVERNANCE" diff --git a/transform/macros/spine.sql b/transform/macros/spine.sql new file mode 100644 index 000000000..85ea8ec4e --- /dev/null +++ b/transform/macros/spine.sql @@ -0,0 +1,15 @@ +{% macro trailing_edge_month() %} + {% set as_of = var('as_of') %} + date_trunc('month', dateadd(month, -1, to_date('{{ as_of }}'))) +{% endmacro %} + +{% macro procurement_month_spine() %} + {% set trailing_edge = trailing_edge_month() %} + + select + cast( + dateadd(month, seq4(), dateadd(month, -23, {{ trailing_edge }})) + as date + ) as month_start + from table(generator(rowcount => 24)) +{% endmacro %} diff --git a/transform/models/L1_inlets/erp/stg_invoices.sql b/transform/models/L1_inlets/erp/stg_invoices.sql new file mode 100644 index 000000000..4b333267f --- /dev/null +++ b/transform/models/L1_inlets/erp/stg_invoices.sql @@ -0,0 +1,70 @@ +with purchase_orders as ( + + select + po_id, + vendor_id, + po_date, + po_amount + from {{ ref('stg_purchase_orders') }} + +), + +vendors as ( + + select + vendor_id, + vendor_seq, + activity_bucket, + invoice_po_through_month, + invoice_lag_months + from {{ ref('stg_vendors') }} + +), + +candidate_invoices as ( + + select + purchase_orders.po_id, + purchase_orders.vendor_id, + purchase_orders.po_amount, + vendors.vendor_seq, + cast(dateadd(month, vendors.invoice_lag_months, purchase_orders.po_date) as date) as invoice_date + from purchase_orders + inner join vendors + on purchase_orders.vendor_id = vendors.vendor_id + where purchase_orders.po_date <= vendors.invoice_po_through_month + and ( + vendors.activity_bucket <> 'C' + or purchase_orders.po_date = vendors.invoice_po_through_month + ) + +), + +invoices_in_span as ( + + select + po_id, + vendor_id, + po_amount, + vendor_seq, + invoice_date + from candidate_invoices + where invoice_date <= {{ trailing_edge_month() }} + +) + +select + replace(po_id, '-PO-', '-INV-')::varchar as invoice_id, + po_id, + vendor_id, + invoice_date, + po_amount::bigint as invoice_amount, + case + when mod(vendor_seq + month(invoice_date), 5) = 0 then null + else dateadd(day, 15, invoice_date)::date + end as paid_date, + case + when mod(vendor_seq + month(invoice_date), 5) = 0 then 'open' + else 'paid' + end::varchar as invoice_status +from invoices_in_span diff --git a/transform/models/L1_inlets/erp/stg_invoices.yml b/transform/models/L1_inlets/erp/stg_invoices.yml new file mode 100644 index 000000000..d9a420eb9 --- /dev/null +++ b/transform/models/L1_inlets/erp/stg_invoices.yml @@ -0,0 +1,46 @@ +version: 2 + +models: + - name: stg_invoices + description: Deterministic ERP invoices, including delayed and missing invoice scenarios. + columns: + - name: invoice_id + description: Invoice primary key. + data_tests: [not_null, unique] + - name: po_id + description: Billed purchase order foreign key. + data_tests: + - not_null + - unique + - relationships: + arguments: + to: ref('stg_purchase_orders') + field: po_id + - name: vendor_id + description: Vendor foreign key. + data_tests: + - not_null + - relationships: + arguments: + to: ref('stg_vendors') + field: vendor_id + - name: invoice_date + description: Invoice activity date. + data_tests: [not_null] + - name: invoice_amount + description: Positive whole-currency invoice amount equal to the PO amount. + data_tests: + - not_null + - dbt_utils.expression_is_true: + arguments: + expression: '> 0' + - name: paid_date + description: Payment date; null while the invoice remains open. + + - name: invoice_status + description: Payment state for the invoice. + data_tests: + - not_null + - accepted_values: + arguments: + values: ['paid', 'open'] diff --git a/transform/models/L1_inlets/erp/stg_purchase_orders.sql b/transform/models/L1_inlets/erp/stg_purchase_orders.sql new file mode 100644 index 000000000..843ec749d --- /dev/null +++ b/transform/models/L1_inlets/erp/stg_purchase_orders.sql @@ -0,0 +1,41 @@ +with month_spine as ( + + {{ procurement_month_spine() }} + +), + +vendors as ( + + select + vendor_id, + category, + onboarded_date, + po_active_through_month, + vendor_seq + from {{ ref('stg_vendors') }} + +) + +select + concat(vendors.vendor_id, '-PO-', to_char(month_spine.month_start, 'YYYYMM'))::varchar as po_id, + vendors.vendor_id, + month_spine.month_start::date as po_date, + vendors.category, + ( + case vendors.category + when 'CRO Services' then 24000 + when 'Clinical Reagents' then 18000 + when 'IT / Software' then 12000 + when 'Logistics' then 9000 + when 'Lab Supplies' then 15000 + when 'Packaging' then 7000 + when 'Consulting' then 16000 + when 'Facilities' then 11000 + end + + mod(vendors.vendor_seq, 4) * 125 + )::bigint as po_amount +from vendors +inner join month_spine + on month_spine.month_start >= vendors.onboarded_date + and month_spine.month_start <= vendors.po_active_through_month +where mod(datediff(month, vendors.onboarded_date, month_spine.month_start), 3) = 0 diff --git a/transform/models/L1_inlets/erp/stg_purchase_orders.yml b/transform/models/L1_inlets/erp/stg_purchase_orders.yml new file mode 100644 index 000000000..8062428e6 --- /dev/null +++ b/transform/models/L1_inlets/erp/stg_purchase_orders.yml @@ -0,0 +1,30 @@ +version: 2 + +models: + - name: stg_purchase_orders + description: Deterministic quarterly ERP purchase orders at monthly date grain. + columns: + - name: po_id + description: Purchase order primary key. + data_tests: [not_null, unique] + - name: vendor_id + description: Vendor foreign key. + data_tests: + - not_null + - relationships: + arguments: + to: ref('stg_vendors') + field: vendor_id + - name: po_date + description: First day of the purchase order month. + data_tests: [not_null] + - name: category + description: Vendor category carried onto the purchase order. + data_tests: [not_null] + - name: po_amount + description: Positive committed spend in whole currency units. + data_tests: + - not_null + - dbt_utils.expression_is_true: + arguments: + expression: '> 0' diff --git a/transform/models/L1_inlets/erp/stg_vendors.sql b/transform/models/L1_inlets/erp/stg_vendors.sql new file mode 100644 index 000000000..c45bfb9f5 --- /dev/null +++ b/transform/models/L1_inlets/erp/stg_vendors.sql @@ -0,0 +1,106 @@ +with month_spine as ( + + {{ procurement_month_spine() }} + +), + +authored_vendors as ( + + select + vendor_id::varchar as vendor_id, + vendor_name::varchar as vendor_name, + category::varchar as category, + status::varchar as status, + onboarded_date::date as onboarded_date + from {{ ref('seed_vendors') }} + +), + +archetypes as ( + + select + archetype_index::integer as archetype_index, + vendor_name::varchar as vendor_name, + category::varchar as category, + status::varchar as status + from {{ ref('seed_vendor_archetypes') }} + +), + +generated_months as ( + + select + month_start, + row_number() over (order by month_start) as generated_vendor_number + from month_spine + where month_start > to_date('{{ var("authored_history_through") }}') + +), + +generated_vendors as ( + + select + concat('VENT', to_char(generated_months.month_start, 'YYYYMM'))::varchar as vendor_id, + concat(archetypes.vendor_name, ' ', to_char(generated_months.month_start, 'YYYYMM'))::varchar as vendor_name, + archetypes.category, + archetypes.status, + generated_months.month_start::date as onboarded_date + from generated_months + join archetypes + on archetypes.archetype_index = mod( + generated_months.generated_vendor_number - 1, + (select count(*) from archetypes) + ) + +), + +all_vendors as ( + + select * from authored_vendors + union all + select * from generated_vendors + +), + +bucketed as ( + + select + vendor_id, + vendor_name, + category, + status, + onboarded_date, + row_number() over (order by vendor_id) as vendor_seq, + case + when vendor_id like 'VENT%' then 'A' + when to_number(replace(vendor_id, 'VEN', '')) <= 6 then 'A' + when to_number(replace(vendor_id, 'VEN', '')) <= 11 then 'B' + when to_number(replace(vendor_id, 'VEN', '')) <= 17 then 'C' + else 'D' + end::varchar as activity_bucket + from all_vendors + +) + +select + vendor_id, + vendor_name, + category, + status, + onboarded_date, + vendor_seq, + activity_bucket, + case + when activity_bucket in ('A', 'B') then {{ trailing_edge_month() }} + else dateadd(month, -15, {{ trailing_edge_month() }}) + end::date as po_active_through_month, + case + when activity_bucket = 'B' then dateadd(month, -15, {{ trailing_edge_month() }}) + when activity_bucket in ('A', 'B') then {{ trailing_edge_month() }} + else dateadd(month, -15, {{ trailing_edge_month() }}) + end::date as invoice_po_through_month, + case + when activity_bucket = 'C' then 14 + else 1 + end::integer as invoice_lag_months +from bucketed diff --git a/transform/models/L1_inlets/erp/stg_vendors.yml b/transform/models/L1_inlets/erp/stg_vendors.yml new file mode 100644 index 000000000..b54fd4493 --- /dev/null +++ b/transform/models/L1_inlets/erp/stg_vendors.yml @@ -0,0 +1,44 @@ +version: 2 + +models: + - name: stg_vendors + description: ERP vendors with deterministic activity-generation drivers. + columns: + - name: vendor_id + description: Vendor primary key. + data_tests: [not_null, unique] + - name: vendor_name + description: Vendor display name. + data_tests: [not_null] + - name: category + description: Vendor spend category. + data_tests: [not_null] + - name: status + description: Recorded lifecycle flag, independent from behavioral activity. + data_tests: + - not_null + - accepted_values: + arguments: + values: ['active', 'inactive'] + - name: onboarded_date + description: First day of the vendor onboarding month. + data_tests: [not_null] + - name: activity_bucket + description: Activity profile controlling PO and invoice recency. + data_tests: + - not_null + - accepted_values: + arguments: + values: ['A', 'B', 'C', 'D'] + - name: vendor_seq + description: Stable sequence used to assign deterministic activity profiles. + data_tests: [not_null] + - name: po_active_through_month + description: Final month in which the vendor can generate purchase orders. + data_tests: [not_null] + - name: invoice_po_through_month + description: Final purchase order month eligible to generate an invoice. + data_tests: [not_null] + - name: invoice_lag_months + description: Whole-month lag between an eligible purchase order and its invoice. + data_tests: [not_null] diff --git a/transform/models/L2_bays/invoices/fct_invoices.sql b/transform/models/L2_bays/invoices/fct_invoices.sql new file mode 100644 index 000000000..ff29ae7e3 --- /dev/null +++ b/transform/models/L2_bays/invoices/fct_invoices.sql @@ -0,0 +1,9 @@ +select + invoice_id, + po_id, + vendor_id, + invoice_date, + invoice_amount, + paid_date, + invoice_status +from {{ ref('stg_invoices') }} diff --git a/transform/models/L2_bays/invoices/fct_invoices.yml b/transform/models/L2_bays/invoices/fct_invoices.yml new file mode 100644 index 000000000..715975ad1 --- /dev/null +++ b/transform/models/L2_bays/invoices/fct_invoices.yml @@ -0,0 +1,45 @@ +version: 2 + +models: + - name: fct_invoices + description: Reusable ERP invoice entity at one row per invoice. + columns: + - name: invoice_id + description: Invoice primary key. + data_tests: [not_null, unique] + - name: po_id + description: Billed purchase order foreign key. + data_tests: + - not_null + - unique + - relationships: + arguments: + to: ref('fct_purchase_orders') + field: po_id + - name: vendor_id + description: Vendor foreign key. + data_tests: + - not_null + - relationships: + arguments: + to: ref('dim_vendors') + field: vendor_id + - name: invoice_date + description: Invoice activity date. + data_tests: [not_null] + - name: invoice_amount + description: Positive whole-currency invoice amount equal to the PO amount. + data_tests: + - not_null + - dbt_utils.expression_is_true: + arguments: + expression: '> 0' + - name: paid_date + description: Null while the invoice is open. + - name: invoice_status + description: Payment state for the invoice. + data_tests: + - not_null + - accepted_values: + arguments: + values: ['paid', 'open'] diff --git a/transform/models/L2_bays/purchase_orders/fct_purchase_orders.sql b/transform/models/L2_bays/purchase_orders/fct_purchase_orders.sql new file mode 100644 index 000000000..f88f735ab --- /dev/null +++ b/transform/models/L2_bays/purchase_orders/fct_purchase_orders.sql @@ -0,0 +1,7 @@ +select + po_id, + vendor_id, + po_date, + category, + po_amount +from {{ ref('stg_purchase_orders') }} diff --git a/transform/models/L2_bays/purchase_orders/fct_purchase_orders.yml b/transform/models/L2_bays/purchase_orders/fct_purchase_orders.yml new file mode 100644 index 000000000..3593fb1ef --- /dev/null +++ b/transform/models/L2_bays/purchase_orders/fct_purchase_orders.yml @@ -0,0 +1,30 @@ +version: 2 + +models: + - name: fct_purchase_orders + description: Reusable ERP purchase order entity at one row per purchase order. + columns: + - name: po_id + description: Purchase order primary key. + data_tests: [not_null, unique] + - name: vendor_id + description: Vendor foreign key. + data_tests: + - not_null + - relationships: + arguments: + to: ref('dim_vendors') + field: vendor_id + - name: po_date + description: First day of the purchase order month. + data_tests: [not_null] + - name: category + description: Vendor category carried onto the purchase order. + data_tests: [not_null] + - name: po_amount + description: Positive committed spend in whole currency units. + data_tests: + - not_null + - dbt_utils.expression_is_true: + arguments: + expression: '> 0' diff --git a/transform/models/L2_bays/vendors/dim_vendors.sql b/transform/models/L2_bays/vendors/dim_vendors.sql new file mode 100644 index 000000000..e7470dc9c --- /dev/null +++ b/transform/models/L2_bays/vendors/dim_vendors.sql @@ -0,0 +1,7 @@ +select + vendor_id, + vendor_name, + category, + status, + onboarded_date +from {{ ref('stg_vendors') }} diff --git a/transform/models/L2_bays/vendors/dim_vendors.yml b/transform/models/L2_bays/vendors/dim_vendors.yml new file mode 100644 index 000000000..27403bb4c --- /dev/null +++ b/transform/models/L2_bays/vendors/dim_vendors.yml @@ -0,0 +1,25 @@ +version: 2 + +models: + - name: dim_vendors + description: Reusable ERP vendor entity at one row per vendor. + columns: + - name: vendor_id + description: Vendor primary key and fact join target. + data_tests: [not_null, unique] + - name: vendor_name + description: Vendor display name. + data_tests: [not_null] + - name: category + description: Vendor spend category. + data_tests: [not_null] + - name: status + description: Recorded lifecycle flag, not a behavioral activity measure. + data_tests: + - not_null + - accepted_values: + arguments: + values: ['active', 'inactive'] + - name: onboarded_date + description: First day of the vendor onboarding month. + data_tests: [not_null] diff --git a/transform/models/L3_coves/accounts_payable/mart_active_vendors.sql b/transform/models/L3_coves/accounts_payable/mart_active_vendors.sql new file mode 100644 index 000000000..eed007312 --- /dev/null +++ b/transform/models/L3_coves/accounts_payable/mart_active_vendors.sql @@ -0,0 +1,70 @@ +{{ config(materialized='table') }} + +with purchase_orders as ( + + select + vendor_id, + po_date as activity_date, + 'purchase_order' as activity_source + from {{ ref('fct_purchase_orders') }} + +), + +invoices as ( + + select + vendor_id, + invoice_date as activity_date, + 'invoice' as activity_source + from {{ ref('fct_invoices') }} + +), + +activity as ( + + select * from purchase_orders + union all + select * from invoices + +), + +bounds as ( + + select max(activity_date) as reference_date + from activity + +) + +select + vendors.vendor_id, + vendors.vendor_name, + vendors.category, + vendors.status as recorded_status, + bounds.reference_date, + max(case when activity.activity_source = 'purchase_order' then activity.activity_date end) + as last_po_date, + max(case when activity.activity_source = 'invoice' then activity.activity_date end) + as last_invoice_date, + count_if( + activity.activity_source = 'purchase_order' + and activity.activity_date > dateadd(month, -12, bounds.reference_date) + and activity.activity_date <= bounds.reference_date + ) as trailing_12_month_po_count, + count_if( + activity.activity_source = 'invoice' + and activity.activity_date > dateadd(month, -12, bounds.reference_date) + and activity.activity_date <= bounds.reference_date + ) as trailing_12_month_invoice_count, + ( + trailing_12_month_po_count + trailing_12_month_invoice_count > 0 + ) as is_active +from {{ ref('dim_vendors') }} as vendors +cross join bounds +left join activity + on vendors.vendor_id = activity.vendor_id +group by + vendors.vendor_id, + vendors.vendor_name, + vendors.category, + vendors.status, + bounds.reference_date diff --git a/transform/models/L3_coves/accounts_payable/mart_active_vendors.yml b/transform/models/L3_coves/accounts_payable/mart_active_vendors.yml new file mode 100644 index 000000000..c26b73e1b --- /dev/null +++ b/transform/models/L3_coves/accounts_payable/mart_active_vendors.yml @@ -0,0 +1,38 @@ +version: 2 + +models: + - name: mart_active_vendors + description: Accounts Payable cove for behavioral Active Vendor analysis using trailing-12-month PO-or-invoice activity. + columns: + - name: vendor_id + description: Vendor primary key. + data_tests: [not_null, unique] + - name: vendor_name + description: Vendor display name. + data_tests: [not_null] + - name: category + description: Vendor spend category. + data_tests: [not_null] + - name: recorded_status + description: Stored ERP lifecycle flag, independent from behavioral activity. + data_tests: + - not_null + - accepted_values: + arguments: + values: ['active', 'inactive'] + - name: reference_date + description: Latest PO or invoice date in the fixture. + data_tests: [not_null] + - name: last_po_date + description: Latest purchase order date for the vendor. + - name: last_invoice_date + description: Latest invoice date for the vendor. + - name: trailing_12_month_po_count + description: Count of purchase orders in the behavioral activity window. + data_tests: [not_null] + - name: trailing_12_month_invoice_count + description: Count of invoices in the behavioral activity window. + data_tests: [not_null] + - name: is_active + description: True when the vendor has at least one PO or invoice in the trailing 12 months. + data_tests: [not_null] diff --git a/transform/seeds/seed_vendor_archetypes.csv b/transform/seeds/seed_vendor_archetypes.csv new file mode 100644 index 000000000..8356fd853 --- /dev/null +++ b/transform/seeds/seed_vendor_archetypes.csv @@ -0,0 +1,5 @@ +archetype_index,vendor_name,category,status +0,Northstar Bioanalytics,CRO Services,active +1,Armitage Cloud Systems,IT / Software,active +2,Bellwether Packaging Co,Packaging,inactive +3,Kestrel Facilities Group,Facilities,active diff --git a/transform/seeds/seed_vendors.csv b/transform/seeds/seed_vendors.csv new file mode 100644 index 000000000..a6afb27b3 --- /dev/null +++ b/transform/seeds/seed_vendors.csv @@ -0,0 +1,27 @@ +vendor_id,vendor_name,category,status,onboarded_date +VEN01,Acadia Clinical Research,CRO Services,inactive,2024-09-01 +VEN02,Bluewater Trial Logistics,Logistics,active,2024-09-01 +VEN03,Caldera Lab Supplies,Lab Supplies,active,2024-09-01 +VEN04,Delphi Data Systems,IT / Software,active,2024-09-01 +VEN05,Evergreen Consulting Partners,Consulting,active,2024-09-01 +VEN06,Fairhaven Packaging,Packaging,active,2024-09-01 +VEN07,Granite Clinical Reagents,Clinical Reagents,inactive,2024-09-01 +VEN08,Harbor Point Facilities,Facilities,active,2024-09-01 +VEN09,Ironwood Logistics Group,Logistics,active,2024-09-01 +VEN10,Juniper Software Labs,IT / Software,active,2024-09-01 +VEN11,Keystone Research Services,CRO Services,active,2024-09-01 +VEN12,Lakeshore Reagents Co,Clinical Reagents,inactive,2024-11-01 +VEN13,Meridian Lab Equipment,Lab Supplies,active,2024-11-01 +VEN14,Norwood Packaging Works,Packaging,active,2024-11-01 +VEN15,Oakridge Advisory Group,Consulting,active,2024-11-01 +VEN16,Parkside Facilities Mgmt,Facilities,active,2024-11-01 +VEN17,Quarry Clinical Research,CRO Services,active,2024-11-01 +VEN18,Riverside Logistics Network,Logistics,active,2024-09-01 +VEN19,Summit Software Solutions,IT / Software,inactive,2024-09-01 +VEN20,Tidewater Lab Supplies,Lab Supplies,inactive,2024-09-01 +VEN21,Union Packaging Partners,Packaging,inactive,2024-09-01 +VEN22,Valley Consulting Group,Consulting,inactive,2024-09-01 +VEN23,Westgate Facilities Services,Facilities,inactive,2024-09-01 +VEN24,Xenon Clinical Reagents,Clinical Reagents,inactive,2024-09-01 +VEN25,Yorktown Research Partners,CRO Services,inactive,2024-09-01 +VEN26,Zephyr Logistics Services,Logistics,inactive,2024-09-01 diff --git a/transform/seeds/seed_vendors.yml b/transform/seeds/seed_vendors.yml new file mode 100644 index 000000000..80ed3f3f1 --- /dev/null +++ b/transform/seeds/seed_vendors.yml @@ -0,0 +1,58 @@ +version: 2 + +seeds: + - name: seed_vendors + description: Curated procurement vendors used to demonstrate behavioral activity. + config: + column_types: + vendor_id: varchar + vendor_name: varchar + category: varchar + status: varchar + onboarded_date: date + columns: + - name: vendor_id + description: Vendor primary key. + data_tests: [not_null, unique] + - name: vendor_name + description: Vendor display name. + data_tests: [not_null] + - name: category + description: Vendor spend category. + data_tests: [not_null] + - name: status + description: Recorded lifecycle flag, independent from behavioral activity. + data_tests: + - not_null + - accepted_values: + arguments: + values: ['active', 'inactive'] + - name: onboarded_date + description: First day of the vendor onboarding month. + data_tests: [not_null] + + - name: seed_vendor_archetypes + description: Dateless profiles used for generated monthly vendor onboardings. + config: + column_types: + archetype_index: integer + vendor_name: varchar + category: varchar + status: varchar + columns: + - name: archetype_index + description: Zero-based round-robin archetype position. + data_tests: [not_null, unique] + - name: vendor_name + description: Generated vendor base name. + data_tests: [not_null] + - name: category + description: Generated vendor spend category. + data_tests: [not_null] + - name: status + description: Recorded lifecycle flag for generated vendors. + data_tests: + - not_null + - accepted_values: + arguments: + values: ['active', 'inactive'] diff --git a/transform/tests/L3_coves/accounts_payable/active_vendor_reference_counts.sql b/transform/tests/L3_coves/accounts_payable/active_vendor_reference_counts.sql new file mode 100644 index 000000000..90e56cfad --- /dev/null +++ b/transform/tests/L3_coves/accounts_payable/active_vendor_reference_counts.sql @@ -0,0 +1,52 @@ +{{ config(tags=['requires_fixture_data']) }} + +with reference_dates as ( + + select to_date('2026-06-01') as reference_date, 32 as expected_total, 17 as expected_active + union all + select to_date('2026-07-01') as reference_date, 33 as expected_total, 24 as expected_active + union all + select to_date('2026-08-01') as reference_date, 34 as expected_total, 25 as expected_active + +), + +activity as ( + + select vendor_id, po_date as activity_date + from {{ ref('fct_purchase_orders') }} + + union all + + select vendor_id, invoice_date as activity_date + from {{ ref('fct_invoices') }} + +), + +actual_counts as ( + + select + reference_dates.reference_date, + count(distinct dim_vendors.vendor_id) as actual_total, + count(distinct activity.vendor_id) as actual_active + from reference_dates + left join {{ ref('dim_vendors') }} as dim_vendors + on dim_vendors.onboarded_date <= reference_dates.reference_date + left join activity + on dim_vendors.vendor_id = activity.vendor_id + and activity.activity_date > dateadd(month, -12, reference_dates.reference_date) + and activity.activity_date <= reference_dates.reference_date + group by reference_dates.reference_date + +) + +select + reference_dates.reference_date, + reference_dates.expected_total, + actual_counts.actual_total, + reference_dates.expected_active, + actual_counts.actual_active +from reference_dates +inner join actual_counts + on reference_dates.reference_date = actual_counts.reference_date +where reference_dates.expected_total <> actual_counts.actual_total + or reference_dates.expected_active <> actual_counts.actual_active