From 43761e59fec3946c0248858ee2d2fe84969484b4 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 17 Sep 2026 19:08:20 +0200 Subject: [PATCH] examples: fail loudly when GCP_PROJECT_ID is unset Four places in examples/bitcoin-tracker silently fell back to a placeholder project id instead of stopping. The placeholder cannot reach a real Google Cloud project, so nothing was writing to a stranger's dataset, but the shape is the one that bit us in docs/walkthrough/gcp.md, where the default was a real-looking id feeding straight into bigquery.Client(project=...). ingest_bitcoin_prices.py was already remediated. The other three scripts now match it: no default, a message naming GCP_PROJECT_ID, and exit 1. - load_bitcoin_price_batch.py: guard in __main__, plus the missing sys import. - runtime/ingest_bitcoin_prices.py: the project id was resolved after the CoinGecko fetch, so the old code did a network round trip before it could notice. The guard now runs first, before any work. - airflow-quickstart.sh: it exported the placeholder into the environment, which would have defeated the parse-time check in the DAG below by setting the variable to garbage rather than leaving it unset. - AIRFLOW_INTEGRATION.md: the documented DAG kept the silent fallback in three spots while the shipped airflow/dags/bitcoin_tracker_enhanced.py already required the variable. The snippet now mirrors the shipped DAG. Verified by running each script under stubbed google.cloud.bigquery and requests that log every network call and client construction. Unset: exit 1, zero logged events. Set: proceeds and targets the given project. The same harness run against the pre-fix code exits 0 and reaches BQ_INSERT against the placeholder, so the check can go red. --- examples/bitcoin-tracker/AIRFLOW_INTEGRATION.md | 10 +++++++--- examples/bitcoin-tracker/airflow-quickstart.sh | 7 ++++++- .../bitcoin-tracker/load_bitcoin_price_batch.py | 13 ++++++++++++- .../runtime/ingest_bitcoin_prices.py | 14 +++++++++++++- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/examples/bitcoin-tracker/AIRFLOW_INTEGRATION.md b/examples/bitcoin-tracker/AIRFLOW_INTEGRATION.md index 082651f..a59702a 100644 --- a/examples/bitcoin-tracker/AIRFLOW_INTEGRATION.md +++ b/examples/bitcoin-tracker/AIRFLOW_INTEGRATION.md @@ -200,6 +200,10 @@ sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + "/../..")) from ingest_bitcoin_prices import fetch_bitcoin_price, insert_to_bigquery +# Configuration from environment variables +GCP_PROJECT_ID = os.environ["GCP_PROJECT_ID"] # required: fail at parse time rather than +# targeting a placeholder project that does not exist + # Default arguments default_args = { "owner": "data-engineering", @@ -256,7 +260,7 @@ with DAG( """Insert price data to BigQuery.""" price_data = context["ti"].xcom_pull(task_ids="fetch_bitcoin_price") - project_id = os.environ.get("GCP_PROJECT_ID", "<>") + project_id = GCP_PROJECT_ID dataset = "crypto_data" table = "bitcoin_prices" @@ -286,7 +290,7 @@ with DAG( task_id="check_data_quality", sql=f""" SELECT COUNT(*) > 0 - FROM `{os.environ.get('GCP_PROJECT_ID', '<>')}.crypto_data.bitcoin_prices` + FROM `{GCP_PROJECT_ID}.crypto_data.bitcoin_prices` WHERE DATE(timestamp) = CURRENT_DATE() """, use_legacy_sql=False, @@ -297,7 +301,7 @@ with DAG( task_id="verify_transformations", sql=f""" SELECT COUNT(*) > 0 - FROM `{os.environ.get('GCP_PROJECT_ID', '<>')}.crypto_data.daily_price_summary` + FROM `{GCP_PROJECT_ID}.crypto_data.daily_price_summary` WHERE price_date >= DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY) """, use_legacy_sql=False, diff --git a/examples/bitcoin-tracker/airflow-quickstart.sh b/examples/bitcoin-tracker/airflow-quickstart.sh index 7ca6788..5eb079e 100755 --- a/examples/bitcoin-tracker/airflow-quickstart.sh +++ b/examples/bitcoin-tracker/airflow-quickstart.sh @@ -18,7 +18,12 @@ NC='\033[0m' # No Color # Configuration PROJECT_DIR=$(pwd) AIRFLOW_HOME=${AIRFLOW_HOME:-$HOME/airflow} -GCP_PROJECT_ID=${GCP_PROJECT_ID:-<>} +GCP_PROJECT_ID=${GCP_PROJECT_ID:-} +if [ -z "$GCP_PROJECT_ID" ]; then + echo "❌ Error: GCP_PROJECT_ID environment variable not set" + echo "Usage: export GCP_PROJECT_ID=your-project-id && ./airflow-quickstart.sh" + exit 1 +fi echo -e "${BLUE}Configuration:${NC}" echo " Project Directory: $PROJECT_DIR" diff --git a/examples/bitcoin-tracker/load_bitcoin_price_batch.py b/examples/bitcoin-tracker/load_bitcoin_price_batch.py index dd45a97..5a24be9 100644 --- a/examples/bitcoin-tracker/load_bitcoin_price_batch.py +++ b/examples/bitcoin-tracker/load_bitcoin_price_batch.py @@ -6,6 +6,7 @@ from google.cloud import bigquery from datetime import datetime import os +import sys import json import tempfile @@ -67,7 +68,17 @@ def load_to_bigquery_batch(row, project_id, dataset_id="crypto_data", table_id=" os.unlink(temp_file) if __name__ == "__main__": - project_id = os.getenv("GCP_PROJECT_ID", "<>") + # Get project ID from environment or command line + project_id = os.getenv("GCP_PROJECT_ID") + + if not project_id and len(sys.argv) > 1: + project_id = sys.argv[1] + + if not project_id: + print("❌ Error: GCP_PROJECT_ID environment variable not set") + print("Usage: python load_bitcoin_price_batch.py [PROJECT_ID]") + print(" or: export GCP_PROJECT_ID=your-project-id && python load_bitcoin_price_batch.py") + sys.exit(1) print(f"🚀 Fetching Bitcoin price...") price_data = fetch_bitcoin_price() diff --git a/examples/bitcoin-tracker/runtime/ingest_bitcoin_prices.py b/examples/bitcoin-tracker/runtime/ingest_bitcoin_prices.py index 8107182..8265138 100644 --- a/examples/bitcoin-tracker/runtime/ingest_bitcoin_prices.py +++ b/examples/bitcoin-tracker/runtime/ingest_bitcoin_prices.py @@ -12,6 +12,19 @@ def main(): """Fetch Bitcoin price from CoinGecko API and insert to BigQuery""" + # Resolve the target project before doing any work, so a missing + # GCP_PROJECT_ID fails here rather than silently targeting a placeholder. + project_id = os.getenv("GCP_PROJECT_ID") + + if not project_id and len(sys.argv) > 1: + project_id = sys.argv[1] + + if not project_id: + print("❌ Error: GCP_PROJECT_ID environment variable not set") + print("Usage: python ingest_bitcoin_prices.py [PROJECT_ID]") + print(" or: export GCP_PROJECT_ID=your-project-id && python ingest_bitcoin_prices.py") + sys.exit(1) + print("🚀 Starting Bitcoin price ingestion...") # Fetch from CoinGecko API (free tier, no auth required) @@ -47,7 +60,6 @@ def main(): } # Insert to BigQuery - project_id = os.getenv("GCP_PROJECT_ID", "<>") dataset_id = "crypto_data" table_id = "bitcoin_prices"