diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d432a26 --- /dev/null +++ b/.env.example @@ -0,0 +1,6 @@ +# place to write secrets, to be filled by CI pipeline or user's machine +ALCHEMY_API_KEY= +ETHERSCAN_API_KEY= +BLOCKSCOUT_API_KEY= +PRIVATE_KEY= +LIQUIDATOR_PRIVATE_KEY= diff --git a/.github/infra/README.md b/.github/infra/README.md index 8b0b17c..1e81ea0 100644 --- a/.github/infra/README.md +++ b/.github/infra/README.md @@ -22,6 +22,12 @@ defs, separate wallets, separate logs. They only share the image so a single `docker push` rolls both venues forward (each can still be pinned to a different image tag). +The non-secret half of that `env:` block is not configured in Terraform +or in GitHub Variables. `deploy-col-mar-mm.yml` reads `config/dev.env` +or `config/prd.env` — the same files the market-maker loads locally — +and passes every key it finds to the task definition, alongside the +secrets it names explicitly. Add a public setting by editing that file. + ## Files * `ecs-task.tf` — reusable Terraform module template for one MM service. diff --git a/.github/workflows/deploy-col-mar-mm.yml b/.github/workflows/deploy-col-mar-mm.yml index 1e64961..b8698af 100644 --- a/.github/workflows/deploy-col-mar-mm.yml +++ b/.github/workflows/deploy-col-mar-mm.yml @@ -10,23 +10,23 @@ name: Deploy Collateral Margin Market Maker # # 1. Builds one Docker image from market-maker/ and pushes to GHCR # 2. Renders a new task-def revision for the portfolio app (MAKER_APP=portfolio) -# with image + env vars + secrets sourced from GitHub Variables and -# Secrets (no AWS Secrets Manager) +# with image + public config from config/.env + GitHub Secrets +# (no AWS Secrets Manager) # 3. Calls ecs:UpdateService to point the service at the new revision # and to scale it to the operator-chosen desired_count # +# Container config (contract addresses, log level, oracle URL, …) is NOT +# declared here. It comes from config/dev.env and config/prd.env, the same +# files the market-maker loads locally, and every key in the chosen file is +# passed to the task definition. To add or change one, edit that file. +# # GitHub Variables (configure per-environment under Settings -> Environments): -# MAKER_LOG_LEVEL default "info" -# MAKER_DRY_RUN default "false" -# MAKER_CANCEL_ORDERS_ON_SHUTDOWN default "true" -# MAKER_HEALTH_PORT default "3001" (must match TF cnt_port) -# MAKER_DESIRED_COUNT default "1" (set 0 to halt without redeploy) -# PERPS_ADDRESS HashPowerPerpsDEX contract address -# FUTURES_ADDRESS Futures contract address -# ETH_PRICE_FEED_ADDRESS optional Chainlink feed +# MAKER_DESIRED_COUNT default "1" (set 0 to halt without +# redeploy) — deploy orchestration, not +# container config # # GitHub Secrets (configure per-environment): -# AWS_ROLE_ARN_DEV / _STG / _LMN OIDC role ARNs from the TF output github_actions_role_arn +# AWS_ROLE_ARN_DEV / _LMN OIDC role ARNs from the TF output github_actions_role_arn # ALCHEMY_API_KEY shared Alchemy project key (composes the RPC URL) # FUTURES_MM_PRIVATE_KEY shared portfolio signer private key (both venues) # SLACK_WEBHOOK_URL (org or repo level) for slack-notify @@ -35,21 +35,20 @@ on: push: branches: - dev - - stg - main - "cicd/**" paths: - "market-maker/**" + - "config/*.env" - ".github/workflows/deploy-col-mar-mm.yml" workflow_dispatch: inputs: environment: - description: "Target environment (dev=DEV, stg=STG, main=LMN/PROD)" + description: "Target environment (dev=DEV, main=LMN/PROD)" required: true type: choice options: - dev - - stg - main permissions: @@ -105,25 +104,23 @@ jobs: # URL_HOST_PREFIX matches the per-env Route53 alias produced by the # collateral-margin TF stack (futures_mm_endpoint in 00_outputs.tf): - # dev/stg get a subdomain, prod uses the apex. + # dev gets a subdomain, prod uses the apex. + # + # MAKER_ENV also selects both the YAML profile baked into the image + # and the config/.env read at deploy time. case "$ENV" in dev) SUFFIX="dev" MAKER_ENV="dev" URL_HOST_PREFIX="dev." ;; - stg) - SUFFIX="stg" - MAKER_ENV="stg" - URL_HOST_PREFIX="stg." - ;; main) SUFFIX="lmn" MAKER_ENV="prd" URL_HOST_PREFIX="" ;; *) - echo "::error::Unknown environment '$ENV'" + echo "::error::Unknown environment '$ENV' (expected dev or main)" exit 1 ;; esac @@ -192,12 +189,17 @@ jobs: environment: ${{ needs.build.outputs.environment }} steps: + # Needed for config/.env, which supplies the container environment. + - name: Checkout code + uses: actions/checkout@v5 + with: + fetch-depth: 1 + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ needs.build.outputs.environment == 'dev' && - secrets.AWS_ROLE_ARN_DEV || needs.build.outputs.environment == 'stg' - && secrets.AWS_ROLE_ARN_STG || secrets.AWS_ROLE_ARN_LMN }} + secrets.AWS_ROLE_ARN_DEV || secrets.AWS_ROLE_ARN_LMN }} aws-region: ${{ needs.build.outputs.aws_region }} role-session-name: GitHubActions-ColMarMM-portfolio-${{ github.run_id }} @@ -234,18 +236,8 @@ jobs: REGION: ${{ needs.build.outputs.aws_region }} IMAGE: ${{ env.GHCR_IMAGE }}:${{ needs.build.outputs.version }} - # Per-environment GitHub Variables (non-secret personality) - MAKER_LOG_LEVEL: ${{ vars.MAKER_LOG_LEVEL }} - MAKER_DRY_RUN: ${{ vars.MAKER_DRY_RUN }} - MAKER_CANCEL_ORDERS_ON_SHUTDOWN: ${{ vars.MAKER_CANCEL_ORDERS_ON_SHUTDOWN }} - MAKER_HEALTH_PORT: ${{ vars.MAKER_HEALTH_PORT }} + # Deploy orchestration (not container config) MAKER_DESIRED_COUNT: ${{ vars.MAKER_DESIRED_COUNT }} - ETH_PRICE_FEED_ADDRESS: ${{ vars.ETH_PRICE_FEED_ADDRESS }} - HASHPRICE_ORACLE_SUBGRAPH_URL: ${{ vars.HASHPRICE_ORACLE_SUBGRAPH_URL }} - - # Both venue addresses — the portfolio app quotes perps + futures. - PERPS_ADDRESS: ${{ vars.PERPS_ADDRESS }} - FUTURES_ADDRESS: ${{ vars.FUTURES_ADDRESS }} # Per-environment GitHub Secrets (encrypted personality). Secrets # only flow into this step's process env; they are never written to @@ -260,6 +252,7 @@ jobs: COMMIT_HASH: ${{ github.sha }} run: | set -euo pipefail + CONFIG="config/${MAKER_ENV}.env" DESIRED_COUNT="${MAKER_DESIRED_COUNT:-1}" echo "🚀 Deploying portfolio to ${{ needs.build.outputs.environment }}" @@ -268,6 +261,7 @@ jobs: echo " Task Family: ${TASK_FAMILY}" echo " Image: ${IMAGE}" echo " MAKER_ENV: ${MAKER_ENV}" + echo " Config: ${CONFIG}" echo " Desired Count: ${DESIRED_COUNT}" # Fetch current task def, strip non-registerable metadata, swap image @@ -286,36 +280,30 @@ jobs: # Build the full env block from scratch. Personality lives ONLY here; # the Terraform stub deliberately ships zero env vars. - echo "🔧 Injecting environment from GitHub Variables / Secrets..." - jq -n \ - --arg MAKER_APP "${MAKER_APP}" \ - --arg MAKER_ENV "${MAKER_ENV}" \ - --arg MAKER_HEALTH_PORT "${MAKER_HEALTH_PORT:-3001}" \ - --arg MAKER_LOG_LEVEL "${MAKER_LOG_LEVEL:-info}" \ - --arg MAKER_DRY_RUN "${MAKER_DRY_RUN:-false}" \ - --arg MAKER_CANCEL_ORDERS_ON_SHUTDOWN "${MAKER_CANCEL_ORDERS_ON_SHUTDOWN:-true}" \ - --arg COMMIT_HASH "${COMMIT_HASH}" \ - --arg ETH_PRICE_FEED_ADDRESS "${ETH_PRICE_FEED_ADDRESS:-}" \ - --arg ALCHEMY_API_KEY "${ALCHEMY_API_KEY}" \ - --arg PRIVATE_KEY "${PRIVATE_KEY}" \ - --arg PERPS_ADDRESS "${PERPS_ADDRESS}" \ - --arg FUTURES_ADDRESS "${FUTURES_ADDRESS}" \ - --arg HASHPRICE_ORACLE_SUBGRAPH_URL "${HASHPRICE_ORACLE_SUBGRAPH_URL}" \ - '[ - {name:"MAKER_APP", value:$MAKER_APP}, - {name:"MAKER_ENV", value:$MAKER_ENV}, - {name:"MAKER_HEALTH_PORT", value:$MAKER_HEALTH_PORT}, - {name:"MAKER_LOG_LEVEL", value:$MAKER_LOG_LEVEL}, - {name:"MAKER_DRY_RUN", value:$MAKER_DRY_RUN}, - {name:"MAKER_CANCEL_ORDERS_ON_SHUTDOWN", value:$MAKER_CANCEL_ORDERS_ON_SHUTDOWN}, - {name:"COMMIT_HASH", value:$COMMIT_HASH}, - {name:"ETH_PRICE_FEED_ADDRESS", value:$ETH_PRICE_FEED_ADDRESS}, - {name:"ALCHEMY_API_KEY", value:$ALCHEMY_API_KEY}, - {name:"PRIVATE_KEY", value:$PRIVATE_KEY}, - {name:"PERPS_ADDRESS", value:$PERPS_ADDRESS}, - {name:"FUTURES_ADDRESS", value:$FUTURES_ADDRESS}, - {name:"HASHPRICE_ORACLE_SUBGRAPH_URL", value:$HASHPRICE_ORACLE_SUBGRAPH_URL} - ]' > env-block.json + # + # The block is every key declared in config/.env plus the + # secrets and computed values from the step env above. Sourcing + # cannot clobber a secret, because no secret is named in the config + # file. jq then reads values straight out of the environment, so each + # key is written exactly once: in the config file. + echo "🔧 Injecting environment from ${CONFIG} + GitHub Secrets..." + set -a && . "$CONFIG" && set +a + + KEYS=$( { sed -n 's/^[[:space:]]*\([A-Za-z_][A-Za-z0-9_]*\)=.*/\1/p' "$CONFIG"; \ + printf '%s\n' MAKER_APP MAKER_ENV COMMIT_HASH ALCHEMY_API_KEY PRIVATE_KEY; } ) + + # Empty values are dropped so the maker applies its own defaults + # instead of parsing an empty string. + jq -n --arg keys "$KEYS" '[ + $keys + | split("\n") + | unique + | .[] + | select(. != "" and ($ENV[.] // "") != "") + | {name: ., value: $ENV[.]} + ]' > env-block.json + + echo " Container env keys: $(jq -r '[.[].name] | join(", ")' env-block.json)" # Replace the entire env block; also drop any stale `secrets` block # (we no longer use Secrets Manager). @@ -356,8 +344,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ needs.build.outputs.environment == 'dev' && - secrets.AWS_ROLE_ARN_DEV || needs.build.outputs.environment == 'stg' - && secrets.AWS_ROLE_ARN_STG || secrets.AWS_ROLE_ARN_LMN }} + secrets.AWS_ROLE_ARN_DEV || secrets.AWS_ROLE_ARN_LMN }} aws-region: ${{ needs.build.outputs.aws_region }} role-session-name: GitHubActions-ColMarMM-portfolio-Verify-${{ github.run_id }} diff --git a/.github/workflows/deploy-keeper.yml b/.github/workflows/deploy-keeper.yml index 19ea242..a95311a 100644 --- a/.github/workflows/deploy-keeper.yml +++ b/.github/workflows/deploy-keeper.yml @@ -6,57 +6,44 @@ name: Deploy Collateral Margin Keeper # Terraform (.bedrock/.terragrunt/06_col_mar_keeper_svc.tf) builds ECS shell, # internal ALB, Route53 keeper.{env}.hashpower.exchange, and log group. # This workflow builds the image, pushes to GHCR, and registers task-def -# revisions with full runtime config from GitHub. +# revisions with the runtime config from config/.env plus secrets. # -# GitHub Variables (per environment: dev / stg / main): -# NETWORK e.g. base-sepolia -# VAULT_ADDRESS CollateralVault proxy -# PME_ADDRESS PortfolioMarginEngine proxy -# PERPS_ADDRESS HashPowerPerpsDEX -# FUTURES_ADDRESS Futures proxy -# HASHPRICE_USD_ADDRESS HashpriceUSD aggregator -# BTC_USD_FEED_ADDRESS Chainlink BTC/USDC feed -# ETH_USD_FEED_ADDRESS optional — gas cost logging -# DRY_RUN "true" | "false" (default false) -# KEEPER_MIN_PROFIT_MARGIN default "0" -# KEEPER_DESIRED_COUNT default "1" (set "0" to halt) -# LOG_LEVEL default "info" -# DISCOVERY_MODE events | webhook | both (default events) -# BACKFILL_FROM_BLOCK optional startup discovery backfill anchor -# DELIVERY_KEEPER_ENABLED default "false" -# HEALTH_PORT default "3000" (must match TF cnt_port) -# PRICE_MOVE_TRIGGER_BPS default "1" -# SWEEP_INTERVAL_MS default "60000" -# ALERT_WEBHOOK_URL optional Slack/Discord webhook +# Container config (NETWORK, contract addresses, log level, intervals, …) is +# NOT declared here. It comes from config/dev.env and config/prd.env, which are +# the same files the keeper loads locally, and every key in the chosen file is +# passed to the task definition. To add or change one, edit that file. +# +# GitHub Variables (per environment: dev / main): +# KEEPER_DESIRED_COUNT default "1" (set "0" to halt) — deploy +# orchestration, not container config # # GitHub Secrets (per environment): -# ALCHEMY_API_KEY RPC (or set ETH_NODE_ADDRESS via var — not secret) -# LIQUIDATOR_PRIVATE_KEY same wallet as legacy perps keeper -# WEBHOOK_SECRET optional — Goldsky bearer token +# ALCHEMY_API_KEY RPC (or set ETH_NODE_ADDRESS in config) +# LIQUIDATOR_PRIVATE_KEY same wallet as legacy perps keeper +# WEBHOOK_SECRET optional — Goldsky bearer token # # Repository secrets (all environments): -# AWS_ROLE_ARN_DEV / _STG / _LMN from terragrunt output github_actions_role_arn +# AWS_ROLE_ARN_DEV / _LMN from terragrunt output github_actions_role_arn # SLACK_WEBHOOK_URL optional — deploy notifications on: push: branches: - dev - - stg - main - "cicd/**" paths: - "keeper/**" + - "config/*.env" - ".github/workflows/deploy-keeper.yml" workflow_dispatch: inputs: environment: - description: "Target environment (dev=DEV, stg=STG, main=LMN/PROD)" + description: "Target environment (dev=DEV, main=LMN/PROD)" required: true type: choice options: - dev - - stg - main permissions: @@ -77,6 +64,7 @@ jobs: tag: ${{ steps.gen_tag.outputs.tag_name }} environment: ${{ steps.gen_tag.outputs.environment }} env_suffix: ${{ steps.env_config.outputs.env_suffix }} + config_env: ${{ steps.env_config.outputs.config_env }} aws_region: ${{ steps.env_config.outputs.aws_region }} ecs_cluster: ${{ steps.env_config.outputs.ecs_cluster }} ecs_service: ${{ steps.env_config.outputs.ecs_service }} @@ -106,26 +94,27 @@ jobs: echo "aws_region=us-east-1" >> $GITHUB_OUTPUT echo "task_family=tsk-col-mar-keeper" >> $GITHUB_OUTPUT + # CONFIG_ENV names the config/.env holding this environment's + # public container config; SUFFIX names the AWS resources. case "$ENV" in dev) SUFFIX="dev" URL_HOST_PREFIX="dev." - ;; - stg) - SUFFIX="stg" - URL_HOST_PREFIX="stg." + CONFIG_ENV="dev" ;; main) SUFFIX="lmn" URL_HOST_PREFIX="" + CONFIG_ENV="prd" ;; *) - echo "::error::Unknown environment '$ENV'" + echo "::error::Unknown environment '$ENV' (expected dev or main)" exit 1 ;; esac echo "env_suffix=${SUFFIX}" >> $GITHUB_OUTPUT + echo "config_env=${CONFIG_ENV}" >> $GITHUB_OUTPUT echo "ecs_cluster=ecs-derivatives-marketplace-${SUFFIX}" >> $GITHUB_OUTPUT echo "ecs_service=svc-col-mar-keeper-${SUFFIX}" >> $GITHUB_OUTPUT echo "keeper_health_url=https://keeper.${URL_HOST_PREFIX}hashpower.exchange/health" >> $GITHUB_OUTPUT @@ -188,10 +177,16 @@ jobs: skipped: ${{ steps.svc_check.outputs.skip }} steps: + # Needed for config/.env, which supplies the container environment. + - name: Checkout code + uses: actions/checkout@v5 + with: + fetch-depth: 1 + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v6 with: - role-to-assume: ${{ needs.build.outputs.environment == 'dev' && secrets.AWS_ROLE_ARN_DEV || needs.build.outputs.environment == 'stg' && secrets.AWS_ROLE_ARN_STG || secrets.AWS_ROLE_ARN_LMN }} + role-to-assume: ${{ needs.build.outputs.environment == 'dev' && secrets.AWS_ROLE_ARN_DEV || secrets.AWS_ROLE_ARN_LMN }} aws-region: ${{ needs.build.outputs.aws_region }} role-session-name: GitHubActions-ColMarKeeper-${{ github.run_id }} @@ -227,34 +222,18 @@ jobs: TASK_FAMILY: ${{ needs.build.outputs.task_family }} REGION: ${{ needs.build.outputs.aws_region }} IMAGE: ${{ env.GHCR_IMAGE }}:${{ needs.build.outputs.version }} - KEEPER_VERSION: ${{ needs.build.outputs.version }} + CONFIG_ENV: ${{ needs.build.outputs.config_env }} - NETWORK: ${{ vars.NETWORK }} - VAULT_ADDRESS: ${{ vars.VAULT_ADDRESS }} - PME_ADDRESS: ${{ vars.PME_ADDRESS }} - PERPS_ADDRESS: ${{ vars.PERPS_ADDRESS }} - FUTURES_ADDRESS: ${{ vars.FUTURES_ADDRESS }} - HASHPRICE_USD_ADDRESS: ${{ vars.HASHPRICE_USD_ADDRESS }} - BTC_USD_FEED_ADDRESS: ${{ vars.BTC_USD_FEED_ADDRESS }} - ETH_USD_FEED_ADDRESS: ${{ vars.ETH_USD_FEED_ADDRESS }} - DRY_RUN: ${{ vars.DRY_RUN }} - KEEPER_MIN_PROFIT_MARGIN: ${{ vars.KEEPER_MIN_PROFIT_MARGIN }} + # Everything public lives in config/.env. Only secrets and + # values computed by this run are declared here. + KEEPER_VERSION: ${{ needs.build.outputs.version }} KEEPER_DESIRED_COUNT: ${{ vars.KEEPER_DESIRED_COUNT }} - LOG_LEVEL: ${{ vars.LOG_LEVEL }} - DISCOVERY_MODE: ${{ vars.DISCOVERY_MODE }} - BACKFILL_FROM_BLOCK: ${{ vars.BACKFILL_FROM_BLOCK }} - DELIVERY_KEEPER_ENABLED: ${{ vars.DELIVERY_KEEPER_ENABLED }} - HEALTH_PORT: ${{ vars.HEALTH_PORT }} - PRICE_MOVE_TRIGGER_BPS: ${{ vars.PRICE_MOVE_TRIGGER_BPS }} - SWEEP_INTERVAL_MS: ${{ vars.SWEEP_INTERVAL_MS }} - ALERT_WEBHOOK_URL: ${{ vars.ALERT_WEBHOOK_URL }} - ETH_NODE_ADDRESS: ${{ vars.ETH_NODE_ADDRESS }} - ALCHEMY_API_KEY: ${{ secrets.ALCHEMY_API_KEY }} LIQUIDATOR_PRIVATE_KEY: ${{ secrets.LIQUIDATOR_PRIVATE_KEY }} WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} run: | set -euo pipefail + CONFIG="config/${CONFIG_ENV}.env" DESIRED_COUNT="${KEEPER_DESIRED_COUNT:-1}" echo "🚀 Deploying keeper to ${{ needs.build.outputs.environment }}" @@ -262,7 +241,7 @@ jobs: echo " Service: ${SERVICE}" echo " Task Family: ${TASK_FAMILY}" echo " Image: ${IMAGE}" - echo " DRY_RUN: ${DRY_RUN:-false}" + echo " Config: ${CONFIG}" echo " Desired Count: ${DESIRED_COUNT}" aws ecs describe-task-definition \ @@ -277,58 +256,28 @@ jobs: .compatibilities, .registeredAt, .registeredBy) ' task-def.json > new-task-def.json - jq -n \ - --arg NETWORK "${NETWORK}" \ - --arg VAULT_ADDRESS "${VAULT_ADDRESS}" \ - --arg PME_ADDRESS "${PME_ADDRESS}" \ - --arg PERPS_ADDRESS "${PERPS_ADDRESS}" \ - --arg FUTURES_ADDRESS "${FUTURES_ADDRESS}" \ - --arg HASHPRICE_USD_ADDRESS "${HASHPRICE_USD_ADDRESS}" \ - --arg BTC_USD_FEED_ADDRESS "${BTC_USD_FEED_ADDRESS}" \ - --arg ETH_USD_FEED_ADDRESS "${ETH_USD_FEED_ADDRESS:-}" \ - --arg DRY_RUN "${DRY_RUN:-false}" \ - --arg KEEPER_MIN_PROFIT_MARGIN "${KEEPER_MIN_PROFIT_MARGIN:-0}" \ - --arg LOG_LEVEL "${LOG_LEVEL:-info}" \ - --arg DISCOVERY_MODE "${DISCOVERY_MODE:-events}" \ - --arg BACKFILL_FROM_BLOCK "${BACKFILL_FROM_BLOCK:-}" \ - --arg DELIVERY_KEEPER_ENABLED "${DELIVERY_KEEPER_ENABLED:-false}" \ - --arg HEALTH_PORT "${HEALTH_PORT:-3000}" \ - --arg PRICE_MOVE_TRIGGER_BPS "${PRICE_MOVE_TRIGGER_BPS:-1}" \ - --arg SWEEP_INTERVAL_MS "${SWEEP_INTERVAL_MS:-60000}" \ - --arg ALERT_WEBHOOK_URL "${ALERT_WEBHOOK_URL:-}" \ - --arg ETH_NODE_ADDRESS "${ETH_NODE_ADDRESS:-}" \ - --arg ALCHEMY_API_KEY "${ALCHEMY_API_KEY}" \ - --arg LIQUIDATOR_PRIVATE_KEY "${LIQUIDATOR_PRIVATE_KEY}" \ - --arg WEBHOOK_SECRET "${WEBHOOK_SECRET:-}" \ - --arg KEEPER_VERSION "${KEEPER_VERSION}" \ - '[ - {name:"KEEPER_VERSION", value:$KEEPER_VERSION}, - {name:"NETWORK", value:$NETWORK}, - {name:"VAULT_ADDRESS", value:$VAULT_ADDRESS}, - {name:"PME_ADDRESS", value:$PME_ADDRESS}, - {name:"PERPS_ADDRESS", value:$PERPS_ADDRESS}, - {name:"FUTURES_ADDRESS", value:$FUTURES_ADDRESS}, - {name:"HASHPRICE_USD_ADDRESS", value:$HASHPRICE_USD_ADDRESS}, - {name:"BTC_USD_FEED_ADDRESS", value:$BTC_USD_FEED_ADDRESS}, - {name:"ETH_USD_FEED_ADDRESS", value:$ETH_USD_FEED_ADDRESS}, - {name:"DRY_RUN", value:$DRY_RUN}, - {name:"KEEPER_MIN_PROFIT_MARGIN", value:$KEEPER_MIN_PROFIT_MARGIN}, - {name:"LOG_LEVEL", value:$LOG_LEVEL}, - {name:"DISCOVERY_MODE", value:$DISCOVERY_MODE}, - {name:"DELIVERY_KEEPER_ENABLED", value:$DELIVERY_KEEPER_ENABLED}, - {name:"HEALTH_PORT", value:$HEALTH_PORT}, - {name:"PRICE_MOVE_TRIGGER_BPS", value:$PRICE_MOVE_TRIGGER_BPS}, - {name:"SWEEP_INTERVAL_MS", value:$SWEEP_INTERVAL_MS}, - {name:"ALERT_WEBHOOK_URL", value:$ALERT_WEBHOOK_URL}, - {name:"ETH_NODE_ADDRESS", value:$ETH_NODE_ADDRESS}, - {name:"ALCHEMY_API_KEY", value:$ALCHEMY_API_KEY}, - {name:"LIQUIDATOR_PRIVATE_KEY", value:$LIQUIDATOR_PRIVATE_KEY}, - {name:"WEBHOOK_SECRET", value:$WEBHOOK_SECRET} - ] + ( - if $BACKFILL_FROM_BLOCK == "" then [] - else [{name:"BACKFILL_FROM_BLOCK", value:$BACKFILL_FROM_BLOCK}] - end - )' > env-block.json + # The container environment is every key declared in config/.env + # plus the secrets and computed values from the step env above. + # Sourcing cannot clobber a secret, because no secret is named in the + # config file. jq then reads values straight out of the environment, + # so each key is written exactly once: in the config file. + set -a && . "$CONFIG" && set +a + + KEYS=$( { sed -n 's/^[[:space:]]*\([A-Za-z_][A-Za-z0-9_]*\)=.*/\1/p' "$CONFIG"; \ + printf '%s\n' KEEPER_VERSION ALCHEMY_API_KEY LIQUIDATOR_PRIVATE_KEY WEBHOOK_SECRET; } ) + + # Empty values are dropped so the keeper applies its own defaults + # instead of parsing an empty string. + jq -n --arg keys "$KEYS" '[ + $keys + | split("\n") + | unique + | .[] + | select(. != "" and ($ENV[.] // "") != "") + | {name: ., value: $ENV[.]} + ]' > env-block.json + + echo " Container env keys: $(jq -r '[.[].name] | join(", ")' env-block.json)" jq --slurpfile env env-block.json ' .containerDefinitions[0].environment = $env[0] | @@ -368,7 +317,7 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v6 with: - role-to-assume: ${{ needs.build.outputs.environment == 'dev' && secrets.AWS_ROLE_ARN_DEV || needs.build.outputs.environment == 'stg' && secrets.AWS_ROLE_ARN_STG || secrets.AWS_ROLE_ARN_LMN }} + role-to-assume: ${{ needs.build.outputs.environment == 'dev' && secrets.AWS_ROLE_ARN_DEV || secrets.AWS_ROLE_ARN_LMN }} aws-region: ${{ needs.build.outputs.aws_region }} role-session-name: GitHubActions-ColMarKeeper-Verify-${{ github.run_id }} diff --git a/.github/workflows/deploy-points-subgraph.yml b/.github/workflows/deploy-points-subgraph.yml index f95dd8d..6c5d9a1 100644 --- a/.github/workflows/deploy-points-subgraph.yml +++ b/.github/workflows/deploy-points-subgraph.yml @@ -4,33 +4,32 @@ on: push: branches: - dev - - stg - main - "cicd/**" paths: - "points-indexer/**" + - "config/*.env" - "contracts/abi/Points.json" - "contracts/abi/PointsRedeemer.json" - ".github/workflows/deploy-points-subgraph.yml" pull_request: branches: - dev - - stg - main paths: - "points-indexer/**" + - "config/*.env" - "contracts/abi/Points.json" - "contracts/abi/PointsRedeemer.json" - ".github/workflows/deploy-points-subgraph.yml" workflow_dispatch: inputs: environment: - description: "Target environment (dev=DEV, stg=STG, main=LMN/PROD)" + description: "Target environment (dev=DEV, main=LMN/PROD)" required: true type: choice options: - dev - - stg - main concurrency: @@ -116,19 +115,21 @@ jobs: fi echo "goldsky_subgraph_name=$GS_NAME" >> $GITHUB_OUTPUT - # Rolling tag and public endpoint per environment + # Rolling tag, public endpoint, and the config/.env to load. case $ENV in dev) echo "goldsky_rolling_tag=dev-latest" >> $GITHUB_OUTPUT echo "goldsky_endpoint=${{ vars.DEV_GS_POINTS }}" >> $GITHUB_OUTPUT - ;; - stg) - echo "goldsky_rolling_tag=stg-latest" >> $GITHUB_OUTPUT - echo "goldsky_endpoint=${{ vars.STG_GS_POINTS }}" >> $GITHUB_OUTPUT + echo "config_env=dev" >> $GITHUB_OUTPUT ;; main) echo "goldsky_rolling_tag=lmn-latest" >> $GITHUB_OUTPUT echo "goldsky_endpoint=${{ vars.LMN_GS_POINTS }}" >> $GITHUB_OUTPUT + echo "config_env=prd" >> $GITHUB_OUTPUT + ;; + *) + echo "::error::Unknown environment '$ENV' (expected dev or main)" + exit 1 ;; esac @@ -138,24 +139,24 @@ jobs: working-directory: ./points-indexer run: pnpm install --frozen-lockfile + # Addresses, start blocks and NETWORK all come from config/.env. + # PR builds render from the dummy values in .env.example, since real + # addresses are only needed at deploy time. - name: Prepare subgraph configuration working-directory: ./points-indexer env: - NETWORK: ${{ vars.NETWORK }} - POINTS_ADDRESS: ${{ vars.POINTS_ADDRESS }} - POINTS_START_BLOCK: ${{ vars.POINTS_START_BLOCK }} - REDEEMER_ADDRESS: ${{ vars.REDEEMER_ADDRESS }} - REDEEMER_START_BLOCK: ${{ vars.REDEEMER_START_BLOCK }} + CONFIG_ENV: ${{ steps.env.outputs.config_env }} + DEPLOY_ENV: ${{ needs.setup.outputs.environment }} run: | + set -euo pipefail if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then echo "ℹ️ PR build — using .env.example dummy values" - set -a && source .env.example && set +a + ENV_FILE=.env.example + else + ENV_FILE="../config/${CONFIG_ENV}.env" fi - echo "⚙️ Preparing subgraph for ${{ needs.setup.outputs.environment }}..." - echo " Network: ${NETWORK}" - echo " Points: ${POINTS_ADDRESS} (block ${POINTS_START_BLOCK})" - echo " Redeemer: ${REDEEMER_ADDRESS} (block ${REDEEMER_START_BLOCK})" - pnpm prepare:env + echo "⚙️ Preparing subgraph for ${DEPLOY_ENV} from ${ENV_FILE}..." + ENV_FILE="$ENV_FILE" pnpm prepare:env echo "✅ Configuration ready" echo "--- subgraph.yaml ---" cat subgraph.yaml @@ -220,7 +221,7 @@ jobs: id: goldsky_deploy working-directory: ./points-indexer env: - GOLDSKY_API_KEY: ${{ needs.build.outputs.environment == 'dev' && secrets.DEV_GOLDSKY_API_KEY || needs.build.outputs.environment == 'stg' && secrets.STG_GOLDSKY_API_KEY || secrets.LMN_GOLDSKY_API_KEY }} + GOLDSKY_API_KEY: ${{ needs.build.outputs.environment == 'dev' && secrets.DEV_GOLDSKY_API_KEY || secrets.LMN_GOLDSKY_API_KEY }} GOLDSKY_SUBGRAPH_NAME: ${{ needs.build.outputs.goldsky_subgraph_name }} SUBGRAPH_VERSION: ${{ needs.build.outputs.version }} GOLDSKY_ROLLING_TAG: ${{ needs.build.outputs.goldsky_rolling_tag }} @@ -347,7 +348,7 @@ jobs: if [ -z "${GOLDSKY_ENDPOINT}" ]; then echo "⚠️ No Goldsky endpoint URL configured — skipping verification" - echo " Set DEV_GS_POINTS / STG_GS_POINTS / LMN_GS_POINTS org variable" + echo " Set DEV_GS_POINTS / LMN_GS_POINTS org variable" exit 0 fi diff --git a/.github/workflows/deploy-subgraph.yml b/.github/workflows/deploy-subgraph.yml index f507403..eb219a2 100644 --- a/.github/workflows/deploy-subgraph.yml +++ b/.github/workflows/deploy-subgraph.yml @@ -4,31 +4,30 @@ on: push: branches: - dev - - stg - main - "cicd/**" paths: - "indexer/**" + - "config/*.env" - "contracts/abi/CollateralVault.json" - ".github/workflows/deploy-subgraph.yml" pull_request: branches: - dev - - stg - main paths: - "indexer/**" + - "config/*.env" - "contracts/abi/CollateralVault.json" - ".github/workflows/deploy-subgraph.yml" workflow_dispatch: inputs: environment: - description: "Target environment (dev=DEV, stg=STG, main=LMN/PROD)" + description: "Target environment (dev=DEV, main=LMN/PROD)" required: true type: choice options: - dev - - stg - main concurrency: @@ -114,19 +113,21 @@ jobs: fi echo "goldsky_subgraph_name=$GS_NAME" >> $GITHUB_OUTPUT - # Rolling tag and public endpoint per environment + # Rolling tag, public endpoint, and the config/.env to load. case $ENV in dev) echo "goldsky_rolling_tag=dev-latest" >> $GITHUB_OUTPUT echo "goldsky_endpoint=${{ vars.DEV_GS_VAULT }}" >> $GITHUB_OUTPUT - ;; - stg) - echo "goldsky_rolling_tag=stg-latest" >> $GITHUB_OUTPUT - echo "goldsky_endpoint=${{ vars.STG_GS_VAULT }}" >> $GITHUB_OUTPUT + echo "config_env=dev" >> $GITHUB_OUTPUT ;; main) echo "goldsky_rolling_tag=lmn-latest" >> $GITHUB_OUTPUT echo "goldsky_endpoint=${{ vars.LMN_GS_VAULT }}" >> $GITHUB_OUTPUT + echo "config_env=prd" >> $GITHUB_OUTPUT + ;; + *) + echo "::error::Unknown environment '$ENV' (expected dev or main)" + exit 1 ;; esac @@ -136,25 +137,24 @@ jobs: working-directory: ./indexer run: pnpm install --frozen-lockfile + # Addresses, start blocks and NETWORK all come from config/.env. + # PR builds render from the dummy values in .env.example, since real + # addresses are only needed at deploy time. - name: Prepare subgraph configuration working-directory: ./indexer env: - NETWORK: ${{ vars.NETWORK }} - VAULT_ADDRESS: ${{ vars.VAULT_ADDRESS }} - VAULT_START_BLOCK: ${{ vars.VAULT_START_BLOCK }} - PERPS_ADDRESS: ${{ vars.PERPS_ADDRESS }} - OPTIONS_ADDRESS: ${{ vars.OPTIONS_ADDRESS }} + CONFIG_ENV: ${{ steps.env.outputs.config_env }} + DEPLOY_ENV: ${{ needs.setup.outputs.environment }} run: | + set -euo pipefail if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then echo "ℹ️ PR build — using .env.example dummy values" - set -a && source .env.example && set +a + ENV_FILE=.env.example + else + ENV_FILE="../config/${CONFIG_ENV}.env" fi - echo "⚙️ Preparing subgraph for ${{ needs.setup.outputs.environment }}..." - echo " Network: ${NETWORK}" - echo " Vault: ${VAULT_ADDRESS} (block ${VAULT_START_BLOCK})" - echo " Perps: ${PERPS_ADDRESS}" - echo " Options: ${OPTIONS_ADDRESS}" - pnpm prepare:env + echo "⚙️ Preparing subgraph for ${DEPLOY_ENV} from ${ENV_FILE}..." + ENV_FILE="$ENV_FILE" pnpm prepare:env echo "✅ Configuration ready" echo "--- subgraph.yaml ---" cat subgraph.yaml @@ -219,7 +219,7 @@ jobs: id: goldsky_deploy working-directory: ./indexer env: - GOLDSKY_API_KEY: ${{ needs.build.outputs.environment == 'dev' && secrets.DEV_GOLDSKY_API_KEY || needs.build.outputs.environment == 'stg' && secrets.STG_GOLDSKY_API_KEY || secrets.LMN_GOLDSKY_API_KEY }} + GOLDSKY_API_KEY: ${{ needs.build.outputs.environment == 'dev' && secrets.DEV_GOLDSKY_API_KEY || secrets.LMN_GOLDSKY_API_KEY }} GOLDSKY_SUBGRAPH_NAME: ${{ needs.build.outputs.goldsky_subgraph_name }} SUBGRAPH_VERSION: ${{ needs.build.outputs.version }} GOLDSKY_ROLLING_TAG: ${{ needs.build.outputs.goldsky_rolling_tag }} @@ -346,7 +346,7 @@ jobs: if [ -z "${GOLDSKY_ENDPOINT}" ]; then echo "⚠️ No Goldsky endpoint URL configured — skipping verification" - echo " Set DEV_GS_VAULT / STG_GS_VAULT / LMN_GS_VAULT org variable" + echo " Set DEV_GS_VAULT / LMN_GS_VAULT org variable" exit 0 fi diff --git a/.github/workflows/indexer-tests.yml b/.github/workflows/indexer-tests.yml index eef8d5d..80ef104 100644 --- a/.github/workflows/indexer-tests.yml +++ b/.github/workflows/indexer-tests.yml @@ -64,8 +64,7 @@ jobs: # are only needed at deploy time. - name: Prepare subgraph (with .env.example defaults) run: | - set -a && source .env.example && set +a - pnpm prepare:env + ENV_FILE=.env.example pnpm prepare:env echo "--- subgraph.yaml ---" cat subgraph.yaml diff --git a/.github/workflows/points-indexer-tests.yml b/.github/workflows/points-indexer-tests.yml index d71c51a..ae0e4ce 100644 --- a/.github/workflows/points-indexer-tests.yml +++ b/.github/workflows/points-indexer-tests.yml @@ -76,8 +76,7 @@ jobs: - name: Prepare subgraph (with .env.example defaults) working-directory: ./points-indexer run: | - set -a && source .env.example && set +a - pnpm prepare:env + ENV_FILE=.env.example pnpm prepare:env echo "--- subgraph.yaml ---" cat subgraph.yaml diff --git a/config/dev.env b/config/dev.env new file mode 100644 index 0000000..57cb539 --- /dev/null +++ b/config/dev.env @@ -0,0 +1,50 @@ +# Public, per-environment values for DEV (base-sepolia). +# +# Loaded by Hardhat (`--env dev`), the market-maker and keeper (`--env-file`), +# the subgraph renderers (`ENV_FILE`), and the deploy workflows, which build the +# ECS environment block from the keys in this file. +# +# Never put secrets here. Those live in the repo-root `.env` locally and in +# GitHub Secrets in CI, and both take precedence over this file. +# +# Values are unquoted on purpose: `docker run --env-file` does not strip quotes +# in every CLI version, unlike Node, bash, and Compose. + +NETWORK=base-sepolia + +# ── Contracts ────────────────────────────────────────────────────────────── +BTC_USD_FEED_ADDRESS=0x37b5E07C59238ad3bB11AC27129387A67F3340B6 +COLLATERAL_TOKEN_ADDRESS=0xdd15eed84065a58c9e9ff9e95fb996be0fff22aa +FUTURES_ADDRESS=0x56d8d4a03a0f34b93B86E0b7941aFF29178D0479 +HASHPRICE_USD_ADDRESS=0x865c4fB61B85CDA3D39A94D4e8DE6962f7626C4D +HOOK_ADDRESS=0x99c28ff216a80e1a14ff276775ec458150979959 +PERPS_ADDRESS=0x0d412BC34a48e434144687Aac03b9C593F5237B6 +PME_ADDRESS=0x3899e429Ef47140eC46c6E23F04253C24F221b69 +POINTS_ADDRESS=0x153F6cb4386d717AD94791E6Ee8ae37f80315972 +VAULT_ADDRESS=0x54A79e2a5C60ACe37b280eBbCda51b4E903d25F0 +SAFE_OWNER_ADDRESS= +# OPTIONS_ADDRESS is unset: no options engine on base-sepolia yet. The vault +# subgraph buckets an internal transfer as OTHER when it does not match. + +# ── Subgraph manifests ───────────────────────────────────────────────────── +POINTS_START_BLOCK=42622435 +# TODO: VAULT_START_BLOCK has never been set in the GitHub dev environment, so +# the vault subgraph has been rendering an empty startBlock. Set it to the +# CollateralVault deployment block. + +# ── PME stress shocks (WAD-scaled) ───────────────────────────────────────── +IM_SPOT_SHOCK=100000000000000000 +MM_SPOT_SHOCK=100000000000000000 +IM_VOL_SHOCK=50000000000000000 +MM_VOL_SHOCK=50000000000000000 + +# ── Keeper runtime ───────────────────────────────────────────────────────── +BACKFILL_FROM_BLOCK=45600575 +DELIVERY_KEEPER_ENABLED=true +HEALTH_PORT=3000 +KEEPER_MIN_PROFIT_MARGIN=0 +LOG_LEVEL=info + +# ── Market-maker runtime ─────────────────────────────────────────────────── +MAKER_LOG_LEVEL=debug +HASHPRICE_ORACLE_SUBGRAPH_URL=https://api.goldsky.com/api/public/project_cmmz59uoa7b5201wthnkxbuqy/subgraphs/hpow-oracles/dev-latest/gn diff --git a/config/prd.env b/config/prd.env new file mode 100644 index 0000000..6c4072d --- /dev/null +++ b/config/prd.env @@ -0,0 +1,48 @@ +# Public, per-environment values for PRD (base mainnet). +# +# Loaded by Hardhat (`--env prd`), the market-maker and keeper (`--env-file`), +# the subgraph renderers (`ENV_FILE`), and the deploy workflows, which build the +# ECS environment block from the keys in this file. +# +# Never put secrets here. Those live in the repo-root `.env` locally and in +# GitHub Secrets in CI, and both take precedence over this file. +# +# Values are unquoted on purpose: `docker run --env-file` does not strip quotes +# in every CLI version, unlike Node, bash, and Compose. + +# `base` is the Graph manifest name, the viem chain key, and the market-maker +# YAML network name. The keeper still accepts the older `base-mainnet` spelling +# as a deprecated alias. +NETWORK=base + +# ── Contracts ────────────────────────────────────────────────────────────── +BTC_USD_ADDRESS=0x64c911996D3c6aC71f9b455B1E8E7266BcbD848F +COLLATERAL_TOKEN_ADDRESS=0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913 +FUTURES_ADDRESS=0xf97a1bbfb5e061ef73dad8ebf25939d93639fb7f +HASHPRICE_BTC_ADDRESS=0x70027c6f1b40e7461172af1241330b499c8c2e22 +PERPS_ADDRESS=0x794f9e63b7666985256f1d2763ee24cc0b528199 +PME_ADDRESS=0x5F047CCE438ae7796140506a5edf3D711034aaF3 +POINTS_ADDRESS=0x52e1b275d7f925e48f74d304e6d7e8ca489de6b9 +POINTS_HOOK_ADDRESS=0x81f47f6c84ffb1a5daa8c54989a4cb9458017188 +SAFE_OWNER_ADDRESS=0x57ac51Ad8b3B5a95e655eD2AF98D9881B136f924 +VAULT_ADDRESS=0x0730422E49B76A2D36d51304ACEcbe4f444821F8 + + +# ── Subgraph manifests ───────────────────────────────────────────────────── +# TODO: VAULT_START_BLOCK / POINTS_START_BLOCK need the mainnet deployment +# blocks before the prd subgraphs will index anything. + +# ── PME stress shocks (WAD-scaled) ───────────────────────────────────────── +IM_SPOT_SHOCK=100000000000000000 +MM_SPOT_SHOCK=100000000000000000 +IM_VOL_SHOCK=50000000000000000 +MM_VOL_SHOCK=50000000000000000 + +# ── Keeper runtime ───────────────────────────────────────────────────────── +DELIVERY_KEEPER_ENABLED=true +HEALTH_PORT=3000 +KEEPER_MIN_PROFIT_MARGIN=0 +LOG_LEVEL=info + +# ── Market-maker runtime ─────────────────────────────────────────────────── +MAKER_LOG_LEVEL=info diff --git a/contracts/.env.example b/contracts/.env.example index 123e6d2..313d9c3 100644 --- a/contracts/.env.example +++ b/contracts/.env.example @@ -5,7 +5,7 @@ ALCHEMY_API_KEY= # ── Deployer ────────────────────────────────────────────────────────────── # Hex-encoded private key used as the deployer for `--network base-sepolia` and -# `--network base-mainnet`. Not required for `localhost` / `hardhat` networks. +# `--network base`. Not required for `localhost` / `hardhat` networks. PRIVATE_KEY= # ── Block explorer verification ─────────────────────────────────────────── diff --git a/contracts/.gitignore b/contracts/.gitignore index 387c22a..8b27c0f 100644 --- a/contracts/.gitignore +++ b/contracts/.gitignore @@ -5,5 +5,4 @@ cache *.tsbuildinfo # Deploy script output (deployed addresses) -points-addr.tmp -points-hook-addr.tmp +*.tmp diff --git a/contracts/hardhat.config.ts b/contracts/hardhat.config.ts index 10e576b..9d6b839 100644 --- a/contracts/hardhat.config.ts +++ b/contracts/hardhat.config.ts @@ -1,13 +1,16 @@ import { configVariable, defineConfig } from "hardhat/config"; import hardhatToolboxViem from "@nomicfoundation/hardhat-toolbox-viem"; import hardhatViemAbi from "hardhat-viem-abi"; -import { tryLoadEnvFile } from "./lib/env.ts"; - -tryLoadEnvFile("./../.env"); -tryLoadEnvFile(".env"); +import envLoader from "./plugins/env-loader/index.ts"; export default defineConfig({ - plugins: [hardhatToolboxViem, hardhatViemAbi], + plugins: [hardhatToolboxViem, hardhatViemAbi, envLoader], + envLoader: { + configDir: "../config", + // Machine/secret values; win over the named env file for overlapping keys. + overrideEnvFiles: ["../.env", ".env"], + }, + codegen: { // Keepers and the UI install `abi/` as this package name; do not rename casually. packageJson: { name: "collateral-margin-abi" }, @@ -50,7 +53,7 @@ export default defineConfig({ etherscan: { apiKey: configVariable("ETHERSCAN_API_KEY"), enabled: true, - }, + } }, networks: { hardhat: { @@ -70,7 +73,7 @@ export default defineConfig({ url: configVariable("ALCHEMY_API_KEY", "https://base-sepolia.g.alchemy.com/v2/{variable}"), accounts: [configVariable("PRIVATE_KEY")], }, - "base-mainnet": { + base: { type: "http", chainType: "l1", chainId: 8453, diff --git a/contracts/lib/env.ts b/contracts/lib/env.ts index 6f17aef..0910ccf 100644 --- a/contracts/lib/env.ts +++ b/contracts/lib/env.ts @@ -16,8 +16,9 @@ export function requireEnvsSet( export function tryLoadEnvFile(path: string): void { try { loadEnvFile(path); + console.info(`Loaded env file ${path}`); } catch (err: unknown) { - console.info(`Failed to load env file ${path}:\n${(err as Error).message}`); + console.info(`Env file ${path} not loaded: ${(err as Error).message}`); } } diff --git a/contracts/package.json b/contracts/package.json index 4c1b8f2..cf64254 100644 --- a/contracts/package.json +++ b/contracts/package.json @@ -12,12 +12,8 @@ "typecheck": "tsgo --noEmit", "lint": "biome lint .", "clean": "rm -rf abi artifacts cache", - "deploy:vault": "hardhat run scripts/deploy-collateral-vault.ts", - "deploy:pme": "hardhat run scripts/deploy-portfolio-margin-engine.ts", - "deploy:points": "hardhat run scripts/deploy-points.ts", - "deploy:points-hook": "hardhat run scripts/deploy-points-hook.ts", - "upgrade:vault": "hardhat run scripts/update-collateral-vault.ts", - "upgrade:pme": "hardhat run scripts/update-portfolio-margin-engine.ts" + "run:dev": "hardhat run --env dev", + "run:prod": "hardhat run --env prd" }, "devDependencies": { "@biomejs/biome": "^2.4.10", diff --git a/contracts/plugins/env-loader/config-hooks.ts b/contracts/plugins/env-loader/config-hooks.ts new file mode 100644 index 0000000..89cad96 --- /dev/null +++ b/contracts/plugins/env-loader/config-hooks.ts @@ -0,0 +1,109 @@ +import { existsSync, readdirSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { loadEnvFile } from "node:process"; +import type { ConfigHooks } from "hardhat/types/hooks"; +import "./type-extensions.ts"; + +/** + * The npm package holding this plugin, which is also the Hardhat project root. + * Resolved from this file so that every path is independent of the cwd. + */ +function findProjectRoot(): string { + let dir = import.meta.dirname; + while (!existsSync(resolve(dir, "package.json"))) { + const parent = dirname(dir); + if (parent === dir) + throw new Error("env-loader: no package.json above the plugin"); + dir = parent; + } + return dir; +} + +/** Every `.env` file in `configDir`, by name. */ +function availableEnvs(configDir: string): string[] { + try { + return readdirSync(configDir) + .filter((file) => file.endsWith(".env")) + .map((file) => file.slice(0, -".env".length)) + .sort(); + } catch { + return []; + } +} + +/** Reads the env name from a `--env ` or `--env=` argument. */ +function readEnvFlag(argv: string[], configDir: string): string | undefined { + const index = argv.findIndex( + (arg) => arg === "--env" || arg.startsWith("--env="), + ); + if (index === -1) return undefined; + const arg = argv[index]; + const name = arg.startsWith("--env=") + ? arg.slice("--env=".length) + : argv[index + 1]; + const known = availableEnvs(configDir); + if (name === undefined || !known.includes(name)) + throw new Error( + `--env must name a file in ${configDir}, one of ${known.join(", ") || "(none found)"}, got ${name ?? "nothing"}`, + ); + return name; +} + +/** + * Loads the env files for the environment named by `--env`, and selects that + * environment's network so scripts cannot be pointed at the wrong chain by + * accident. + * + * `loadEnvFile` never overwrites a variable that is already set, so files are + * read most-specific first: `overrideEnvFiles` (machine/secret), then the + * named env file. The real process environment always wins. + */ +export function loadEnv( + configDir: string, + overrideEnvFiles: string[], + projectRoot: string, + argv = process.argv, +): void { + for (const file of overrideEnvFiles) { + tryLoadEnvFile(resolve(projectRoot, file)); + } + const name = readEnvFlag(argv, configDir); + if (name !== undefined) tryLoadEnvFile(resolve(configDir, `${name}.env`)); + if (name === undefined) return; + + const network = process.env.NETWORK; + if (!network) throw new Error(`${name}.env must set NETWORK`); + // An explicit `--network` still wins: Hardhat prefers CLI args over env vars. + process.env.HARDHAT_NETWORK ??= network; +} + +export default async (): Promise> => ({ + // This is the earliest hook Hardhat runs, and crucially it runs before + // global options are resolved, so `HARDHAT_NETWORK` is still read from here. + async extendUserConfig(config, next) { + // The config file is loaded untypechecked, so this is worth stating plainly. + const { configDir, overrideEnvFiles } = config.envLoader ?? {}; + if (typeof configDir !== "string") + throw new Error("envLoader.configDir is required and must be a string"); + if ( + !Array.isArray(overrideEnvFiles) || + !overrideEnvFiles.every((p) => typeof p === "string") + ) + throw new Error( + "envLoader.overrideEnvFiles is required and must be an array of strings", + ); + + const projectRoot = findProjectRoot(); + loadEnv(resolve(projectRoot, configDir), overrideEnvFiles, projectRoot); + return next(config); + }, +}); + +export function tryLoadEnvFile(path: string): void { + try { + loadEnvFile(path); + console.info(`Loaded env file ${path}`); + } catch (err: unknown) { + console.info(`Env file ${path} not loaded: ${(err as Error).message}`); + } +} diff --git a/contracts/plugins/env-loader/index.ts b/contracts/plugins/env-loader/index.ts new file mode 100644 index 0000000..8f8f5e3 --- /dev/null +++ b/contracts/plugins/env-loader/index.ts @@ -0,0 +1,25 @@ +import { globalOption } from "hardhat/config"; +import { ArgumentType } from "hardhat/types/arguments"; +import type { HardhatPlugin } from "hardhat/types/plugins"; +import "./type-extensions.ts"; + +/** + * For `--env `, loads `envLoader.overrideEnvFiles` then `.env` + * from `envLoader.configDir`, and connects to the network named by its `NETWORK`. + */ +const envLoaderPlugin: HardhatPlugin = { + id: "env-loader", + globalOptions: [ + globalOption({ + name: "env", + description: "The environment to load .env for", + type: ArgumentType.STRING_WITHOUT_DEFAULT, + defaultValue: undefined, + }), + ], + hookHandlers: { + config: () => import("./config-hooks.ts"), + }, +}; + +export default envLoaderPlugin; diff --git a/contracts/plugins/env-loader/type-extensions.ts b/contracts/plugins/env-loader/type-extensions.ts new file mode 100644 index 0000000..b0d05fb --- /dev/null +++ b/contracts/plugins/env-loader/type-extensions.ts @@ -0,0 +1,18 @@ +import "hardhat/types/config"; + +declare module "hardhat/types/config" { + interface EnvLoaderUserConfig { + /** Directory holding the `.env` files, relative to the project root. */ + configDir: string; + /** + * Machine-specific or secret `.env` files, relative to the project root. + * Loaded before the named env file so their values win for overlapping keys + * (`loadEnvFile` never overwrites an already-set variable). + */ + overrideEnvFiles: string[]; + } + + interface HardhatUserConfig { + envLoader?: EnvLoaderUserConfig; + } +} diff --git a/indexer/package.json b/indexer/package.json index 0477030..04fea16 100644 --- a/indexer/package.json +++ b/indexer/package.json @@ -7,8 +7,9 @@ "type": "module", "scripts": { "clean": "rm -rf data generated build subgraph.yaml", - "prepare:env": "envsubst < subgraph.template.yaml > subgraph.yaml", - "prepare-local": "set -a && source ../.env && set +a && pnpm prepare:env", + "//prepare:env": "`.` searches PATH when its operand has no slash, so a bare filename must be made explicitly relative for dash, which is /bin/sh on the CI runners.", + "prepare:env": "ENV_FILE=\"${ENV_FILE:?ENV_FILE must point at an env file, e.g. ../config/dev.env}\"; case \"$ENV_FILE\" in */*) ;; *) ENV_FILE=\"./$ENV_FILE\" ;; esac; set -a && . \"$ENV_FILE\" && set +a && envsubst < subgraph.template.yaml > subgraph.yaml", + "prepare-local": "ENV_FILE=../config/dev.env pnpm prepare:env", "codegen": "graph codegen", "build": "graph build", "deploy": "graph deploy --node https://api.studio.thegraph.com/deploy/ collateral-vault", @@ -18,7 +19,7 @@ "setup-local": "pnpm prepare-local && pnpm codegen && pnpm build && pnpm create-local && pnpm deploy-local", "test": "graph test -v 0.6.0", "lint": "biome lint .", - "indexer": "docker compose --env-file ../.env up", + "indexer": "docker compose --env-file ../config/dev.env --env-file ../.env up", "graph:api": "open http://localhost:8030/graphql/playground", "lint:fix": "biome check --write .", "typecheck": "graph build" diff --git a/keeper/README.md b/keeper/README.md index 36fa5de..b40a99a 100644 --- a/keeper/README.md +++ b/keeper/README.md @@ -144,7 +144,7 @@ See `src/config.ts` for the authoritative shape. The minimum-viable set: | Var | Required | Purpose | | ------------------------------ | -------- | -------------------------------------- | -| `NETWORK` | yes | Chain label (e.g. `arbitrum-sepolia`) | +| `NETWORK` | yes | `hardhat`, `base-sepolia`, or `base` | | `ETH_NODE_ADDRESS` | yes | RPC URL | | `LIQUIDATOR_PRIVATE_KEY` | yes | Signer (single key for both venues) | | `VAULT_ADDRESS` | yes | Shared CollateralVault | diff --git a/keeper/package.json b/keeper/package.json index 16d8813..abe223f 100644 --- a/keeper/package.json +++ b/keeper/package.json @@ -4,13 +4,14 @@ "type": "module", "private": true, "engines": { - "node": ">=22.6.0" + "node": ">=22.9.0" }, "scripts": { "node": "node --import=amaro/strip", - "dev": "pnpm node --env-file=../.env src/index.ts | pino-pretty", - "dev:dry": "DRY_RUN=true pnpm node --env-file=.env src/index.ts | pino-pretty", - "start": "pnpm node --env-file=.env src/index.ts", + "dev": "pnpm node --env-file=../config/dev.env --env-file-if-exists=../.env --env-file-if-exists=.env src/index.ts | pino-pretty", + "dev:dry": "DRY_RUN=true pnpm node --env-file=../config/dev.env --env-file-if-exists=../.env --env-file-if-exists=.env src/index.ts | pino-pretty", + "prd": "pnpm node --env-file=../config/prd.env --env-file-if-exists=../.env --env-file-if-exists=.env src/index.ts", + "start": "pnpm node --env-file-if-exists=../.env --env-file-if-exists=.env src/index.ts", "test": "pnpm node --test --test-force-exit --test-concurrency=1 'tests/*.test.ts' 'tests/alert/**/*.test.ts' 'tests/coordinator/**/*.test.ts' 'tests/delivery/**/*.test.ts' 'tests/discovery/**/*.test.ts' 'tests/oracle/**/*.test.ts' 'tests/pme/**/*.test.ts' 'tests/predict/**/*.test.ts' 'tests/runtime/**/*.test.ts' 'tests/tx/**/*.test.ts' 'tests/venues/**/*.test.ts'", "test:watch": "pnpm node --test --watch --test-concurrency=1 'tests/**/*.test.ts'", "pretest:integration": "pnpm node ./scripts/compile-siblings.ts", diff --git a/keeper/scripts/debug-delivery-bootstrap.ts b/keeper/scripts/debug-delivery-bootstrap.ts index d03977f..2d14848 100644 --- a/keeper/scripts/debug-delivery-bootstrap.ts +++ b/keeper/scripts/debug-delivery-bootstrap.ts @@ -17,7 +17,7 @@ if (FUTURES === undefined || ALCHEMY === undefined) { } const RPC_URL = `https://${NETWORK}.g.alchemy.com/v2/${ALCHEMY}`; -const CHAINS = { "base-sepolia": baseSepolia, "base-mainnet": base, hardhat }; +const CHAINS = { "base-sepolia": baseSepolia, base, hardhat }; const chain = CHAINS[NETWORK as keyof typeof CHAINS]; const USERS: Address[] = ["0x1441Bc52156Cf18c12cde6A92aE6BDE8B7f775D4"]; diff --git a/keeper/src/chain.ts b/keeper/src/chain.ts index 6c8502f..8ac57ed 100644 --- a/keeper/src/chain.ts +++ b/keeper/src/chain.ts @@ -24,7 +24,7 @@ export interface Chain { const VIEM_CHAINS: Record = { hardhat, "base-sepolia": baseSepolia, - "base-mainnet": base, + base, }; /** diff --git a/keeper/src/config.ts b/keeper/src/config.ts index c3b214e..6b545d2 100644 --- a/keeper/src/config.ts +++ b/keeper/src/config.ts @@ -21,14 +21,24 @@ import type pino from "pino"; * - coordinator: cross-account ordering + concurrency * - runtime: healthcheck port, log level, dry-run, intervals */ -export type NetworkName = "hardhat" | "base-sepolia" | "base-mainnet"; +export type NetworkName = "hardhat" | "base-sepolia" | "base"; export const SUPPORTED_NETWORKS: readonly NetworkName[] = [ "hardhat", "base-sepolia", - "base-mainnet", + "base", ] as const; +/** + * Names accepted on input but normalized before use. `base-mainnet` was the + * keeper's own spelling before `NETWORK` was unified across the subgraph + * manifests and the market-maker; task definitions registered before that + * change still carry it. + */ +const NETWORK_ALIASES: Readonly> = { + "base-mainnet": "base", +}; + export interface Config { /** * Build identity stamp (image tag / git describe), surfaced on `/health` @@ -293,10 +303,11 @@ function parseAddressList(name: string): readonly Address[] { } function requireNetwork(): NetworkName { - const value = requireEnv("NETWORK"); + const raw = requireEnv("NETWORK"); + const value = NETWORK_ALIASES[raw] ?? raw; if (!(SUPPORTED_NETWORKS as readonly string[]).includes(value)) { throw new Error( - `NETWORK must be one of ${SUPPORTED_NETWORKS.join("|")}, got "${value}"`, + `NETWORK must be one of ${SUPPORTED_NETWORKS.join("|")}, got "${raw}"`, ); } return value as NetworkName; @@ -315,9 +326,10 @@ function resolveRpcUrl(network: NetworkName): string { return process.env.HARDHAT_RPC_URL ?? "http://127.0.0.1:8545"; } + // Alchemy keeps its own spelling for mainnet. const alchemySubdomain: Record, string> = { "base-sepolia": "base-sepolia", - "base-mainnet": "base-mainnet", + base: "base-mainnet", }; const apiKey = requireEnv("ALCHEMY_API_KEY"); return `https://${alchemySubdomain[network]}.g.alchemy.com/v2/${apiKey}`; diff --git a/market-maker/package.json b/market-maker/package.json index efd1030..49258e0 100644 --- a/market-maker/package.json +++ b/market-maker/package.json @@ -4,7 +4,7 @@ "type": "module", "private": true, "engines": { - "node": ">=22.6.0" + "node": ">=22.9.0" }, "scripts": { "test": "pnpm node --test --test-force-exit --test-concurrency=1 'tests/**/*.test.ts'", @@ -18,18 +18,15 @@ "perps": "pnpm node --watch src/apps/perps/main.ts", "futures": "pnpm node --watch src/apps/futures/main.ts", "portfolio": "pnpm node --watch src/apps/portfolio/main.ts", - "local:perps": "pnpm perps --config configs/perps.local.yml | pino-pretty", - "local:futures": "pnpm futures --config configs/futures.local.yml | pino-pretty", - "local:portfolio": "pnpm portfolio --config configs/portfolio.local.yml | pino-pretty", - "dev:perps": "pnpm perps --config configs/perps.dev.yml | pino-pretty", - "dev:futures": "pnpm futures --config configs/futures.dev.yml | pino-pretty", - "dev:portfolio": "pnpm portfolio --config configs/portfolio.dev.yml | pino-pretty", - "stg:perps": "pnpm perps --config configs/perps.stg.yml", - "stg:futures": "pnpm futures --config configs/futures.stg.yml", - "stg:portfolio": "pnpm portfolio --config configs/portfolio.stg.yml", - "prd:perps": "pnpm perps --config configs/perps.prd.yml", - "prd:futures": "pnpm futures --config configs/futures.prd.yml", - "prd:portfolio": "pnpm portfolio --config configs/portfolio.prd.yml", + "local:perps": "pnpm node --env-file-if-exists=../.env --env-file-if-exists=.env --watch src/apps/perps/main.ts --config configs/perps.local.yml | pino-pretty", + "local:futures": "pnpm node --env-file-if-exists=../.env --env-file-if-exists=.env --watch src/apps/futures/main.ts --config configs/futures.local.yml | pino-pretty", + "local:portfolio": "pnpm node --env-file-if-exists=../.env --env-file-if-exists=.env --watch src/apps/portfolio/main.ts --config configs/portfolio.local.yml | pino-pretty", + "dev:perps": "pnpm node --env-file=../config/dev.env --env-file-if-exists=../.env --env-file-if-exists=.env --watch src/apps/perps/main.ts --config configs/perps.dev.yml | pino-pretty", + "dev:futures": "pnpm node --env-file=../config/dev.env --env-file-if-exists=../.env --env-file-if-exists=.env --watch src/apps/futures/main.ts --config configs/futures.dev.yml | pino-pretty", + "dev:portfolio": "pnpm node --env-file=../config/dev.env --env-file-if-exists=../.env --env-file-if-exists=.env --watch src/apps/portfolio/main.ts --config configs/portfolio.dev.yml | pino-pretty", + "prd:perps": "pnpm node --env-file=../config/prd.env --env-file-if-exists=../.env --env-file-if-exists=.env src/apps/perps/main.ts --config configs/perps.prd.yml", + "prd:futures": "pnpm node --env-file=../config/prd.env --env-file-if-exists=../.env --env-file-if-exists=.env src/apps/futures/main.ts --config configs/futures.prd.yml", + "prd:portfolio": "pnpm node --env-file=../config/prd.env --env-file-if-exists=../.env --env-file-if-exists=.env src/apps/portfolio/main.ts --config configs/portfolio.prd.yml", "lint:fix": "biome check --write ." }, "dependencies": { diff --git a/market-maker/src/apps/futures/main.ts b/market-maker/src/apps/futures/main.ts index d17233f..6a2604e 100644 --- a/market-maker/src/apps/futures/main.ts +++ b/market-maker/src/apps/futures/main.ts @@ -1,5 +1,4 @@ import pino from "pino"; -import { loadDotenvFiles } from "../../core/env.ts"; import { createNetworkClients } from "../../core/client.ts"; import { WalletRegistry } from "../../core/wallet.ts"; import { OracleTracker } from "../../core/oracleTracker.ts"; @@ -19,7 +18,6 @@ import { sanitiseConfig } from "../../core/config/base.ts"; import { loadFuturesConfig } from "./config.ts"; async function main(): Promise { - loadDotenvFiles(import.meta.dirname); const config = loadFuturesConfig(); const logger = pino({ level: config.logLevel, diff --git a/market-maker/src/apps/perps/main.ts b/market-maker/src/apps/perps/main.ts index c466fab..4ef60e3 100644 --- a/market-maker/src/apps/perps/main.ts +++ b/market-maker/src/apps/perps/main.ts @@ -1,5 +1,4 @@ import pino from "pino"; -import { loadDotenvFiles } from "../../core/env.ts"; import { createNetworkClients } from "../../core/client.ts"; import { WalletRegistry } from "../../core/wallet.ts"; import { OracleTracker } from "../../core/oracleTracker.ts"; @@ -20,7 +19,6 @@ import { sanitiseConfig } from "../../core/config/base.ts"; import { loadPerpsConfig } from "./config.ts"; async function main(): Promise { - loadDotenvFiles(import.meta.dirname); const config = loadPerpsConfig(); const logger = pino({ level: config.logLevel, diff --git a/market-maker/src/apps/portfolio/main.ts b/market-maker/src/apps/portfolio/main.ts index 8915eec..29a70b9 100644 --- a/market-maker/src/apps/portfolio/main.ts +++ b/market-maker/src/apps/portfolio/main.ts @@ -1,5 +1,4 @@ import pino from "pino"; -import { loadDotenvFiles } from "../../core/env.ts"; import { createNetworkClients } from "../../core/client.ts"; import { WalletRegistry } from "../../core/wallet.ts"; import { OracleTracker } from "../../core/oracleTracker.ts"; @@ -177,7 +176,6 @@ function buildMarket( } async function main(): Promise { - loadDotenvFiles(import.meta.dirname); const config = loadPortfolioConfig(); const logger = pino({ level: config.logLevel, serializers: { err: serializeError } }); logger.info( diff --git a/market-maker/src/core/env.ts b/market-maker/src/core/env.ts deleted file mode 100644 index 7095061..0000000 --- a/market-maker/src/core/env.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { existsSync } from "node:fs"; -import { resolve } from "node:path"; - -/** - * Load `.env` files at startup, in priority order: - * - * 1. market-maker/.env (closest to the app, wins) - * 2. collateral-margin/.env (shared between contracts/indexer/mm) - * - * Existing `process.env` values always take precedence over file contents - * (Node's documented behaviour for `process.loadEnvFile`), so CI/docker - * runtime env still wins. - * - * Missing files are silently skipped — `.env` is a dev convenience only. - * - * The market-maker root is two directories above `src/`. The - * collateral-margin root is three directories above `src/`. We resolve - * from `import.meta.dirname` of the caller (passed in) so the paths - * work regardless of cwd. - */ -export function loadDotenvFiles(callerDir: string): void { - // src/apps/ → src/apps → src → market-maker → collateral-margin - const marketMakerRoot = resolve(callerDir, "..", "..", ".."); - const repoRoot = resolve(marketMakerRoot, ".."); - - for (const path of [ - resolve(marketMakerRoot, ".env"), - resolve(repoRoot, ".env"), - ]) { - if (existsSync(path)) { - try { - process.loadEnvFile(path); - } catch { - // ignore parse errors — runtime config validation will catch - // truly missing values. - } - } - } -} diff --git a/points-indexer/package.json b/points-indexer/package.json index 8badcc6..88dce6b 100644 --- a/points-indexer/package.json +++ b/points-indexer/package.json @@ -7,8 +7,9 @@ "type": "module", "scripts": { "clean": "rm -rf data generated build subgraph.yaml", - "prepare:env": "envsubst < subgraph.template.yaml > subgraph.yaml", - "prepare-local": "set -a && source ../.env && set +a && pnpm prepare:env", + "//prepare:env": "`.` searches PATH when its operand has no slash, so a bare filename must be made explicitly relative for dash, which is /bin/sh on the CI runners.", + "prepare:env": "ENV_FILE=\"${ENV_FILE:?ENV_FILE must point at an env file, e.g. ../config/dev.env}\"; case \"$ENV_FILE\" in */*) ;; *) ENV_FILE=\"./$ENV_FILE\" ;; esac; set -a && . \"$ENV_FILE\" && set +a && envsubst < subgraph.template.yaml > subgraph.yaml", + "prepare-local": "ENV_FILE=../config/dev.env pnpm prepare:env", "codegen": "graph codegen", "build": "graph build", "deploy": "graph deploy --node https://api.studio.thegraph.com/deploy/ points", @@ -20,7 +21,6 @@ "test:integration": "hardhat test nodejs", "test:integration:debug": "MATCHSTICK_VERBOSE=true hardhat test nodejs", "lint": "biome lint .", - "indexer": "docker compose --env-file ../.env up", "graph:api": "open http://localhost:8030/graphql/playground", "lint:fix": "biome check --write .", "typecheck": "tsgo --noEmit -p integration/tsconfig.json"