Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,4 +45,6 @@ jobs:
az containerapp update \
--resource-group "$AZURE_RESOURCE_GROUP" \
--name "$AZURE_CONTAINER_APP" \
--min-replicas 0 \
--max-replicas 3 \
--image "$AZURE_CONTAINER_REGISTRY.azurecr.io/$IMAGE_NAME:${GITHUB_SHA}"
127 changes: 127 additions & 0 deletions .github/workflows/optimize-azure-costs.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
name: Apply Azure cost optimizations

on:
workflow_dispatch:
inputs:
confirmation:
description: Type APPLY-COST-OPTIMIZATION to continue
required: true
type: string

permissions:
contents: read
id-token: write

env:
COSMOS_SUBSCRIPTION_ID: 5ba12f7d-8235-4c6a-857c-2dc8e4fcb50a
COSMOS_RESOURCE_GROUP: rg-devglobe
COSMOS_ACCOUNT: devglobe-cosmos
COSMOS_DATABASE: devglobe
AUTOSCALE_MAX_RU: 4000
IMPACT_HISTORY_TTL_SECONDS: 10368000

jobs:
optimize:
if: inputs.confirmation == 'APPLY-COST-OPTIMIZATION'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5

- name: Log in to Azure
uses: azure/login@v2
with:
client-id: ${{ vars.AZURE_CLIENT_ID }}
tenant-id: ${{ vars.AZURE_TENANT_ID }}
subscription-id: ${{ env.COSMOS_SUBSCRIPTION_ID }}

- name: Validate current Cosmos throughput
id: throughput
shell: bash
run: |
set -euo pipefail
current_manual="$(az cosmosdb sql database throughput show \
--subscription "$COSMOS_SUBSCRIPTION_ID" \
--resource-group "$COSMOS_RESOURCE_GROUP" \
--account-name "$COSMOS_ACCOUNT" \
--name "$COSMOS_DATABASE" \
--query resource.throughput \
--output tsv)"
current_autoscale="$(az cosmosdb sql database throughput show \
--subscription "$COSMOS_SUBSCRIPTION_ID" \
--resource-group "$COSMOS_RESOURCE_GROUP" \
--account-name "$COSMOS_ACCOUNT" \
--name "$COSMOS_DATABASE" \
--query resource.autoscaleSettings.maxThroughput \
--output tsv)"
if [[ "$current_autoscale" == "$AUTOSCALE_MAX_RU" ]]; then
echo "Cosmos autoscale is already configured at ${AUTOSCALE_MAX_RU} RU/s."
echo "already_optimized=true" >> "$GITHUB_OUTPUT"
echo "migration_required=false" >> "$GITHUB_OUTPUT"
exit 0
fi
if [[ -n "$current_autoscale" && "$current_autoscale" != "null" ]]; then
if (( current_autoscale < AUTOSCALE_MAX_RU )); then
echo "Refusing update: autoscale maximum ${current_autoscale} is below the reviewed target." >&2
exit 1
fi
echo "Autoscale is active at ${current_autoscale} RU/s; the target maximum will be repaired."
echo "already_optimized=false" >> "$GITHUB_OUTPUT"
echo "migration_required=false" >> "$GITHUB_OUTPUT"
exit 0
fi
if [[ "$current_manual" != "4000" ]]; then
echo "Refusing migration: expected manual throughput 4000 RU/s, found '${current_manual:-none}'." >&2
exit 1
fi
echo "already_optimized=false" >> "$GITHUB_OUTPUT"
echo "migration_required=true" >> "$GITHUB_OUTPUT"

- name: Migrate shared throughput to autoscale
if: steps.throughput.outputs.migration_required == 'true'
run: |
az cosmosdb sql database throughput update \
--subscription "$COSMOS_SUBSCRIPTION_ID" \
--resource-group "$COSMOS_RESOURCE_GROUP" \
--account-name "$COSMOS_ACCOUNT" \
--name "$COSMOS_DATABASE" \
--throughput-type autoscale
- name: Set autoscale maximum throughput
if: steps.throughput.outputs.already_optimized != 'true'
run: |
az cosmosdb sql database throughput update \
--subscription "$COSMOS_SUBSCRIPTION_ID" \
--resource-group "$COSMOS_RESOURCE_GROUP" \
--account-name "$COSMOS_ACCOUNT" \
--name "$COSMOS_DATABASE" \
--max-throughput "$AUTOSCALE_MAX_RU"

- name: Bound impact history retention
run: |
az cosmosdb sql container update \
--subscription "$COSMOS_SUBSCRIPTION_ID" \
--resource-group "$COSMOS_RESOURCE_GROUP" \
--account-name "$COSMOS_ACCOUNT" \
--database-name "$COSMOS_DATABASE" \
--name impact-history \
--ttl "$IMPACT_HISTORY_TTL_SECONDS"

- name: Verify optimized Cosmos configuration
shell: bash
run: |
set -euo pipefail
actual_max="$(az cosmosdb sql database throughput show \
--subscription "$COSMOS_SUBSCRIPTION_ID" \
--resource-group "$COSMOS_RESOURCE_GROUP" \
--account-name "$COSMOS_ACCOUNT" \
--name "$COSMOS_DATABASE" \
--query resource.autoscaleSettings.maxThroughput \
--output tsv)"
[[ "$actual_max" == "$AUTOSCALE_MAX_RU" ]]
az cosmosdb sql container show \
--subscription "$COSMOS_SUBSCRIPTION_ID" \
--resource-group "$COSMOS_RESOURCE_GROUP" \
--account-name "$COSMOS_ACCOUNT" \
--database-name "$COSMOS_DATABASE" \
--name impact-history \
--query '{ttl:resource.defaultTtl,indexingPolicy:resource.indexingPolicy}' \
--output json
31 changes: 23 additions & 8 deletions app/api/country-stats/route.js
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
import { CosmosClient } from '@azure/cosmos';
import { NextResponse } from 'next/server';
import { promises as fs } from 'fs';
import path from 'path';
import { withNumericScore } from '../../../lib/developer-score.js';
import { computeCountryStats } from '../../../lib/country-stats.js';
import { getCosmosContainer } from '../../../lib/cosmos.js';

const COSMOS_ENDPOINT = process.env.COSMOS_ENDPOINT;
const COSMOS_KEY = process.env.COSMOS_KEY;
const DATABASE = process.env.COSMOS_DATABASE || 'devglobe';
const CONTAINER = process.env.COSMOS_CONTAINER || 'developers';
const COUNTRY_STATS_CACHE_MS = 60 * 60 * 1000;
let countryStatsCache;
let countryStatsPromise;

async function getSampleData() {
const filePath = path.join(process.cwd(), 'data', 'developers-sample.json');
Expand All @@ -22,8 +24,8 @@ async function getDevelopersForStats() {
}

try {
const client = new CosmosClient({ endpoint: COSMOS_ENDPOINT, key: COSMOS_KEY });
const container = client.database(DATABASE).container(CONTAINER);
const container = getCosmosContainer(CONTAINER);
if (!container) return (await getSampleData()).map(withNumericScore);
const fields = 'c.location, c.score, c.topLanguage';
const query = `SELECT ${fields} FROM c WHERE (NOT IS_DEFINED(c.nomination) OR c.nomination.status = 'approved')`;
const { resources } = await container.items.query(query).fetchAll();
Expand All @@ -39,12 +41,25 @@ async function getDevelopersForStats() {
// normalization the globe/leaderboard country filter already relies on.
export async function GET() {
try {
const developers = await getDevelopersForStats();
const countries = computeCountryStats(developers);
const totalDevelopers = developers.filter(d => d.location).length;
if (!countryStatsCache || countryStatsCache.expiresAt <= Date.now()) {
countryStatsPromise ||= getDevelopersForStats()
.then(developers => {
countryStatsCache = {
value: {
countries: computeCountryStats(developers),
totalDevelopers: developers.filter(developer => developer.location).length,
},
expiresAt: Date.now() + COUNTRY_STATS_CACHE_MS,
};
})
.finally(() => {
countryStatsPromise = null;
});
await countryStatsPromise;
}

return NextResponse.json(
{ countries, totalDevelopers },
countryStatsCache.value,
{ headers: { 'Cache-Control': 's-maxage=3600, stale-while-revalidate=600' } },
);
} catch (err) {
Expand Down
7 changes: 3 additions & 4 deletions app/api/developer/route.js
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
import { CosmosClient } from '@azure/cosmos';
import { NextResponse } from 'next/server.js';
import { promises as fs } from 'fs';
import path from 'path';
import { getPublicAiProfile } from '../../../lib/ai-profile.js';
import { withNumericScore } from '../../../lib/developer-score.js';
import { apiError } from '../../../lib/api-error.js';
import { getCosmosContainer } from '../../../lib/cosmos.js';

const COSMOS_ENDPOINT = process.env.COSMOS_ENDPOINT;
const COSMOS_KEY = process.env.COSMOS_KEY;
const DATABASE = process.env.COSMOS_DATABASE || 'devglobe';
const CONTAINER = process.env.COSMOS_CONTAINER || 'developers';

async function getSampleData() {
Expand Down Expand Up @@ -51,8 +50,8 @@ export async function GET(request) {
}

try {
const client = new CosmosClient({ endpoint: COSMOS_ENDPOINT, key: COSMOS_KEY });
const container = client.database(DATABASE).container(CONTAINER);
const container = getCosmosContainer(CONTAINER);
if (!container) throw new Error('Cosmos DB is not configured');

const { resources } = await container.items.query({
query: "SELECT c.id, c.login, c.name, c.avatarUrl, c.bio, c.githubUrl, c.location, c.lat, c.lng, c.followers, c.totalStars, c.totalForks, c.totalWatchers, c.totalCommits, c.topLanguage, c.languages, c.publicRepos, c.topRepos, c.soReputation, c.soAnswers, c.soAcceptRate, c.soBadges, c.soUserId, c.score, c.scoreDimensions, c.scoreWeights, c.scoreHasSO, c.scorePercentile, c.specialTags, c.claimed, c.claimedAt, c.metricsUpdatedAt, c.aiProfile FROM c WHERE (c.id = @id OR c.login = @id) AND (NOT IS_DEFINED(c.nomination) OR c.nomination.status = 'approved')",
Expand Down
12 changes: 3 additions & 9 deletions app/api/developers/route.js
Original file line number Diff line number Diff line change
@@ -1,21 +1,14 @@
import { CosmosClient } from '@azure/cosmos';
import { NextResponse } from 'next/server';
import { promises as fs } from 'fs';
import path from 'path';
import { projectAgentReadiness } from '../../../lib/agent-network.js';
import { withNumericScore } from '../../../lib/developer-score.js';
import { parsePaginationParams } from '../../../lib/pagination.js';
import { getCosmosContainer } from '../../../lib/cosmos.js';

const COSMOS_ENDPOINT = process.env.COSMOS_ENDPOINT;
const COSMOS_KEY = process.env.COSMOS_KEY;
const DATABASE = process.env.COSMOS_DATABASE || 'devglobe';
const CONTAINER = process.env.COSMOS_CONTAINER || 'developers';
let cosmosClient;

function getContainer() {
cosmosClient ||= new CosmosClient({ endpoint: COSMOS_ENDPOINT, key: COSMOS_KEY });
return cosmosClient.database(DATABASE).container(CONTAINER);
}

async function getSampleData() {
const filePath = path.join(process.cwd(), 'data', 'developers-sample.json');
Expand Down Expand Up @@ -51,7 +44,8 @@ export async function GET(request) {
}

try {
const container = getContainer();
const container = getCosmosContainer(CONTAINER);
if (!container) throw new Error('Cosmos DB is not configured');
const fields = 'c.id, c.login, c.name, c.avatarUrl, c.location, c.lat, c.lng, c.followers, c.publicRepos, c.totalStars, c.totalForks, c.totalCommits, c.topLanguage, c.soUserId, c.soReputation, c.soAnswers, c.soBadges, c.score, c.specialTags, c.claimed, c.metricsUpdatedAt, c.aiProfile';
const baseQuery = `SELECT ${fields} FROM c WHERE (NOT IS_DEFINED(c.nomination) OR c.nomination.status = 'approved') ORDER BY c.score DESC`;

Expand Down
7 changes: 3 additions & 4 deletions app/api/search/route.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { CosmosClient } from '@azure/cosmos';
import { NextResponse } from 'next/server.js';
import { promises as fs } from 'fs';
import path from 'path';
Expand All @@ -9,6 +8,7 @@ import {
tokenizeDeveloperSearchQuery,
} from '../../../lib/developer-search.js';
import { attachSearchMatches } from '../../../lib/search-match.js';
import { getCosmosContainer } from '../../../lib/cosmos.js';

const COSMOS_ENDPOINT = process.env.COSMOS_ENDPOINT;
const COSMOS_KEY = process.env.COSMOS_KEY;
Expand All @@ -21,7 +21,6 @@ const OPENAI_CONFIGURED = Boolean(
&& !OPENAI_ENDPOINT.includes('your-resource.openai.azure.com')
);

const DATABASE = process.env.COSMOS_DATABASE || 'devglobe';
const CONTAINER = process.env.COSMOS_CONTAINER || 'developers';

// Excludes pending/rejected self-nominations from every search mode. Legacy
Expand Down Expand Up @@ -150,8 +149,8 @@ export async function GET(request) {
}

try {
const client = new CosmosClient({ endpoint: COSMOS_ENDPOINT, key: COSMOS_KEY });
const container = client.database(DATABASE).container(CONTAINER);
const container = getCosmosContainer(CONTAINER);
if (!container) throw new Error('Cosmos DB is not configured');
let results;
let resolvedMode = mode;
let fallback = null;
Expand Down
9 changes: 4 additions & 5 deletions app/api/trending/route.js
Original file line number Diff line number Diff line change
@@ -1,17 +1,16 @@
import { CosmosClient } from '@azure/cosmos';
import { NextResponse } from 'next/server';
import { promises as fs } from 'fs';
import path from 'path';
import { withNumericScore } from '../../../lib/developer-score.js';
import { addDeveloperRanks } from '../../../lib/ranking.js';
import { listLatestSnapshotsOnOrBeforeDay } from '../../../lib/impact-history-store.js';
import { buildTrending, windowStartDay } from '../../../lib/trending.js';
import { getCosmosContainer } from '../../../lib/cosmos.js';

const COSMOS_ENDPOINT = process.env.COSMOS_ENDPOINT;
const COSMOS_KEY = process.env.COSMOS_KEY;
const DATABASE = process.env.COSMOS_DATABASE || 'devglobe';
const CONTAINER = process.env.COSMOS_CONTAINER || 'developers';
const TRENDING_CACHE_MS = 10 * 60 * 1000;
const TRENDING_CACHE_MS = 60 * 60 * 1000;
const TRENDING_QUERY_TIMEOUT_MS = 5000;
const trendingCache = new Map();

Expand All @@ -33,8 +32,8 @@ async function loadDevelopers() {
return rankDevelopers(await getSampleData());
}
try {
const client = new CosmosClient({ endpoint: COSMOS_ENDPOINT, key: COSMOS_KEY });
const container = client.database(DATABASE).container(CONTAINER);
const container = getCosmosContainer(CONTAINER);
if (!container) return rankDevelopers(await getSampleData());
const fields = 'c.login, c.name, c.avatarUrl, c.location, c.topLanguage, c.score';
const { resources } = await container.items.query({
query: `SELECT ${fields} FROM c WHERE (NOT IS_DEFINED(c.nomination) OR c.nomination.status = 'approved') ORDER BY c.score DESC`,
Expand Down
36 changes: 36 additions & 0 deletions docs/azure-cost-review.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# Azure cost review

Reviewed on 2026-09-29. Figures below are configuration and utilization facts, not a billing forecast.

## Implemented in this branch

- Azure Container Apps scales `devglobe-web` to zero when idle. The app remains at 0.5 vCPU and 1 GiB because observed memory peaked at 72%, making a 0.5 GiB limit unsafe.
- The deployment re-applies `--min-replicas 0`, preventing later image deployments from restoring an always-on replica.
- A manual, confirmation-gated workflow migrates the `devglobe` Cosmos database from 4,000 RU/s manual shared throughput to autoscale with a 4,000 RU/s maximum.
- The same workflow sets `impact-history` retention to 120 days. Product views use at most 90 days, leaving a 30-day operational buffer.
- Public read routes reuse a singleton Cosmos client. Country statistics and trending results use one-hour in-process caches, and public search responses advertise a five-minute shared cache policy.

## Verified production baseline

| Resource | Current state | Finding |
| --- | --- | --- |
| Cosmos DB | 4,000 RU/s manual shared throughput | Advisor recommends autoscale. Average normalized RU consumption was 14.25%; peak reached 100%. Autoscale retains the 4,000 RU/s peak while reducing idle provisioning toward 400 RU/s. |
| Cosmos data | About 1.24 GB | `impact-history` holds about 956k documents and roughly 1.0 GB. Retention prevents unbounded growth; storage itself is not the main bill. |
| Container App | Consumption, 0.5 vCPU, 1 GiB, min 1, max 3 | Average CPU was 6.06%, average memory 29.25%, and average replicas 1.01. Scale-to-zero is appropriate for the observed low traffic, with cold-start latency as the tradeoff. |
| Container Registry | Basic | Already the lowest paid ACR tier. |
| Log Analytics | PerGB2018, 30-day retention | Retention is already conservative. Keep it unless ingestion cost becomes material. |
| Function plans | Y1 Consumption | No fixed plan charge; migrate to Flex Consumption only as a separate reliability/runtime project. |

## Applying the one-time Cosmos change

Run the **Apply Azure cost optimizations** workflow manually and enter `APPLY-COST-OPTIMIZATION` exactly. The GitHub OIDC identity must have Cosmos DB control-plane permission in subscription `5ba12f7d-8235-4c6a-857c-2dc8e4fcb50a` and resource group `rg-devglobe`.

The workflow refuses to migrate if throughput no longer matches the reviewed 4,000 RU/s manual baseline. It is safe to rerun after migration and always verifies the final autoscale maximum and container TTL.

## Follow-up after seven days

1. Compare hourly autoscale maximum billing and normalized RU consumption. If sustained peaks remain below 1,000 RU/s, evaluate a lower autoscale maximum after load testing.
2. Measure cold-start latency and error rates. Restore min replicas to 1 only if first-request latency harms real users.
3. Confirm `impact-history` expiry and document count stabilize after 120 days.
4. Review the stopped `devglobe-activity-ingest` and `devglobe-activity-timer` apps. Delete them only after confirming no storage, DNS, or deployment dependency remains.
5. Consider moving container images to an existing registry only if eliminating the Basic ACR fee outweighs migration and credential-management complexity.
10 changes: 8 additions & 2 deletions lib/cosmos.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { CosmosClient } from '@azure/cosmos';

const clients = new Map();
const containers = new Map();

export function getCosmosContainer(containerName) {
Expand All @@ -16,10 +17,15 @@ export function getCosmosContainer(containerName) {

const database = process.env.COSMOS_DATABASE || 'devglobe';
const container = containerName || process.env.COSMOS_CONTAINER || 'developers';
const cacheKey = `${endpoint}|${database}|${container}`;
const clientKey = `${endpoint}|${key}`;
const cacheKey = `${clientKey}|${database}|${container}`;
if (containers.has(cacheKey)) return containers.get(cacheKey);

const client = new CosmosClient({ endpoint, key });
let client = clients.get(clientKey);
if (!client) {
client = new CosmosClient({ endpoint, key });
clients.set(clientKey, client);
}
const cosmosContainer = client.database(database).container(container);
containers.set(cacheKey, cosmosContainer);
return cosmosContainer;
Expand Down
6 changes: 3 additions & 3 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading