Optimized GPT-style Language Model Engine — Initial Setup → main

.github/workflows/ci.yml

@@ -0,0 +1,110 @@
+name: CI
+
+
+on:
+  # PRs into main are the merge gate (TDR-014). Pushes to any branch also run,
+  # so breakage on Linux or an older Python is found while the work is in
+  # progress rather than at PR time -- this workflow sat unexecuted for its
+  # whole life because nothing had been pushed to main or opened as a PR.
+  push:
+    branches: ["**"]
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  # Floor is 3.12 (see the comment on requires-python in pyproject.toml).
+  PYTHON_VERSION: "3.12"
+
+jobs:
+  lint:
+    name: Lint (ruff)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install ruff
+        run: pip install ruff
+      - name: Ruff check
+        run: ruff check .
+      - name: Ruff format (check only)
+        run: ruff format --check .
+
+  typecheck:
+    name: Type check (mypy)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+      - name: Install package + type stubs
+        run: pip install -e ".[dev]"
+      - name: mypy
+        run: mypy tokenizer dataset model attention training inference optimization benchmark
+
+  test:
+    name: Tests (pytest, py${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # 3.14 is the version this project is developed on; 3.12 is the floor.
+        python-version: ["3.12", "3.13", "3.14"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+      - name: Install package
+        run: pip install -e ".[dev]"
+      - name: Run tests with coverage
+        run: pytest --cov=tokenizer --cov=dataset --cov=model --cov=attention --cov=training --cov=inference --cov=optimization --cov=benchmark --cov-report=term-missing
+
+  import-smoke:
+    name: Import smoke
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+      - name: Install package
+        run: pip install -e .
+      - name: Modules import cleanly
+        run: |
+          python -c "import tokenizer, dataset, model, attention, training, inference, optimization, benchmark; print('all modules import OK')"
+          python -c "from tokenizer import BPE; t=BPE(vocab_size=256); ids=t.encode('hello'); assert t.decode(ids)=='hello'; print('tokenizer smoke OK')"
+          python -c "
+          import torch
+          from attention import CausalAttention
+          from dataset import DataEngine
+          e = DataEngine.from_config({'dataset': {'source': 'synthetic', 'seq_len': 16, 'batch_size': 2}}, vocab_size=64)
+          b = e.next_batch()
+          assert b['input_ids'].shape == (2, 16)
+          a = CausalAttention(d_model=32, n_heads=4, n_kv_heads=2, max_seq_len=64).eval()
+          out, cache = a(torch.randn(2, 16, 32), use_cache=True)
+          assert out.shape == (2, 16, 32) and len(cache) == 16
+          print('dataset + attention smoke OK')
+          "
+      - name: Training entry point runs
+        # Bounded deliberately. This step is a wiring check, not a training run:
+        # the shipped config is 2000 steps of a 32.5M model, which on a CPU
+        # runner is ~1.8 minutes and writes ~1.8 GB of checkpoints.
+        run: |
+          python -m training.train --config configs/default.yaml \
+            dataset.batch_size=2 dataset.seq_len=32 \
+            training.steps=5 training.warmup_steps=1 \
+            training.ckpt_every=0 training.log_every=1 \
+            training.out_dir="${RUNNER_TEMP:-/tmp}/ci-checkpoints"

.gitignore

@@ -0,0 +1,125 @@
+# ---------------------------------------------------------------------------
+# Python
+# ---------------------------------------------------------------------------
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# ---------------------------------------------------------------------------
+# Testing / Linting / Type-checking caches
+# ---------------------------------------------------------------------------
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+.coverage
+.coverage.*
+htmlcov/
+.tox/
+.nox/
+.hypothesis/
+nosetests.xml
+coverage.xml
+
+# ---------------------------------------------------------------------------
+# Virtual environments / package managers
+# ---------------------------------------------------------------------------
+.venv/
+venv/
+env/
+ENV/
+env.bak/
+venv.bak/
+.conda/
+.python-version
+poetry.lock.persona
+uv.lock.persona
+
+# ---------------------------------------------------------------------------
+# Model checkpoints / training artifacts (large binaries, never commit)
+# ---------------------------------------------------------------------------
+checkpoints/*
+!checkpoints/.gitkeep
+runs/
+logs/
+wandb/
+*.pt
+*.pth
+*.ckpt
+*.safetensors
+*.bin
+*.onnx
+
+# ---------------------------------------------------------------------------
+# Data files / datasets (large, regenerable, or licensed)
+# ---------------------------------------------------------------------------
+dataset/data/
+data/
+*.npy
+*.npz
+*.memmap
+*.h5
+*.hdf5
+*.parquet
+*.arrow
+
+# ---------------------------------------------------------------------------
+# Raw / source text dumps
+# ---------------------------------------------------------------------------
+arxiv.txt
+
+# ---------------------------------------------------------------------------
+# Secrets & environment files (NEVER commit API keys / tokens)
+# ---------------------------------------------------------------------------
+.env
+.env.*
+*.env
+.env.local
+.env.production
+secrets/
+credentials/
+*.pem
+*.key
+*.p12
+config/api_keys*.yaml
+
+# ---------------------------------------------------------------------------
+# OS / editor / editor-config files
+# ---------------------------------------------------------------------------
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes/
+ehthumbs.db
+Thumbs.db
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+*.orig
+*.rej
+.vscode/
+
+# ---------------------------------------------------------------------------
+# Notebooks / scratch experiments (work-in-progress, not part of the package)
+# ---------------------------------------------------------------------------
+test_agent.py