diff --git a/.cspell.yml b/.cspell.yml index c491cfc..ae5b664 100644 --- a/.cspell.yml +++ b/.cspell.yml @@ -33,4 +33,6 @@ words: - regexs - rels - seborama + - stretchr + - TESTFLAGS - wjdp diff --git a/Makefile b/Makefile index 2b004b7..46d5c07 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,8 @@ # cSpell:ignore TESTFLAGS coverprofile gopath golangci ldflags covermode coverpkg gofmt benchmem .PHONY: build build-verify install run -.PHONY: test test-race test-coverage test-ci test-bench -.PHONY: test-tdd test-tdd-cache clean-cache refcache-check _test-tdd +.PHONY: test test-fast test-race test-coverage test-ci test-bench +.PHONY: test-tdd test-tdd-fast test-tdd-cache clean-cache refcache-check _test-tdd .PHONY: lint fmt fmt-check vet check ci .PHONY: clean deps help @@ -48,18 +48,24 @@ build-verify: build ## test: Run all tests (use TESTFLAGS to pass additional flags, e.g., make test TESTFLAGS="-v") test: - @echo "Running tests..." - @go test $(TESTFLAGS) ./... + @echo "Running ALL tests (includes slow and external tests)..." + @go test ./... $(TESTFLAGS) + +## test-fast: Run all tests but skip slow ones (timeout waits) +test-fast: + @echo "Running tests (skipping slow tests)..." + @go test ./htmldoc ./issues ./refcache + @go test ./htmltest -skip-slow=true ## test-race: Run tests with race detector (recommended) test-race: - @echo "Running tests with race detector..." + @echo "Running tests with race detector (includes slow and external tests)..." @go test -v -race ./... ## test-coverage: Generate and display test coverage test-coverage: - @echo "Generating coverage report..." - @go test $(TESTFLAGS) -coverprofile=coverage.txt ./... + @echo "Generating coverage report (includes slow and external tests)..." + @go test -coverprofile=coverage.txt ./... $(TESTFLAGS) @go tool cover -func=coverage.txt @echo "" @echo "To view HTML coverage report, run: go tool cover -html=coverage.txt" @@ -84,16 +90,21 @@ clean-cache: ## test-tdd: TDD mode - run specific test(s) with clean cache (use TEST_RUN to match tests). ## Examples: -## make test-tdd TEST_RUN=TestTimeoutIsCached # Run single test with clean cache -## make test-tdd TEST_RUN='.*Cache.*' # Run all cache tests -## make test-tdd TEST_RUN=TestTimeout # Run tests matching pattern -## make test-clean TEST_RUN=TestTimeout # Same but without cache inspection -## make clean-cache # Just clean the cache +## make test-tdd TEST_RUN=TestTimeoutIsCached # Run single test +## make test-tdd TEST_RUN='.*Cache.*' # Run all cache tests +## make test-tdd-fast TEST_RUN='.*Cache.*' # Fast mode (skip slow tests) +## make test-tdd TEST_RUN='.*Cache.*' TESTFLAGS=-skip-slow=true # Same as -fast +## make test-tdd-cache TEST_RUN=TestTimeout # With cache inspection +## make clean-cache # Just clean the cache test-tdd: clean-cache _test-tdd +## test-tdd-fast: Fast TDD mode - same as test-tdd but skips slow tests +test-tdd-fast: + @$(MAKE) test-tdd TESTFLAGS=-skip-slow=true TEST_RUN="$(TEST_RUN)" TEST_PKG="$(TEST_PKG)" + _test-tdd: @echo "TDD mode: Running $(TEST_RUN) in $(TEST_PKG)..." - @go test -v -run $(TEST_RUN) $(TEST_PKG) || true + @go test -v -run $(TEST_RUN) $(TEST_PKG) $(TESTFLAGS) || true @echo "" @echo "Test completed." diff --git a/README.md b/README.md index 431c869..ede4d97 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,7 @@ htmltest uses a YAML configuration file. Put `.htmltest.yml` in the same directo | `IgnoreSSLVerify` | Turns off x509 errors for self-signed certificates. | `false` | | `IgnoreTagAttribute` | Specify the ignore attribute. All tags with this attribute or with this class will be excluded from every check. | `"data-proofer-ignore"` | | `RetryCachedErrors` | By default, links that timeout or with a previous non-ok status are retried. When false, timeouts are cached and such links are not retried. This speeds up repeated test runs with known broken links. | `true` | +| `CacheAllExternal` | Cache all external links: timeouts when `CheckExternal: true`, discovered links when `CheckExternal: false`. Enables fast re-runs and link discovery mode. | `false` | | `HTTPHeaders` | Dictionary of headers to include in external requests | `{"Range": "bytes=0-0", "Accept": "*/*"}` | | `TestFilesConcurrently` | :warning: :construction: *EXPERIMENTAL* Turns on [concurrent](https://github.com/wjdp/htmltest/wiki/Concurrency) checking of files. | `false` | | `DocumentConcurrencyLimit` | Maximum number of documents to process at once. | `128` | diff --git a/docs/tasks/cache-unchecked-external-links.md b/docs/tasks/cache-unchecked-external-links.md index 0147265..cf6a11b 100644 --- a/docs/tasks/cache-unchecked-external-links.md +++ b/docs/tasks/cache-unchecked-external-links.md @@ -1,10 +1,24 @@ --- title: CacheAllExternal Feature date: 2025-10-18 -lastmod: 2025-10-18 -status: in-progress +lastmod: 2025-10-19 +status: complete +cSpell:ignore: statuscodes --- +## Status + +- ✅ **Phase 0 Complete**: `RetryCachedErrors` semantics cleaned up, timeout + caching now controlled by `CacheAllExternal` +- ✅ **Test Infrastructure**: Migrated to `testify/assert`, cache helpers, fast + test targets +- ✅ **Phase 1 Complete**: Discovery mode + feature interactions verified. All 6 + behavior matrix rows covered by tests. +- ✅ **Phase 2 Complete**: Network, certificate, and generic client errors now + cached +- ✅ **Documentation Complete**: README updated with CacheAllExternal option +- 🎉 **Feature Complete**: CacheAllExternal fully implemented and tested! + # CacheAllExternal Feature ## Use Case @@ -44,30 +58,76 @@ Example workflow: 3. **`CacheAllExternal: true | false`** (NEW) - Whether to cache everything including timeouts and unchecked links +### RetryCachedErrors behavior cleanup + +`RetryCachedErrors` currently conflates two concerns: + +1. Whether to retry cached errors (its intended purpose) +2. Whether to cache timeouts (side effect when set to `false`) + +With `CacheAllExternal`, we can separate these orthogonal concerns: + +- `CacheAllExternal` controls **what gets cached** (just checked results, or + everything including timeouts/unchecked) +- `RetryCachedErrors` controls **retry behavior only** (use cached errors, or + retry them) + +This cleanup must happen BEFORE implementing the discovery mode feature. + ### Complete Behavior Matrix -| CheckExternal | CacheAllExternal | RetryCachedErrors | Links Checked? | Errors Retried? | What Gets Cached | Use Case | -| ------------- | ---------------- | ----------------- | -------------- | --------------- | ----------------- | ------------------------------------ | -| `true` | `false` | `true` | ✓ | ✓ | 200, 4XX | **Default/Legacy** | -| `true` | `false` | `false` | ✓ | ✗ | 200, 4XX | Reuse errors, retry timeouts | -| `true` | `true` | `true` | ✓ | ✓ | 200, 4XX, TSC[^1] | Cache timeouts, but retry | -| `true` | `true` | `false` | ✓ | ✗ | 200, 404, TSC[^1] | **Fast re-runs** - cache & reuse all | -| `false` | `false` | (N/A) | ✗ | N/A | Nothing | **Default skip** - no cache | -| `false` | `true` | (N/A) | ✗ | N/A | unchecked links | **Link discovery** | +| Row | CheckExternal | CacheAllExternal | RetryCachedErrors | Links Checked? | Errors Retried? | What Gets Cached | Use Case | +| --- | ------------- | ---------------- | ----------------- | -------------- | --------------- | ------------------- | ------------------------------------ | +| 1 | `true` | `false` | `true` | ✓ | ✓ | 200, 4XX | **Default/Legacy** | +| 2 | `true` | `false` | `false` | ✓ | ✗ | 200, 4XX | Failed links are not retried | +| 3 | `true` | `true` | `true` | ✓ | ✓ | 200, 4XX, TSC[^TSC] | Cache timeouts, but retry | +| 4 | `true` | `true` | `false` | ✓ | ✗ | 200, 4XX, TSC[^TSC] | **Fast re-runs** - cache & reuse all | +| 5 | `false` | `false` | (N/A) | ✗ | N/A | Nothing | **Default skip** - no cache | +| 6 | `false` | `true` | (N/A) | ✗ | N/A | unchecked links | **Link discovery** | -[^1]: +[^TSC]: TSC = Tool-specific status code used by htmltest. See `@docs/tasks/design.md` for details. ### Key Insights - `CacheAllExternal` extends caching behavior in **both** modes: - - When `CheckExternal: true` → Also caches timeouts (as `StatusTimeout`) + - When `CheckExternal: true` → Caches **all tool-specific errors** (timeouts, + network failures, etc.) - When `CheckExternal: false` → Caches discovered links (as `StatusUnchecked`) +- `RetryCachedErrors` now has clean, single-purpose semantics (retry vs. reuse + cached errors) - When `CheckExternal: false`, `RetryCachedErrors` has no effect (nothing to retry) - See `@docs/tasks/design.md` for status code details +### Tool-Specific Errors to Cache + +When `CacheAllExternal: true` and `CheckExternal: true`, **all** non-HTTP errors +should be cached with appropriate status codes: + +1. **Timeout errors** (`StatusTimeout = -10`): + - Request exceeds `ExternalTimeout` setting + - Currently: ✅ Already implemented in Phase 0 + +2. **DNS/Network errors** (`StatusNetworkError = -20`): + - "dial tcp" failures + - DNS lookup failures + - Connection refused + - Currently: ❌ Not cached (returns early without caching) + +3. **Certificate errors** (`StatusCertError = -30`): + - x509.UnknownAuthorityError + - Invalid/expired certificates + - Incomplete certificate chains + - Currently: ❌ Not cached (returns early without caching) + +4. **Generic client errors** (`StatusClientError = -40`): + - Other unhandled HTTP client errors + - Currently: ❌ Not cached (returns early without caching) + +**Note**: HTTP status codes (200, 404, etc.) are already cached in all modes. + ## Implementation Steps (TDD Approach) **Incremental TDD**: Write one test at a time, implement, verify, then move to @@ -84,21 +144,40 @@ For each behavior in the matrix below: ### Test Order & Behaviors -| # | Behavior to Test | Config | Expected Result | Test Name | -| --- | ------------------- | -------------------------------------------------- | -------------------------------- | --------------------------------- | -| 1 | Default: no caching | `CheckExternal: false`, `CacheAllExternal: false` | Nothing cached | `TestCacheAllExternalDisabled` | -| 2 | **Discovery mode** | `CheckExternal: false`, `CacheAllExternal: true` | Cache with `StatusUnchecked` | `TestCacheAllExternalDiscovery` | -| 3 | **Timeout caching** | `CheckExternal: true`, `CacheAllExternal: true` | Cache timeout as `StatusTimeout` | `TestCacheAllExternalTimeout` | -| 4 | Ignored URLs | `CacheAllExternal: true`, `IgnoreURLs: [pattern]` | Ignored URLs NOT cached | `TestCacheAllExternalIgnored` | -| 5 | Query stripping | `CacheAllExternal: true`, `StripQueryString: true` | Query stripped | `TestCacheAllExternalQueryString` | +Phase 1 focuses on implementing **Discovery Mode** (rows 5-6 from Complete +Behavior Matrix). Timeout caching (rows 3-4) was already implemented in Phase 0. + +| Test# | Incr | Matrix Row | Behavior to Test | CheckExternal-related config[^Config] | Expected Result | Test Name | Status | +| ----- | ---- | ---------- | ------------------------------ | ------------------------------------------------ | ---------------------------- | ------------------------------------------ | ------------ | +| — | — | Row 1 | Default/Legacy | Check: true, All: false, Retry: true | 200, 4XX cached & retried | `TestExternalErrorCachedRetried` | ✅ Existing | +| — | — | Row 2 | Errors not retried | Check: true, All: false, Retry: false | 200, 4XX cached & reused | `TestExternalBrokenRetryCachedErrorsDisabled` | ✅ Existing | +| — | — | Row 3 | Timeout cached & retried | Check: true, All: true, Retry: true | Timeout cached but retried | `TestTimeoutCachedButRetried` | ✅ Inc 4 | +| — | — | Row 4 | Timeout cached & reused | Check: true, All: true, Retry: false | Timeout cached & reused | `TestTimeoutCachedReused` | ✅ Phase 0 | +| 1 | 0 | Row 5 | Default: no caching | Check: false, All: false | Nothing cached | `TestCacheAllExternalDisabled` | ✅ Inc 0 | +| 2 | 1 | Row 6 | **Discovery mode** | Check: false, All: true | Cache with `StatusUnchecked` | `TestCacheAllExternalDiscovery` | ✅ Inc 1 | +| 3 | 2 | — | IgnoreURLs interaction | Check: false, All: true, IgnoreURLs: `[pattern]` | Ignored URLs NOT cached | `TestCacheAllExternalAndIgnoredURLs` | ✅ Inc 2 | +| 4 | 3 | — | StripQueryString interaction | Check: false, All: true, StripQueryString: true | Query stripped before cache | `TestCacheAllExternalQueryString` | ✅ Inc 3 | + +[^Config]: + Abbreviations: `Check` = `CheckExternal`, `All` = `CacheAllExternal`, + `Retry` = `RetryCachedErrors`. -**Commands**: Use `make test-tdd TEST_RUN=` or -`make test-tdd-cache TEST_RUN=` +**Commands**: +- `make test-tdd-fast TEST_RUN=` - Fast TDD (skip slow tests, ~1.5s) +- `make test-tdd TEST_RUN=` - Full TDD (includes slow tests, ~7.8s) +- `make test-tdd-cache TEST_RUN=` - With cache inspection + +**Note**: Phase 1 focuses on discovery mode (row 6) and its edge cases. Rows 1-4 +from the Complete Behavior Matrix are already covered by existing tests. ### 1. Write Tests (One at a Time) **File**: `htmltest/check-link-cache_test.go` +**Best Practice**: Use `tTestFileOptsFromCleanOutputDir()` for the first test +call in each test function to ensure test isolation with a clean cache +directory. Use `tTestFileOpts()` for subsequent calls within the same test. + ### 2. Add Configuration Option **File**: `htmltest/options.go` @@ -160,10 +239,238 @@ Two modifications needed: - **No need for backward compat with dev/main features**: We can change `RetryCachedErrors` behavior if needed +## Incremental Implementation Strategy + +### Phase 0: Cleanup `RetryCachedErrors` Semantics ✅ COMPLETE + +**Purpose**: Separate concerns before adding new functionality + +**Step 0.1**: Write test for new semantics + +- ✅ Created `TestTimeoutNotCachedWithRetryCacheErrorsOnly` - verifies timeouts + are NOT cached when only `RetryCachedErrors: false` (without + `CacheAllExternal`) +- ✅ Test run: **GREEN** (existing code already has correct behavior for this + case) + +**Step 0.2**: Update existing test to expect new behavior (RED) + +- ✅ Updated `TestTimeoutIsCached` to use `CacheAllExternal: true` instead of + `RetryCachedErrors: false` +- ✅ Test cannot run yet (references non-existent option) + +**Step 0.3**: Change implementation to use new option (RED) + +- ✅ Changed `check-link.go` timeout caching condition from + `if !hT.opts.RetryCachedErrors` to `if hT.opts.CacheAllExternal` +- ✅ Result: **Compilation error** - option doesn't exist yet + +**Step 0.4**: Add option to make code compile (GREEN) + +- ✅ Added `CacheAllExternal bool` to `Options` struct (`htmltest/options.go`) +- ✅ Added default value `"CacheAllExternal": false` in `DefaultOptions()` +- ✅ Test run: **GREEN** - code compiles and test passes + +**Step 0.5**: Update remaining timeout tests + +- ✅ Updated `TestTimeoutCachedReused` to use `CacheAllExternal: true` +- ✅ Updated `TestTimeoutCachedMessage` to use `CacheAllExternal: true` +- ✅ All cache tests: **GREEN** + +**Step 0.6**: Add sanity check test + +- ✅ Created `TestCacheOptions` to verify default option values +- ✅ Test run: **GREEN** +- ✅ Full test suite: All packages pass + +**Result**: Clean semantics established, timeout caching now controlled by +`CacheAllExternal` (not `RetryCachedErrors`), all tests passing, no regression + +**Key TDD Insight**: We changed the implementation BEFORE the option existed, +causing a compilation error (deep RED), then added just enough to compile and +pass (GREEN) + +**Infrastructure Improvements**: + +1. **Testify migration**: + - ✅ Migrated `check-link-cache_test.go` to + `github.com/stretchr/testify/assert` + - ✅ Benefits: Industry-standard library, better error messages, active + maintenance + - ✅ Cleaner assertion syntax: `assert.Equal(t, expected, actual)` + +2. **Cache assertion helpers** (`test_helpers_extra_test.go`): + - ✅ `tExpectCached(t, hT, url, statusCode...)` - Assert URL is cached + (optionally with status) + - ✅ `tExpectNotCached(t, hT, url)` - Assert URL is not cached + - ✅ Reduces boilerplate from ~7 lines to 1 line per assertion + +3. **Slow test control**: + - ✅ Added `tSkipSlow(t)` helper with `-skip-slow` flag + - ✅ Applied to 4 cache timeout tests (wait 1s each) + - ✅ Updated Makefile with `TESTFLAGS` support + - ✅ Added fast test targets: + - `make test-fast` - All packages, skip slow tests (~7s) + - `make test-tdd-fast TEST_RUN='...'` - TDD mode, skip slow tests (~1.5s) + - ✅ Result: 76% faster cache test runs (1.5s vs 7.8s) + +4. **Test refactoring** (DRY patterns): + - ✅ Extract `fixture` variables for paths + - ✅ Extract `opts` maps and reuse across test runs + - ✅ Consistent test structure across all cache tests + +--- + +### Phase 1: Discovery Mode Feature + +**Scope**: Implement `StatusUnchecked` caching when `CheckExternal: false` and +`CacheAllExternal: true`. + +**Note**: Phase 1 focuses on discovery mode only. Caching additional tool-specific +errors (network failures, cert errors) will be addressed in future phases. + +#### Confirm default behavior + +Confirm that we test for the expected default behavior: + +- ✅ Test already exists: `TestCacheAllExternalDisabled` +- ✅ Verifies `CacheAllExternal: false` doesn't cache discovered links +- ✅ Test: PASS (baseline behavior confirmed) +- ✅ No work needed: Baseline already covered + +#### Increment 1: Test #2 - Discovery Mode (Core Feature) ✅ + +**Purpose**: Implement row 6 - the main use case + +**Actual TDD Steps**: + +- ✅ Wrote `TestCacheAllExternalDiscovery` test +- ✅ Test run: **RED** (URL not cached, feature doesn't exist) +- ✅ Fixed `tExpectCached` helper to avoid panic on failed assertion +- ✅ Implemented discovery mode in `checkExternal()`: + - Early return when `CheckExternal: false && CacheAllExternal: false` + - Fall through to URL processing when `CacheAllExternal: true` + - Cache with `StatusUnchecked` and return +- ✅ Test run: **GREEN** +- ✅ Refactored: Moved URL processing before discovery mode check (DRY) +- ✅ Added invariant check to document control flow assumption +- ✅ All cache tests: **GREEN** + +**Result**: Core discovery mode feature working! External links cached with +`StatusUnchecked` when `CheckExternal: false` and `CacheAllExternal: true`. + +#### Increment 2: Test #5 - IgnoreURLs Feature Interaction ✅ + +**Purpose**: Verify CacheAllExternal respects IgnoreURLs patterns + +- ✅ Wrote `TestCacheAllExternalAndIgnoredURLs` +- ✅ Created `tExpectCacheEmpty` helper (more robust than checking specific URL) +- ✅ Test run: **GREEN** (URL processing order is correct - ignored URLs filtered + before caching) +- ✅ No code changes needed +- ✅ Updated `TestCacheAllExternalDisabled` and + `TestTimeoutNotCachedWithRetryCacheErrorsOnly` to use `tExpectCacheEmpty` + +**Result**: Feature interaction verified - ignored URLs correctly excluded from +cache + +#### Increment 3: Test #6 - StripQueryString Feature Interaction ✅ + +**Purpose**: Verify CacheAllExternal works with query string stripping + +- ✅ Wrote `TestCacheAllExternalQueryString` using `check_just_once.html` fixture +- ✅ Tests `github.com` URLs (not in default `StripQueryExcludes`) +- ✅ Test run: **GREEN** (query stripping happens before caching) +- ✅ No code changes needed +- ✅ Verifies: Multiple URLs with different query params → single cached entry + +**Result**: Query string stripping correctly applied before caching in discovery +mode + +#### Increment 4: Verify Row 3 & Row 4 Coverage ✅ + +**Purpose**: Ensure timeout+retry combinations work + +- ✅ Row 4 already covered: `TestTimeoutCachedReused` (All: true, Retry: false) +- ✅ Row 3 needed explicit test: Wrote `TestTimeoutCachedButRetried` +- ✅ Test verifies: Timeout cached (All: true) but retried on second run (Retry: + true, default) +- ✅ Test run: **GREEN** (behavior already works correctly) +- ✅ Updated table to show all 6 matrix rows covered + +**Result**: Complete behavior matrix coverage verified. All 6 combinations of +CheckExternal, CacheAllExternal, and RetryCachedErrors have test coverage. + +**Rationale for order**: Clean up semantics first (Phase 0), then add +infrastructure and core discovery feature (Phase 1), validate assumptions while +fresh, verify complete matrix coverage. + ## To-dos -- [ ] Write test cases for CacheAllExternal feature (TDD) -- [ ] Add CacheAllExternal field to Options struct and DefaultOptions() -- [ ] Implement discovery mode caching (CheckExternal: false) -- [ ] Implement timeout caching (CheckExternal: true) -- [ ] Add CacheAllExternal to README configuration table +### Phase 0: Cleanup + +- [x] Step 0.1-0.6: All steps complete ✅ + +### Phase 1: Discovery Mode + +- [x] Increment 0: Baseline test (default behavior - row 5) ✅ +- [x] Increment 1: Discovery mode test + implementation (row 6) ✅ +- [x] Increment 2: IgnoreURLs feature interaction ✅ +- [x] Increment 3: StripQueryString feature interaction ✅ +- [x] Increment 4: Verify row 3 & 4 coverage ✅ + +### Documentation + +- [x] Update README configuration table ✅ + +## Future Work + +### Phase 2: Tool-Specific Error Caching (Future) + +**Scope**: Extend `CacheAllExternal` to cache all tool-specific errors (beyond +timeouts) when `CheckExternal: true` + +**Currently**: Only `StatusTimeout` is cached (Phase 0) + +**Goal**: Cache all error types that prevent successful HTTP responses, enabling +truly offline re-runs with `CacheAllExternal: true` and `RetryCachedErrors: +false`. + +#### Error Types to Add: + +**1. Network errors** (`StatusNetworkError = -20`): +- ✅ DNS lookup failures ("no such host") +- ✅ Connection refused +- ✅ Network unreachable +- **Tests**: `TestNetworkErrorCached`, `TestNetworkErrorCachedReused`, + `TestNetworkErrorDNS` +- **Implementation**: Cached when `CacheAllExternal: true` (lines 242-255 in + `check-link.go`) + +**2. Certificate errors** (`StatusCertError = -30`): +- ✅ x509 certificate errors (expired, untrusted, hostname mismatch, etc.) +- ✅ Detection via string check for "x509:" in error message +- **Tests**: `TestCertErrorCached`, `TestCertErrorCachedReused` (using + `expired.badssl.com`) +- **Fixture**: `link_expired_cert.html` +- **Implementation**: Status code determined before caching to avoid control flow + changes (lines 258-266 in `check-link.go`) + +**3. Generic client errors** (`StatusClientError = -40`): +- ✅ Catch-all for other unhandled HTTP client errors +- **Tests**: Skipped (rare edge case, no reliable test fixture) +- **Implementation**: Default status code if not network/cert error (lines + 258-266 in `check-link.go`) + +#### Implementation Approach: + +- Determine status code **before** caching to minimize control flow changes +- Use string matching for error type detection ("dial tcp" for network, "x509:" + for certs) +- Maintain existing error reporting behavior + +**Benefits**: Complete "fast re-runs" feature - no network calls at all when all +errors are cached. + +**Status**: ✅ Complete - All tool-specific errors now cached when +`CacheAllExternal: true` diff --git a/docs/tasks/summary.md b/docs/tasks/summary.md index 268ffae..a994efb 100644 --- a/docs/tasks/summary.md +++ b/docs/tasks/summary.md @@ -1,7 +1,7 @@ --- title: Summary of Changes on dev/main Branch date: 2025-10-18 -lastmod: 2025-10-18 +lastmod: 2025-10-19 status: active --- @@ -12,99 +12,88 @@ This document summarizes the changes made on the `dev/main` branch relative to ## Features -So far all features are related to external link checking, and whether such -links are cached and/or retried on subsequent encounters. +All features enhance external link checking and caching behavior. -### 1. RetryCachedErrors +### 1. CacheAllExternal -This Introduced the `RetryCachedErrors` configuration option. When set to false, -This feature allows htmltest to reuse cached error status codes (e.g., 404) -without retrying them on subsequent runs. +Cache all external links, including timeouts and tool-specific errors, for fast +re-runs and link discovery mode. -- **Config parameter**: `RetryCachedErrors` (boolean, default: `true`) -- **Behavior**: When set to `false`, cached errors are reused instead of - retrying -- **Backward compatibility**: Default `true` maintains existing behavior - -### 2. Timeout Caching - -**Status**: Completed - -Extended caching behavior to save timeout errors to the refcache when -`RetryCachedErrors: false`. - -- Timeouts are cached as status code `-10` (tool-specific timeout code) -- Prevents repeated timeout attempts on unreachable URLs -- Works in conjunction with `RetryCachedErrors` option +- **Config parameter**: `CacheAllExternal` (boolean, default: `false`) +- **Discovery mode**: When `CheckExternal: false`, discovered links are cached + with `StatusUnchecked = 0` +- **Error caching**: When `CheckExternal: true`, caches timeouts, network + errors, certificate errors, and generic client errors +- **Status codes**: + - `StatusUnchecked = 0` - Discovered but not checked + - `StatusTimeout = -10` - Timeout errors + - `StatusNetworkError = -20` - DNS/connection failures + - `StatusCertError = -30` - Certificate errors (expired, untrusted, etc.) + - `StatusClientError = -40` - Generic client errors +- **Tests**: 16+ tests covering discovery mode, feature interactions, and error + caching +- **Fixtures**: `link_expired_cert.html` for cert error testing -### 3. Status Code Migration +See `@docs/tasks/cache-unchecked-external-links.md` for full details. -**Status**: Completed +### 2. RetryCachedErrors -Migrated timeout status codes from `408` to `-10` for clarity. +Control whether cached errors are retried on subsequent runs. -- Created `htmltest/statuscodes.go` with status code constants -- `StatusTimeout = -10` (tool-specific timeout) -- `StatusUnchecked = 0` (for future use) -- Helper functions: `IsHTTPStatus()`, `IsUnchecked()`, `IsToolError()` -- See `@docs/tasks/design.md` for status code conventions - -### 4. URL Encoding in Cache +- **Config parameter**: `RetryCachedErrors` (boolean, default: `true`) +- **Behavior**: When set to `false`, cached errors are reused without retrying +- **Use case**: Fast re-runs using only cached results (combine with + `CacheAllExternal: true`) +- **Backward compatibility**: Default `true` maintains existing behavior -**Status**: Completed +### 3. URL Encoding in Cache Fixed URL escaping in the JSON refcache file. - Disabled HTML escaping when encoding URLs to JSON -- URLs are now stored in their unescaped form in `refcache.json` +- URLs stored in their unescaped form in `refcache.json` - Improves readability and prevents double-escaping issues ## Infrastructure & Tooling -### Version ID Suffix - -Added repository-specific suffix to version IDs for better build tracking. - -### CI/CD Improvements +### Status Code System -- Added `workflow_dispatch` trigger to GitHub Actions CI workflow -- Allows manual workflow runs from GitHub UI +Custom status code system for tool-specific states: -### Project Structure - -- Added `Makefile` for build automation -- Added `CONTRIBUTING.md` with development guidelines - -## Infrastructure & Tooling (continued) +- `htmltest/statuscodes.go` with constants and helpers +- Positive integers: HTTP status codes +- Zero: Unchecked/undiscovered links +- Negative integers: Tool-specific errors (timeout, network, cert, client) +- Helper functions: `IsHTTPStatus()`, `IsUnchecked()`, `IsToolError()` +- See `@docs/tasks/design.md` for conventions ### TDD Makefile Targets -**Status**: Completed - -Added Makefile targets for TDD workflow: +Makefile targets for TDD workflow: - `make test-tdd TEST_RUN=TestName` - Run test with clean cache +- `make test-tdd-fast` - Skip slow tests (timeouts) - `make test-tdd-cache TEST_RUN=TestName` - Run test and show cache state - `make clean-cache` - Remove refcache file - Supports pattern matching for running multiple tests ### Documentation Structure -**Status**: Completed - Organized documentation under `docs/`: -- `docs/ops/` - Operational documentation (session-start, agent-guidance) +- `docs/ops/` - Operational documentation (session-start) - `docs/tasks/` - Task and feature documentation - `AGENTS.md` - AI agent guidance (Cursor auto-loads) - `.github/copilot-instructions.md` - GitHub Copilot support -## Planned Features +### Project Structure -Most features are documented under the `docs/tasks/` directory. +- Added `Makefile` for build automation +- Added `CONTRIBUTING.md` with development guidelines +- Version ID suffix for better build tracking -### CacheAllExternal (In Progress) +### CI/CD -See `@docs/tasks/cache-unchecked-external-links.md` for details. This feature -will allow caching external links without checking them when -`CheckExternal: false`. +- Added `workflow_dispatch` trigger to GitHub Actions +- Updated action versions +- Prevents duplicate runs on PRs diff --git a/go.mod b/go.mod index 15ccfb8..95091e4 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/golangplus/sort v1.0.0 // indirect github.com/imdario/mergo v0.3.11 github.com/seborama/govcr v4.5.0+incompatible // indirect + github.com/stretchr/testify v1.11.1 golang.org/x/net v0.17.0 gopkg.in/seborama/govcr.v4 v4.5.0 gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index 91103b3..c3caa80 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,8 @@ github.com/badoux/checkmail v1.2.1 h1:TzwYx5pnsV6anJweMx2auXdekBwGr/yt1GgalIx9nBQ= github.com/badoux/checkmail v1.2.1/go.mod h1:XroCOBU5zzZJcLvgwU15I+2xXyCdTWXyR9MGfRhBYy0= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/daviddengcn/go-algs v0.0.0-20180330170136-fe23fabd9d06 h1:jEHTltplMBbYsHlSnvZD1J4CsfweUSJIqS8uP56q1Ng= github.com/daviddengcn/go-algs v0.0.0-20180330170136-fe23fabd9d06/go.mod h1:CpyLopUWBmqupyWU6OlSfrzgIuzNq0R6DXzM74O9RMs= github.com/daviddengcn/go-assert v0.0.0-20150305222929-ba7e68aeeff6 h1:OPIYL/VhQiSpoaxIcmeYdghLswBylfk6JDVCUqadXxg= @@ -25,8 +28,19 @@ github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/seborama/govcr v4.5.0+incompatible h1:XvdHtXi0d4cUAn+0aWolvwfS3nmhNC8Z+yMQwn/M64I= github.com/seborama/govcr v4.5.0+incompatible/go.mod h1:EgcISudCCYDLzbiAImJ8i7kk4+wTA44Kp+j4S0LhASI= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -77,3 +91,6 @@ gopkg.in/seborama/govcr.v4 v4.5.0/go.mod h1:GSulKzJ4LIk6XH6sE+OKL76FwDIVQEja2Oh2 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/htmltest/check-link-cache_test.go b/htmltest/check-link-cache_test.go index adfa6bc..2920c60 100644 --- a/htmltest/check-link-cache_test.go +++ b/htmltest/check-link-cache_test.go @@ -3,71 +3,75 @@ package htmltest import ( "testing" - "github.com/wjdp/htmltest/issues" + "github.com/stretchr/testify/assert" ) // Tests for cache-related functionality // Feature: RetryCachedErrors // Added by @chalin +func TestCacheOptions(t *testing.T) { + opts := DefaultOptions() + + // Sanity check: ensure that non-existent option doesn't exist + _, exists := opts["NonExistentOption"] + assert.False(t, exists, "NonExistentOption should not exist") + + // Verify cache-related option defaults + assert.Equal(t, false, opts["CacheAllExternal"], "CacheAllExternal default") + assert.Equal(t, true, opts["RetryCachedErrors"], "RetryCachedErrors default") +} + +// ======================================== +// External Error Caching Tests +// ======================================== + // TestExternalErrorCached : Test that URLs with HTTP error status codes (such // as 404) are saved to the refcache. func TestExternalErrorCached(t *testing.T) { - hT := tTestFileOpts("fixtures/images/imageExternal404.html", + hT := tTestFileOptsFromCleanOutputDir("fixtures/images/imageExternal404.html", map[string]interface{}{"VCREnable": true, "EnableCache": true}) tExpectIssueCount(t, hT, 1) - - // Verify the 404 was saved to cache - cR, ok := hT.refCache.Get("https://upload.wikimedia.org/wikipedia/en/404") - if !ok { - t.Error("expected 404 response to be cached, but it wasn't") - } - // Verify it's actually a 404 - if cR.StatusCode != 404 { - t.Errorf("expected status code 404 in cache, got %d", cR.StatusCode) - } + tExpectCached(t, hT, "https://upload.wikimedia.org/wikipedia/en/404", 404) } // TestExternalErrorCachedRetried : Test that by default cached error responses // (404, etc.) are retried on subsequent runs. func TestExternalErrorCachedRetried(t *testing.T) { + fixture := "fixtures/images/imageExternal404.html" + opts := map[string]interface{}{"VCREnable": true, "EnableCache": true} + // First run: populate cache with 404 - hT := tTestFileOpts("fixtures/images/imageExternal404.html", - map[string]interface{}{"VCREnable": true, "EnableCache": true}) + hT := tTestFileOptsFromCleanOutputDir(fixture, opts) tExpectIssueCount(t, hT, 1) // Second run: WITH VCR - should retry even though 404 is cached (default behavior) - hT2 := tTestFileOpts("fixtures/images/imageExternal404.html", - map[string]interface{}{"VCREnable": true, "EnableCache": true, "LogLevel": issues.LevelDebug}) + hT2 := tTestFileOpts(fixture, opts) tExpectIssueCount(t, hT2, 1) // Verify it did NOT use the cache (should see "fresh" not "from cache") - if hT2.issueStore.MessageMatchCount("from cache") > 0 { - t.Error("expected cached 404 to be retried (should NOT see 'from cache' message by default)") - } - if hT2.issueStore.MessageMatchCount("fresh") == 0 { - t.Error("expected cached 404 to be retried (should see 'fresh' message)") - } + tExpectIssue(t, hT2, "from cache", 0) + tExpectIssue(t, hT2, "fresh", 1) } // TestExternalBrokenRetryCachedErrorsDisabled : Test that URLs with non-OK -// status are not retried when RetryCachedErrors is set to false. -// Uses elements for variety. +// status are not retried when RetryCachedErrors is false. func TestExternalBrokenRetryCachedErrorsDisabled(t *testing.T) { - // First run: populate cache with 404 - hT := tTestFileOpts("fixtures/generic/citeBroken.html", - map[string]interface{}{"VCREnable": true, "EnableCache": true, "RetryCachedErrors": false}) - tExpectIssueCount(t, hT, 4) // 4 broken citations in the fixture + fixture := "fixtures/images/imageExternal404.html" + opts := map[string]interface{}{"VCREnable": true, "EnableCache": true, "RetryCachedErrors": false} - // Second run: WITHOUT VCR - should use cached 404s, not retry (which would fail without VCR) - hT2 := tTestFileOpts("fixtures/generic/citeBroken.html", - map[string]interface{}{"EnableCache": true, "RetryCachedErrors": false, "LogLevel": issues.LevelDebug}) - tExpectIssueCount(t, hT2, 4) + // First run WITH VCR: populate cache with 404 + hT := tTestFileOptsFromCleanOutputDir(fixture, opts) + tExpectIssueCount(t, hT, 1) + tExpectCached(t, hT, "https://upload.wikimedia.org/wikipedia/en/404", 404) - // Verify it used the cache by checking for "from cache" messages - if hT2.issueStore.MessageMatchCount("from cache") == 0 { - t.Error("expected cached 404s to be reused (should see 'from cache' messages)") - } + // Second run: should use cached 404 (not retry because RetryCachedErrors is false) + hT2 := tTestFileOpts(fixture, opts) + tExpectIssueCount(t, hT2, 1) + + // Verify it used the cache (not retried) + tExpectIssue(t, hT2, "from cache", 1) + tExpectIssue(t, hT2, "hitting", 0) } // ======================================== @@ -78,84 +82,269 @@ func TestExternalBrokenRetryCachedErrorsDisabled(t *testing.T) { // and are retried on every run. This ensures backward compatibility with the // original behavior. func TestTimeoutNotCachedByDefault(t *testing.T) { + tSkipSlow(t) tSkipShortExternal(t) + fixture := "fixtures/links/ip_timeout.html" // First run: timeout occurs - hT := tTestFileOpts("fixtures/links/ip_timeout.html", - map[string]interface{}{"ExternalTimeout": 1, "EnableCache": true, "LogLevel": issues.LevelDebug}) + opts := map[string]interface{}{"ExternalTimeout": 1, "EnableCache": true} + hT := tTestFileOptsFromCleanOutputDir(fixture, opts) tExpectIssueCount(t, hT, 1) tExpectIssue(t, hT, "request exceeded our ExternalTimeout", 1) // Second run: should retry (not use cache) because RetryCachedErrors is true (default) - hT2 := tTestFileOpts("fixtures/links/ip_timeout.html", - map[string]interface{}{"ExternalTimeout": 1, "EnableCache": true, "LogLevel": issues.LevelDebug}) + hT2 := tTestFileOpts(fixture, opts) tExpectIssueCount(t, hT2, 1) // Verify it retried (should see "fresh" and "hitting", not "from cache") - if hT2.issueStore.MessageMatchCount("fresh") == 0 { - t.Error("timeout should be retried by default (should see 'fresh' message)") - } - if hT2.issueStore.MessageMatchCount("hitting") == 0 { - t.Error("timeout should be retried by default (should see 'hitting' message)") - } + tExpectIssue(t, hT2, "fresh", 1) + tExpectIssue(t, hT2, "hitting", 1) + tExpectIssue(t, hT2, "from cache", 0) +} + +// TestTimeoutNotCachedWithRetryCacheErrorsOnly : Test that timeouts are NOT +// cached when ONLY RetryCachedErrors is false (without CacheAllExternal). This +// ensures that the legacy behavior of RetryCachedErrors is no longer active. +func TestTimeoutNotCachedWithRetryCacheErrorsOnly(t *testing.T) { + tSkipSlow(t) + tSkipShortExternal(t) + opts := map[string]interface{}{"ExternalTimeout": 1, "EnableCache": true, "RetryCachedErrors": false} + + hT := tTestFileOptsFromCleanOutputDir("fixtures/links/ip_timeout.html", opts) + tExpectIssueCount(t, hT, 1) + tExpectIssue(t, hT, "request exceeded our ExternalTimeout", 1) + tExpectCacheEmpty(t, hT) } // TestTimeoutIsCached : Test that URLs that timeout are saved to the refcache -// when RetryCachedErrors is false. +// when CacheAllExternal is true. func TestTimeoutIsCached(t *testing.T) { + tSkipSlow(t) tSkipShortExternal(t) - hT := tTestFileOpts("fixtures/links/ip_timeout.html", - map[string]interface{}{"ExternalTimeout": 1, "EnableCache": true, "RetryCachedErrors": false}) + opts := map[string]interface{}{"ExternalTimeout": 1, "EnableCache": true, "CacheAllExternal": true} + hT := tTestFileOptsFromCleanOutputDir("fixtures/links/ip_timeout.html", opts) tExpectIssueCount(t, hT, 1) tExpectIssue(t, hT, "request exceeded our ExternalTimeout", 1) + tExpectCached(t, hT, "http://5.6.7.8", StatusTimeout) +} - // Verify the timeout was saved to cache with StatusTimeout - cR, ok := hT.refCache.Get("http://5.6.7.8") - if !ok { - t.Error("expected timeout to be cached when RetryCachedErrors is false, but it wasn't") - } - if cR.StatusCode != StatusTimeout { - t.Errorf("expected status code %d (StatusTimeout), got %d", StatusTimeout, cR.StatusCode) - } +// TestTimeoutCachedButRetried : Test that timeouts are cached when CacheAllExternal +// is true, but still retried on subsequent runs when RetryCachedErrors is true (default). +// This verifies Row 3 of the behavior matrix. +func TestTimeoutCachedButRetried(t *testing.T) { + tSkipSlow(t) + tSkipShortExternal(t) + fixture := "fixtures/links/ip_timeout.html" + opts := map[string]interface{}{"ExternalTimeout": 1, "EnableCache": true, "CacheAllExternal": true, "RetryCachedErrors": true} + + // First run: timeout occurs and is cached + hT := tTestFileOptsFromCleanOutputDir(fixture, opts) + tExpectIssueCount(t, hT, 1) + tExpectCached(t, hT, "http://5.6.7.8", StatusTimeout) + + // Second run: should retry despite being cached (RetryCachedErrors: true) + hT2 := tTestFileOpts(fixture, opts) + tExpectIssueCount(t, hT2, 1) + + // Verify it was retried (not reused from cache) + tExpectIssue(t, hT2, "fresh", 1) + tExpectIssue(t, hT2, "hitting", 1) + tExpectIssue(t, hT2, "from cache", 0) } // TestTimeoutCachedReused : Test that cached timeout results are reused on -// subsequent runs without retrying the URL when RetryCachedErrors is false. +// subsequent runs without retrying the URL when CacheAllExternal is true and +// RetryCachedErrors is false. func TestTimeoutCachedReused(t *testing.T) { + tSkipSlow(t) tSkipShortExternal(t) + fixture := "fixtures/links/ip_timeout.html" + opts := map[string]interface{}{"EnableCache": true, "CacheAllExternal": true, "RetryCachedErrors": false} // First run: cause and cache a timeout - hT := tTestFileOpts("fixtures/links/ip_timeout.html", - map[string]interface{}{"ExternalTimeout": 1, "EnableCache": true, "RetryCachedErrors": false}) + opts["ExternalTimeout"] = 1 + hT := tTestFileOptsFromCleanOutputDir(fixture, opts) tExpectIssueCount(t, hT, 1) - // Second run: WITHOUT timeout set - should use cached 524, not actually try the request - hT2 := tTestFileOpts("fixtures/links/ip_timeout.html", - map[string]interface{}{"EnableCache": true, "RetryCachedErrors": false, "LogLevel": issues.LevelDebug}) + // Second run: WITHOUT timeout set - should use cached timeout, not actually try the request + delete(opts, "ExternalTimeout") + hT2 := tTestFileOpts(fixture, opts) tExpectIssueCount(t, hT2, 1) // Verify it used the cache (should see "from cache" not "hitting") - if hT2.issueStore.MessageMatchCount("from cache") == 0 { - t.Error("expected cached timeout to be reused (should see 'from cache' message)") + tExpectIssue(t, hT2, "from cache", 1) + tExpectIssue(t, hT2, "hitting", 0) + tExpectIssue(t, hT2, "request exceeded our ExternalTimeout (cached)", 1) +} + +// ======================================== +// CacheAllExternal Tests +// ======================================== + +// TestCacheAllExternalDisabled : Test that external links are NOT cached when +// CacheAllExternal is false (default behavior). +// This is a regression test to ensure the default behavior doesn't change. +func TestCacheAllExternalDisabled(t *testing.T) { + // Run with CheckExternal disabled and CacheAllExternal disabled (defaults) + hT := tTestFileOptsFromCleanOutputDir("fixtures/links/brokenLinkExternalSingle.html", + map[string]interface{}{"CheckExternal": false, "EnableCache": true}) + tExpectIssueCount(t, hT, 0) // No errors since external checking is disabled + tExpectCacheEmpty(t, hT) +} + +// TestCacheAllExternalDiscovery : Test that external links ARE cached with +// StatusUnchecked when CacheAllExternal is true and CheckExternal is false. +// This is the core discovery mode feature. +func TestCacheAllExternalDiscovery(t *testing.T) { + fixture := "fixtures/links/brokenLinkExternalSingle.html" + opts := map[string]interface{}{"CheckExternal": false, "EnableCache": true, "CacheAllExternal": true} + + hT := tTestFileOptsFromCleanOutputDir(fixture, opts) + tExpectIssueCount(t, hT, 0) // No errors since external checking is disabled + + // Verify the external link WAS cached with StatusUnchecked + tExpectCached(t, hT, "http://www.asdo3IRJ395295jsingrkrg4.com", StatusUnchecked) +} + +// TestCacheAllExternalAndIgnoredURLs : Test that ignored URLs are NOT cached in +// discovery mode, verifying CacheAllExternal respects IgnoreURLs patterns. +func TestCacheAllExternalAndIgnoredURLs(t *testing.T) { + fixture := "fixtures/links/brokenLinkExternalSingle.html" + opts := map[string]interface{}{ + "CheckExternal": false, + "EnableCache": true, + "CacheAllExternal": true, + "IgnoreURLs": []interface{}{"asd.*\\.com"}, } - if hT2.issueStore.MessageMatchCount("hitting") > 0 { - t.Error("should not retry when timeout is cached and RetryCachedErrors is false") + + hT := tTestFileOptsFromCleanOutputDir(fixture, opts) + tExpectIssueCount(t, hT, 0) + tExpectCacheEmpty(t, hT) +} + +// TestCacheAllExternalQueryString : Test that query strings are stripped before +// caching in discovery mode, verifying CacheAllExternal works with StripQueryString. +func TestCacheAllExternalQueryString(t *testing.T) { + fixture := "fixtures/links/check_just_once.html" + opts := map[string]interface{}{ + "CheckExternal": false, + "EnableCache": true, + "CacheAllExternal": true, + "StripQueryString": true, } + + hT := tTestFileOptsFromCleanOutputDir(fixture, opts) + tExpectIssueCount(t, hT, 0) + + // Verify the URL was cached WITHOUT the query string + tExpectCached(t, hT, "https://github.com/contact", StatusUnchecked) + + // Verify the URL WITH query string is NOT in cache (proves stripping happened) + tExpectNotCached(t, hT, "https://github.com/contact?form%5Bsubject%5D=New+Assigned+Events") } -// TestTimeoutCachedMessage : Test that a URL that previously timed out will be -// reported as "(cached)" on subsequent runs. when RetryCachedErrors is false. -func TestTimeoutCachedMessage(t *testing.T) { +// ======================================== +// Phase 2: Tool-Specific Error Caching +// ======================================== + +// TestNetworkErrorCached : Test that network/DNS errors are cached with +// StatusNetworkError when CacheAllExternal is true. +func TestNetworkErrorCached(t *testing.T) { + fixture := "fixtures/generic/citeBroken.html" + opts := map[string]interface{}{ + "VCREnable": true, + "EnableCache": true, + "CacheAllExternal": true, + } + + hT := tTestFileOptsFromCleanOutputDir(fixture, opts) + tExpectIssueCount(t, hT, 4) // 4 broken citations + + // Verify the network error was cached with StatusNetworkError + tExpectCached(t, hT, "http://invalid.invalid", StatusNetworkError) +} + +// TestNetworkErrorCachedReused : Test that cached network errors are reused on +// subsequent runs when CacheAllExternal is true and RetryCachedErrors is false. +func TestNetworkErrorCachedReused(t *testing.T) { + fixture := "fixtures/generic/citeBroken.html" + opts := map[string]interface{}{ + "VCREnable": true, + "EnableCache": true, + "CacheAllExternal": true, + "RetryCachedErrors": false, + } + + // First run: populate cache with network error + hT := tTestFileOptsFromCleanOutputDir(fixture, opts) + tExpectIssueCount(t, hT, 4) + tExpectCached(t, hT, "http://invalid.invalid", StatusNetworkError) + + // Second run WITHOUT VCR: should use cached errors (not retry) + opts["VCREnable"] = false + hT2 := tTestFileOpts(fixture, opts) + tExpectIssueCount(t, hT2, 4) + + // Verify cached errors were reused (fixture has 2 external links: invalid.invalid + Wikipedia 404) + tExpectIssue(t, hT2, "from cache", 2) + tExpectIssue(t, hT2, "hitting", 0) +} + +// TestCertErrorCached : Test that certificate errors are cached with +// StatusCertError when CacheAllExternal is true. +func TestCertErrorCached(t *testing.T) { tSkipShortExternal(t) + url := "https://expired.badssl.com/" + fixture := "fixtures/links/link_expired_cert.html" + opts := map[string]interface{}{ + "EnableCache": true, + "CacheAllExternal": true, + "IgnoreSSLVerify": false, + } - // First run: cause and cache a timeout - hT := tTestFileOpts("fixtures/links/ip_timeout.html", - map[string]interface{}{"ExternalTimeout": 1, "EnableCache": true, "RetryCachedErrors": false}) + hT := tTestFileOptsFromCleanOutputDir(fixture, opts) tExpectIssueCount(t, hT, 1) + tExpectCached(t, hT, url, StatusCertError) +} + +// TestCertErrorCachedReused : Test that cached certificate errors are reused +// when Cache AllExternal is true and RetryCachedErrors is false. +func TestCertErrorCachedReused(t *testing.T) { + tSkipShortExternal(t) + url := "https://expired.badssl.com/" + fixture := "fixtures/links/link_expired_cert.html" + opts := map[string]interface{}{ + "EnableCache": true, + "CacheAllExternal": true, + "IgnoreSSLVerify": false, + "RetryCachedErrors": false, + "VCREnable": false, // Disable VCR to verify cache behavior + } + + // First run: cache the cert error + hT1 := tTestFileOptsFromCleanOutputDir(fixture, opts) + tExpectIssueCount(t, hT1, 1) + tExpectCached(t, hT1, url, StatusCertError) - // Second run: with RetryCachedErrors disabled, should use cached timeout and report with "(cached)" message - hT2 := tTestFileOpts("fixtures/links/ip_timeout.html", - map[string]interface{}{"EnableCache": true, "RetryCachedErrors": false}) + // Second run: reuse cached cert error (no external request) + hT2 := tTestFileOpts(fixture, opts) tExpectIssueCount(t, hT2, 1) - tExpectIssue(t, hT2, "request exceeded our ExternalTimeout (cached)", 1) + tExpectIssue(t, hT2, "certificate error (cached)", 1) + tExpectCached(t, hT2, url, StatusCertError) +} + +// TestClientErrorCached : Test that generic HTTP client errors are cached with +// StatusClientError when CacheAllExternal is true. +// Note: Generic client errors are rare - this is a catch-all for unhandled error types. +func TestClientErrorCached(t *testing.T) { + // TODO: Need to find or create a fixture that triggers a generic client error + // that isn't already handled by timeout, network, or cert error handlers. + // This might require mocking or a very specific edge case. + t.Skip("Need fixture that triggers generic client error (not timeout/network/cert)") +} + +// TestClientErrorCachedReused : Test that cached generic client errors are reused +// when CacheAllExternal is true and RetryCachedErrors is false. +func TestClientErrorCachedReused(t *testing.T) { + t.Skip("Depends on TestClientErrorCached fixture") } diff --git a/htmltest/check-link.go b/htmltest/check-link.go index befa2e7..e5cd1da 100644 --- a/htmltest/check-link.go +++ b/htmltest/check-link.go @@ -137,7 +137,9 @@ func (hT *HTMLTest) checkExternal(ref *htmldoc.Reference) { Message: "skipping external check", Reference: ref, }) - return + if !hT.opts.CacheAllExternal { + return + } } urlStr := ref.URLString() @@ -150,6 +152,17 @@ func (hT *HTMLTest) checkExternal(ref *htmldoc.Reference) { if hT.opts.StripQueryString && !InList(hT.opts.StripQueryExcludes, urlStr) { urlStr = htmldoc.URLStripQueryString(urlStr) } + + // Discovery mode: cache as unchecked and return early + if !hT.opts.CheckExternal { + // Invariant: CacheAllExternal must be true, otherwise would have returned earlier + if !hT.opts.CacheAllExternal { + panic("Invariant violation: CacheAllExternal should be true") + } + hT.refCache.Save(urlStr, StatusUnchecked) + return + } + var statusCode int cR, isCached := hT.refCache.Get(urlStr) @@ -199,7 +212,7 @@ func (hT *HTMLTest) checkExternal(ref *htmldoc.Reference) { if err != nil { if strings.Contains(err.Error(), "Client.Timeout") { - if !hT.opts.RetryCachedErrors { + if hT.opts.CacheAllExternal { hT.refCache.Save(urlStr, StatusTimeout) } hT.issueStore.AddIssue(issues.Issue{ @@ -211,6 +224,9 @@ func (hT *HTMLTest) checkExternal(ref *htmldoc.Reference) { } if certErr, ok := err.(*url.Error).Err.(x509.UnknownAuthorityError); ok { + if hT.opts.CacheAllExternal { + hT.refCache.Save(urlStr, StatusCertError) + } err = validateCertChain(certErr.Cert) if err == nil { hT.issueStore.AddIssue(issues.Issue{ @@ -224,6 +240,9 @@ func (hT *HTMLTest) checkExternal(ref *htmldoc.Reference) { // More generic, should be kept below more specific cases if strings.Contains(err.Error(), "dial tcp") { + if hT.opts.CacheAllExternal { + hT.refCache.Save(urlStr, StatusNetworkError) + } // Remove long prefix prefix := "Get " + urlStr + ": dial tcp: lookup " cleanedMessage := strings.TrimPrefix(err.Error(), prefix) @@ -237,6 +256,14 @@ func (hT *HTMLTest) checkExternal(ref *htmldoc.Reference) { } // Unhandled client error, return generic error + if hT.opts.CacheAllExternal { + statusCode := StatusClientError + if strings.Contains(err.Error(), "x509:") { + statusCode = StatusCertError + } + hT.refCache.Save(urlStr, statusCode) + } + hT.issueStore.AddIssue(issues.Issue{ Level: issueLevel, Message: err.Error(), @@ -263,6 +290,24 @@ func (hT *HTMLTest) checkExternal(ref *htmldoc.Reference) { Message: http.StatusText(statusCode), Reference: ref, }) + case StatusCertError: + hT.issueStore.AddIssue(issues.Issue{ + Level: issueLevel, + Message: "certificate error (cached)", + Reference: ref, + }) + case StatusClientError: + hT.issueStore.AddIssue(issues.Issue{ + Level: issueLevel, + Message: "client error (cached)", + Reference: ref, + }) + case StatusNetworkError: + hT.issueStore.AddIssue(issues.Issue{ + Level: issueLevel, + Message: "network error (cached)", + Reference: ref, + }) case StatusTimeout: hT.issueStore.AddIssue(issues.Issue{ Level: issueLevel, diff --git a/htmltest/fixtures/links/link_expired_cert.html b/htmltest/fixtures/links/link_expired_cert.html new file mode 100644 index 0000000..3ab28c3 --- /dev/null +++ b/htmltest/fixtures/links/link_expired_cert.html @@ -0,0 +1,9 @@ + + + + Test expired certificate + + + expired cert link + + diff --git a/htmltest/options.go b/htmltest/options.go index 09241ba..39282c2 100644 --- a/htmltest/options.go +++ b/htmltest/options.go @@ -68,12 +68,13 @@ type Options struct { StripQueryString bool StripQueryExcludes []interface{} - EnableCache bool - EnableLog bool - OutputDir string - OutputCacheFile string - OutputLogFile string - CacheExpires string // Accepts golang time period strings, hours (16h) is really only useful option + EnableCache bool + CacheAllExternal bool + EnableLog bool + OutputDir string + OutputCacheFile string + OutputLogFile string + CacheExpires string // Accepts golang time period strings, hours (16h) is really only useful option // --- Internals below here --- NoRun bool // When true does not run tests, used to inspect state in unit tests @@ -140,12 +141,13 @@ func DefaultOptions() map[string]interface{} { "StripQueryString": true, "StripQueryExcludes": []interface{}{"fonts.googleapis.com"}, - "EnableCache": true, - "EnableLog": true, - "OutputDir": path.Join("tmp", ".htmltest"), - "OutputCacheFile": "refcache.json", - "OutputLogFile": "htmltest.log", - "CacheExpires": "336h", + "EnableCache": true, + "CacheAllExternal": false, + "EnableLog": true, + "OutputDir": path.Join("tmp", ".htmltest"), + "OutputCacheFile": "refcache.json", + "OutputLogFile": "htmltest.log", + "CacheExpires": "336h", "NoRun": false, "VCREnable": false, diff --git a/htmltest/statuscodes.go b/htmltest/statuscodes.go index 40411c3..99dcea7 100644 --- a/htmltest/statuscodes.go +++ b/htmltest/statuscodes.go @@ -11,8 +11,11 @@ For details, see @docs/tasks/migrate-status-codes.md */ const ( - StatusUnchecked = 0 - StatusTimeout = -10 + StatusUnchecked = 0 + StatusTimeout = -10 + StatusNetworkError = -20 // DNS failures, connection refused, etc. + StatusCertError = -30 // Certificate validation errors (x509, expired, invalid, etc.) + StatusClientError = -40 // Generic HTTP client errors ) func IsHTTPStatus(code int) bool { diff --git a/htmltest/test_helpers_extra_test.go b/htmltest/test_helpers_extra_test.go new file mode 100644 index 0000000..be48033 --- /dev/null +++ b/htmltest/test_helpers_extra_test.go @@ -0,0 +1,81 @@ +package htmltest + +import ( + "encoding/json" + "flag" + "os" + "path" + "testing" + + "github.com/imdario/mergo" + "github.com/stretchr/testify/assert" +) + +var skipSlow = flag.Bool("skip-slow", false, "skip slow tests (timeouts, etc.)") + +// Test fixture helpers + +// tTestFileOptsFromCleanOutputDir runs a test after having removed the default +// output directory. This is ensures a clean fixture for tests that might be +// influenced by the output directory content, such as the refcache file. +func tTestFileOptsFromCleanOutputDir(filename string, tOpts map[string]interface{}) *HTMLTest { + tRemoveOutputDir(tOpts) + return tTestFileOpts(filename, tOpts) +} + +func tRemoveOutputDir(tOpts map[string]interface{}) { + opts := DefaultOptions() + _ = mergo.MergeWithOverwrite(&opts, tOpts) + _ = os.RemoveAll(opts["OutputDir"].(string)) +} + +// Test skip helpers + +// tSkipSlow skips slow tests that involve actual timeouts (seconds of waiting). +// Controlled by the -skip-slow flag. Use -skip-slow=true to skip these tests +// during rapid development. +func tSkipSlow(t *testing.T) { + if *skipSlow { + t.Skip("Skipping slow test (involves actual timeout waits)") + } +} + +// Cache assertion helpers + +// tExpectCached asserts that a URL is present in the refcache. +// If statusCode is provided, also asserts the cached status code matches. +func tExpectCached(t *testing.T, hT *HTMLTest, url string, statusCode ...int) { + cR, ok := hT.refCache.Get(url) + if !assert.True(t, ok, "URL should be cached: "+url) { + return // Stop if URL not in cache to avoid nil pointer panic + } + if len(statusCode) > 0 { + assert.Equal(t, statusCode[0], cR.StatusCode, "cached status code") + } +} + +// tExpectNotCached asserts that a URL is NOT present in the refcache. +func tExpectNotCached(t *testing.T, hT *HTMLTest, url string) { + _, ok := hT.refCache.Get(url) + assert.False(t, ok, "URL should not be cached: "+url) +} + +// tExpectCacheEmpty asserts that the refcache is empty. +func tExpectCacheEmpty(t *testing.T, hT *HTMLTest) { + cachePath := path.Join(hT.opts.OutputDir, hT.opts.OutputCacheFile) + data, err := os.ReadFile(cachePath) + if err != nil { + if os.IsNotExist(err) { + return // Cache file doesn't exist - that's empty! + } + t.Fatalf("Error reading cache file: %v", err) + } + + // Parse JSON to check if empty + var cache map[string]interface{} + if err := json.Unmarshal(data, &cache); err != nil { + t.Fatalf("Cache file is not valid JSON: %v", err) + } + + assert.Equal(t, 0, len(cache), "cache should be empty") +}