Repository navigation
Expand file tree
/
Copy pathcheck-contract.cjs
More file actions
383 lines (359 loc) · 16.7 KB
/
Copy pathcheck-contract.cjs
File metadata and controls
383 lines (359 loc) · 16.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
#!/usr/bin/env node
/**
* Guard for a contract that was transcribed from a specification: prove the contract can actually judge.
*
* A contract is the linchpin of contract-first delegation, and under this plugin's rules the architect can
* neither write it nor read it back -- reading it would put implementation-shaped code into the metered
* context. So the architect specifies behaviour, a worker transcribes the tests, and the architect is left
* unable to tell a working contract from a broken one. This guard answers that without reading a line of it.
*
* The judgement is made against a NULL IMPLEMENTATION, not against the real module, and that choice is the
* whole design. A test that fails by malfunctioning -- a broken harness, an unloadable module, a rejected
* fixture -- looks exactly like a test that caught a bug, and the architect cannot tell them apart. Against
* a real module the ambiguity is unresolvable: the module may simply be throwing. Against a null
* implementation that never throws, the module cannot be blamed, so a malfunction is unambiguously the
* contract's own fault. The run against the real module is reported and not judged, for the same reason.
*
* The contract is never modified and the real tree is never touched: it is mirrored into a temp directory
* with the stub placed at the same relative position, which works because a contract reaches its module by
* a path relative to its own directory.
*
* A second optional check closes the one gap the null run cannot see: a contract can be well-formed and
* discriminating and still test the wrong thing. The contract-first experiment in docs/experiment.md spent
* three rounds failing to satisfy a contract that asserted a predicate and its own negation while the null
* run called it sound every time. The `--spec` flag takes a specification conformance suite owned by the
* architect, which is the null trick run backwards: null proves the module cannot be blamed, and the suite
* proves the module CAN be trusted, after which a contract that still rejects a conforming module is the
* artifact at fault. The verdict names the failing contract tests and leaves the resolution open, because it
* is only as strong as the suite — the contract may demand more than the specification states, or the suite
* may not cover what the contract tests.
*
* Usage: node scripts/check-contract.cjs --contract <file> --module <file> [--spec <file>]
* Exit 0 when the contract is fit to judge, 1 when it is not, 2 on bad usage.
*/
'use strict'
const { spawnSync } = require('node:child_process')
const fs = require('node:fs')
const os = require('node:os')
const path = require('node:path')
/**
* A module that answers every property with itself and every call with itself, and never throws.
*
* Not throwing is the point: it is what makes the module blameless. Iteration and await are handled
* explicitly so that a contract exercising them still reaches its own assertions rather than dying in the
* stub, which would be a malfunction the contract did not cause and must not be charged for.
*/
const NULL_IMPLEMENTATION = [
"'use strict'",
'const NULL = new Proxy(function () {}, {',
' get: (target, prop) => {',
' if (prop === Symbol.iterator) return function* () {}',
' if (prop === Symbol.asyncIterator) return async function* () {}',
' if (prop === Symbol.toPrimitive) return () => 0',
' if (prop === Symbol.toStringTag) return "NullImplementation"',
' if (prop === "then") return undefined',
' return NULL',
' },',
' apply: () => NULL,',
'})',
'module.exports = NULL',
'',
].join('\n')
function argValue(name) {
const at = process.argv.indexOf('--' + name)
return at !== -1 && process.argv[at + 1] ? process.argv[at + 1] : null
}
const contractArg = argValue('contract')
const moduleArg = argValue('module')
const specArg = argValue('spec')
if (!contractArg || !moduleArg) {
console.log('usage: node scripts/check-contract.cjs --contract <file> --module <file> [--spec <file>]')
process.exit(2)
}
const contract = path.resolve(contractArg)
const modulePath = path.resolve(moduleArg)
const specPath = specArg ? path.resolve(specArg) : null
const pairs = [['contract', contract], ['module', modulePath]]
if (specPath) pairs.push(['specification suite', specPath])
for (const pair of pairs) {
if (!fs.existsSync(pair[1])) {
console.log('FAIL: the ' + pair[0] + ' does not exist: ' + pair[1])
process.exit(1)
}
}
/**
* Run one test file and classify its failures.
*
* Output is captured through a FILE DESCRIPTOR rather than a pipe, deliberately: DSH's confined sandbox
* modes refuse a piped spawn outright (spawn EPERM), which would make this guard unusable in exactly the
* environment it is meant to run in. `verification.ts` captures the same way for the same reason.
*/
function runTests(file) {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'dsh-contract-run-'))
const outFile = path.join(dir, 'out.txt')
const fd = fs.openSync(outFile, 'w')
// The test runner passes NODE_TEST_CONTEXT to its own children; handing it to a grandchild silently redirects the report away from stdout.
const env = { ...process.env }
delete env.NODE_TEST_CONTEXT
const run = spawnSync(process.execPath, ['--test', file], { stdio: ['ignore', fd, fd], env })
fs.closeSync(fd)
const output = fs.readFileSync(outFile, 'utf8')
// The capture has been read into memory, so the directory holding it is finished with.
//
// It used to be left behind: one `dsh-contract-run-*` directory per runner invocation, two per contract,
// so a full gate run scattered fourteen of them into the shared temp directory and nothing ever removed
// them. Captured test output is not a secret, but this script's whole claim is that it judges a
// CONTROLLED condition, and a guard that cannot clean up after itself is not exercising the control it
// asserts. Found while measuring the mirror-containment defect next door, and fixed in the same pass.
fs.rmSync(dir, { recursive: true, force: true })
const number = (re) => {
const m = output.match(re)
return m ? Number(m[1]) : 0
}
const notOk = (output.match(/^not ok /gm) || []).length
const assertion = (output.match(/code: 'ERR_ASSERTION'/g) || []).length
return {
status: run.status,
output,
total: number(/^# tests (\d+)/m),
passed: number(/^# pass (\d+)/m),
notOk,
assertion,
nonAssertion: notOk - assertion,
}
}
function failingNames(output) {
const names = []
for (const line of output.split('\n')) {
if (/^not ok /.test(line)) names.push(line.trim())
}
return names
}
console.log('contract : ' + contract)
console.log('module : ' + modulePath)
console.log('')
let failed = 0
// ---- A. the controlled condition, and the only thing judged -----------------------------------------
const mirrorRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'dsh-contract-null-'))
/**
* True when `candidate` is strictly inside `base`, by PATH ARITHMETIC rather than by string prefix.
*
* A `startsWith` comparison is not sufficient on Windows: `C:\tmp\dsh-1x` is a prefix of
* `C:\tmp\dsh-1xyz`, and the separator is the difference between a containment test and a coincidence.
*/
const isStrictlyInside = (base, candidate) => {
const rel = path.relative(path.resolve(base), path.resolve(candidate))
return rel !== '' && !rel.startsWith('..') && !path.isAbsolute(rel)
}
/**
* Refuse from a position where the temporary mirror root already exists.
*
* The cleanup at the end of this file is a plain statement, not a `finally`, and `process.exit` runs no
* `finally` regardless -- so every refusal below has to remove the root itself. Measured before this helper:
* one abandoned `dsh-contract-null-*` directory per refused run, in the same shared temp directory the
* containment fix was about.
*/
const refuse = (message) => {
console.error(' FAIL ' + message)
fs.rmSync(mirrorRoot, { recursive: true, force: true })
process.exit(1)
}
// A test reaches its module by a path RELATIVE to its own directory, so the null stub has to be placed at
// that same relative position -- and the mirror has to be deep enough to absorb the `..` ascents in it.
//
// The previous version mirrored the test one level down, at `<root>/<basename of its directory>/<basename>`.
// A contract in `tests/contracts` reaches its module by `../../tools/x.ts`, so two ascents from a one-level
// mirror resolved OUTSIDE the temporary root: at `<tmpdir>/tools/x.ts`, a directory every process on the
// machine shares, where the stub then stayed, because the `rmSync(mirrorRoot)` below cannot reach outside
// its own root. Measured on this machine before the fix: seven 432-byte null stubs in `%TEMP%\tools\`, one
// per contract in the repository, plus six abandoned mirror roots.
//
// That is worse than a leak of scratch files. This guard's entire premise is that it judges the contract
// against a CONTROLLED condition, and the module it judged was a file outside its control that outlived the
// run -- a later process could have created that path first, and the null run would have judged ITS stub.
//
// And the wrong path was not merely wrong, it was ATTACKER-SHAPED. The stub location is
// `resolve(mirrorDir, relative)`, and `relative` is derived from the `--module` argument, so a hostile
// invocation could aim the write at any path on the drive (`../../../..` and onward) with no containment
// test anywhere in the script. The content is only ever the null stub, but overwriting a file the caller
// names is still a write primitive, handed to the caller by the guard.
//
// The fix mirrors the test at a depth EQUAL TO THE NUMBER OF ASCENTS in its own module path, so the ascents
// always arrive back inside the mirror root. That holds for a test anywhere on the drive -- including
// outside the repository, which `contract-spec-conformance.test.cjs` exercises -- and it needs no assumption
// about where the repository is. The containment assertions after each resolve are then belt and braces:
// they should be unreachable, and they refuse loudly rather than passing quietly if they are not.
const mirrorDepthOf = (fromDir, toFile) => {
const rel = path.relative(fromDir, toFile)
// A different drive has no relative path at all, so no amount of nesting can contain the stub.
if (path.isAbsolute(rel)) return null
let depth = 0
const parts = rel.split(path.sep)
while (depth < parts.length && parts[depth] === '..') depth += 1
return { rel, depth }
}
const mirrorTestFile = (testFile) => {
const plan = mirrorDepthOf(path.dirname(testFile), modulePath)
if (plan === null) {
refuse(
modulePath + ' and ' + testFile + ' are on different drives, so no relative mirror can ' +
'contain the null stub. Refusing rather than writing outside the mirror.'
)
}
// One `mirror` directory per ascent: whatever the test's real location, resolving its module path from
// the bottom of this nest lands inside `mirrorRoot`.
const mirrorDir = path.join(mirrorRoot, ...Array(plan.depth).fill('mirror'))
let mirrorPath = path.join(mirrorDir, path.basename(testFile))
// A contract and its specification suite can share a basename, which would silently run one where the
// other was meant. The mirrored NAME does not matter to the module resolution -- only the depth does --
// so a collision is resolved by renaming rather than by refusing.
for (let n = 1; fs.existsSync(mirrorPath); n += 1) {
mirrorPath = path.join(mirrorDir, 'mirror-' + n + '-' + path.basename(testFile))
}
if (!isStrictlyInside(mirrorRoot, mirrorPath)) {
refuse(
'cannot place ' + testFile + ' inside ' + mirrorRoot + '. Without a controlled mirror the ' +
'null run would judge a module this guard does not own, so it is refused rather than attempted.'
)
}
fs.mkdirSync(mirrorDir, { recursive: true })
fs.copyFileSync(testFile, mirrorPath)
const mirrorModule = path.resolve(mirrorDir, plan.rel)
if (!isStrictlyInside(mirrorRoot, mirrorModule)) {
refuse(
'the null stub for ' + modulePath + ' resolves to ' + mirrorModule + ', which is outside ' +
mirrorRoot + '. Refusing to write a stub outside the mirror.'
)
}
fs.mkdirSync(path.dirname(mirrorModule), { recursive: true })
fs.writeFileSync(mirrorModule, NULL_IMPLEMENTATION, 'utf8')
return mirrorPath
}
const mirrorContract = mirrorTestFile(contract)
const mirrorSpec = specPath ? mirrorTestFile(specPath) : null
const nullRun = runTests(mirrorContract)
console.log(
'A. against a null implementation: ' +
nullRun.total +
' tests, ' +
nullRun.passed +
' passed, ' +
nullRun.assertion +
' failed by assertion, ' +
nullRun.nonAssertion +
' by malfunction'
)
if (nullRun.total === 0) {
failed++
console.log(' FAIL: the contract did not run -- no test count was reported.')
} else if (nullRun.nonAssertion > 0) {
failed++
console.log(' FAIL: ' + nullRun.nonAssertion + ' test(s) never reached an assertion. The module here')
console.log(' cannot be blamed, because it never throws, so the contract is malfunctioning on its')
console.log(' own account. This is the defect the architect cannot see by reading it:')
for (const name of failingNames(nullRun.output)) console.log(' ' + name)
} else if (nullRun.assertion > 0) {
console.log(' ok: it asserts, and every failure is a behavioural disagreement')
} else {
failed++
console.log(' FAIL: it passed a module whose every call returns itself, so it constrains nothing')
}
// ---- B. reported, deliberately not judged ------------------------------------------------------------
const real = runTests(contract)
console.log(
'B. against the real module: ' +
real.total +
' tests, ' +
real.passed +
' passed, ' +
real.assertion +
' failed by assertion, ' +
real.nonAssertion +
' by malfunction'
)
console.log(' reported only. A malfunction here may be the module throwing rather than the contract')
console.log(' failing, and nothing in the output distinguishes the two -- which is why A decides.')
if (nullRun.passed > 0) {
console.log('')
console.log(
'note: ' +
nullRun.passed +
' test(s) pass a module that returns itself for everything. They assert nothing about the module'
)
console.log(' and are worth a second look from whoever can read the file.')
}
if (specPath) {
const realSpec = runTests(specPath)
const nullSpec = runTests(mirrorSpec)
console.log('')
console.log(
'C. the specification suite against the real module: ' +
realSpec.total +
' test(s), ' +
realSpec.passed +
' passed, ' +
realSpec.assertion +
' failed by assertion, ' +
realSpec.nonAssertion +
' failed by malfunction'
)
console.log('')
console.log(
'D. the specification suite against a null implementation: ' +
nullSpec.total +
' test(s), ' +
nullSpec.passed +
' passed, ' +
nullSpec.assertion +
' failed by assertion, ' +
nullSpec.nonAssertion +
' failed by malfunction'
)
let specRuleSatisfied = false
if (nullSpec.total === 0) {
failed++
console.log(' FAIL: the specification suite did not run -- no test count was reported.')
} else if (nullSpec.nonAssertion > 0) {
failed++
console.log(
' FAIL: the module here cannot be blamed, so the suite is malfunctioning on its own account.'
)
for (const name of failingNames(nullSpec.output)) {
console.log(' ' + name)
}
} else if (nullSpec.assertion > 0) {
specRuleSatisfied = true
console.log(' ok: it asserts, and every failure is a behavioural disagreement')
} else {
failed++
console.log(
' FAIL: it passed a module whose every call returns itself, so the specification suite constrains nothing'
)
}
if (
realSpec.total > 0 &&
realSpec.passed === realSpec.total &&
specRuleSatisfied &&
real.assertion > 0
) {
failed++
console.log('')
console.log('DISAGREEMENT: the module satisfies the specification suite, and the contract still rejects it.')
console.log('Either the contract demands more than the specification states, or the specification suite is incomplete.')
console.log('Resolve which before delegating another fix. Contract tests that failed against a conforming module:')
for (const name of failingNames(real.output)) {
console.log(' ' + name)
}
}
}
try {
fs.rmSync(mirrorRoot, { recursive: true, force: true })
} catch (err) {}
console.log('')
if (failed === 0) {
console.log('OK: the contract is well-formed and discriminating.')
process.exit(0)
}
console.log('The contract is not fit to judge a unit. It was transcribed from a specification, so the')
console.log('specification or the transcription is at fault; the architect cannot tell which by reading it.')
process.exit(1)