feat(t27mcp): find.t27 -- t27_find, intent to ranked corpus specs or an honest miss

specs/t27mcp/find.t27

@@ -0,0 +1,369 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/t27mcp/find.t27 -- t27_find of the t27 MCP server (gHashTag/t27-mcp): which spec of the published
+; corpus answers an intent such as "clocked counter with sync reset", or none. See #10066 (epic), the first
+; step of its validated-code contract: reuse a validated spec when one exists, write a new one when none does.
+; --
+; The lookup tools filter by path, tag or regex. A model that asks for a counter does not know the file is
+; called clocked_counter. This spec decides which manifest rows answer an intent; the server feeds it one row
+; at a time (name, path, tags, lines) and sorts the rows it admits.
+; --
+; The rules:
+; - The words of the intent are its runs of letters and digits. A word is dropped when it carries no intent:
+;   one character or a stop word (articles, "write", "module", "test": every spec in the corpus
+;   has a module and an admitted spec has tests). What is left are the content words, n of them.
+; - A content word is stemmed (ing, ed, er, plural s) and looked for as the START of a token of the file
+;   name (score 2) or of the path (score 1). The start of a token, not any substring: "count" is not
+;   found in "account". A stem of one or two letters must be a whole token. ALT_FROM -> ALT_TO adds the
+;   abbreviations a name uses (clock -> clk, reset -> rst).
+; - A row answers the intent when it covers it: with n <= 3 every word is found, with more at least two
+;   thirds of them, and the file name carries at least one word, and at least half of all n words. Fewer
+;   is a MISS, not a weak hit: a spec that covers "counter" but not "sync reset" is not a counter with a sync
+;   reset, and a model copies what it is handed (CodeRAG-Bench, arXiv 2406.14497).
+; - Relevance is not enough. The row must also be healthy (tag health/ok) and carry tests (tag has/tests).
+;   Otherwise it is counted under unhealthy or untested and not served. The tag says a test block exists, not
+;   that it passes: the receipt (test-report, 0 vacuous) is the next tool, t27_validate.
+; - Rows are ranked by rank_key(): more found words, name matches over path matches, fewer name tokens, then
+;   fewer lines. A key of 0 means "not a hit".
+; - Nothing is matched on a summary: the manifest summary is the same boilerplate for every spec.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module T27mcpFind;
+
+use t27mcp::check_hints;
+
+/// verdict() answers.
+pub const V_NONE : u32 = 0;
+pub const V_UNHEALTHY : u32 = 1;
+pub const V_UNTESTED : u32 = 2;
+pub const V_HIT : u32 = 3;
+pub const V_EMPTY : u32 = 4;
+
+/// At most this many hits are returned.
+pub const LIMIT : u32 = 5;
+
+pub const HIT_NOTE : str = "Each hit is a corpus spec that covers the asked words (all of them up to three words, two thirds of a longer intent) in its name or path, has compiled clean and carries tests. The tests are a tag of the manifest, not a receipt: read the spec with t27_spec and run its tests before you rely on it.";
+pub const MISS_NOTE : str = "No spec of the corpus covers this intent, so nothing is returned. Write a .t27 spec with tests: t27_examples for the idiom, the test block first, t27_check, then the golden pipeline (t27c test-report with 0 vacuous, a negative control, tools/dupe_scan.py) and a PR to gHashTag/t27.";
+pub const EMPTY_NOTE : str = "The intent holds no searchable word: say what the module does (counter, uart, fifo, shift register).";
+
+pub const STOPS : [61]str = [
+    "me", "my", "it", "is", "to", "of", "in", "on", "at", "do",
+    "be", "by", "as", "or", "if", "we", "us", "an", "the", "with",
+    "and", "for", "from", "that", "this", "its", "can", "how", "does", "any",
+    "some", "new", "use", "using", "get", "give", "show", "find", "want", "need",
+    "please", "write", "make", "create", "build", "implement", "simple", "basic", "small", "module",
+    "spec", "specs", "test", "tests", "code", "design", "circuit", "have", "has", "are",
+    "was"
+];
+
+pub const ALT_FROM : [10]str = [
+    "clock", "reset", "count", "register", "multipli", "multiply", "synchronous", "divid", "divide", "interrupt"
+];
+pub const ALT_TO : [10]str = [
+    "clk", "rst", "cnt", "reg", "mul", "mul", "sync", "div", "div", "irq"
+];
+
+/// The length of a string, so a table entry can be measured through a parameter.
+pub fn str_len(s: string) -> u32 {
+    return (s.len as u32);
+}
+
+/// A byte in lower case.
+pub fn low(c: u32) -> u32 {
+    if (c >= 65) {
+        if (c <= 90) { return c + 32; }
+    }
+    return c;
+}
+
+/// 1 for a digit or a letter.
+pub fn is_word_char(c: u32) -> u32 {
+    var d : u32 = low(c);
+    if (d >= 48) {
+        if (d <= 57) { return 1; }
+    }
+    if (d >= 97) {
+        if (d <= 122) { return 1; }
+    }
+    return 0;
+}
+
+/// 1 when the n bytes of a from ai equal the n bytes of b from bi, ignoring case; 0 when either runs out.
+pub fn same_at(a: string, ai: u32, b: string, bi: u32, n: u32) -> u32 {
+    if (ai + n > str_len(a)) { return 0; }
+    if (bi + n > str_len(b)) { return 0; }
+    var k : u32 = 0;
+    while (k < n) {
+        if (low((a[ai + k] as u32)) != low((b[bi + k] as u32))) { return 0; }
+        k = k + 1;
+    }
+    return 1;
+}
+
+/// 1 when some token of hay starts with the n bytes of w from wi; with whole = 1 the token must also end there.
+pub fn token_hit(hay: string, w: string, wi: u32, n: u32, whole: u32) -> u32 {
+    var hn : u32 = str_len(hay);
+    var i : u32 = 0;
+    while (i + n <= hn) {
+        var starts : u32 = 1;
+        if (i > 0) { starts = 1 - is_word_char((hay[i - 1] as u32)); }
+        if (starts == 1) {
+            if (same_at(hay, i, w, wi, n) == 1) {
+                if (whole == 0) { return 1; }
+                if (i + n == hn) { return 1; }
+                if (is_word_char((hay[i + n] as u32)) == 0) { return 1; }
+            }
+        }
+        i = i + 1;
+    }
+    return 0;
+}
+
+/// 1 when the word w[wi..wi+n) ends with suf.
+pub fn ends_with(w: string, wi: u32, n: u32, suf: string) -> u32 {
+    var sn : u32 = str_len(suf);
+    if (n < sn) { return 0; }
+    return same_at(w, wi + n - sn, suf, 0, sn);
+}
+
+/// The length of the stem of a word: ing, ed, er and a plural s are cut off a long enough word.
+pub fn stem_n(w: string, wi: u32, n: u32) -> u32 {
+    if (n >= 6) {
+        if (ends_with(w, wi, n, "ing") == 1) { return n - 3; }
+    }
+    if (n >= 5) {
+        if (ends_with(w, wi, n, "ed") == 1) { return n - 2; }
+        if (ends_with(w, wi, n, "er") == 1) { return n - 2; }
+    }
+    if (n >= 4) {
+        if (ends_with(w, wi, n, "s") == 1) {
+            if (ends_with(w, wi, n, "ss") == 0) { return n - 1; }
+        }
+    }
+    return n;
+}
+
+/// 1 + the index of the ALT row whose key is the stem, or 0.
+pub fn alt_of(w: string, wi: u32, sn: u32) -> u32 {
+    var k : u32 = 0;
+    while (k < 10) {
+        if (str_len(ALT_FROM[k]) == sn) {
+            if (same_at(w, wi, ALT_FROM[k], 0, sn) == 1) { return k + 1; }
+        }
+        k = k + 1;
+    }
+    return 0;
+}
+
+/// 1 when the word w[wi..wi+n) is a stop word or a single character. A number is a content word: "sha-256" is not "sha".
+pub fn is_noise(w: string, wi: u32, n: u32) -> u32 {
+    if (n < 2) { return 1; }
+    var k : u32 = 0;
+    while (k < 61) {
+        if (str_len(STOPS[k]) == n) {
+            if (same_at(w, wi, STOPS[k], 0, n) == 1) { return 1; }
+        }
+        k = k + 1;
+    }
+    return 0;
+}
+
+/// 2 when the word (or its abbreviation) starts a token of the file name, 1 when it starts one of the path, else 0.
+pub fn word_score(w: string, wi: u32, n: u32, name: string, path: string) -> u32 {
+    var sn : u32 = stem_n(w, wi, n);
+    var whole : u32 = 0;
+    if (sn < 3) { whole = 1; }
+    if (token_hit(name, w, wi, sn, whole) == 1) { return 2; }
+    var a : u32 = alt_of(w, wi, sn);
+    if (a == 0) { a = alt_of(w, wi, n); }
+    if (a > 0) {
+        if (token_hit(name, ALT_TO[a - 1], 0, str_len(ALT_TO[a - 1]), 0) == 1) { return 2; }
+    }
+    if (token_hit(path, w, wi, sn, whole) == 1) { return 1; }
+    if (a > 0) {
+        if (token_hit(path, ALT_TO[a - 1], 0, str_len(ALT_TO[a - 1]), 0) == 1) { return 1; }
+    }
+    return 0;
+}
+
+/// n * 10000 + m * 100 + m2: content words, words found in the name or path, words found in the name.
+pub fn relevance(intent: string, name: string, path: string) -> u32 {
+    var len : u32 = str_len(intent);
+    var i : u32 = 0;
+    var n : u32 = 0;
+    var m : u32 = 0;
+    var m2 : u32 = 0;
+    while (i < len) {
+        var j : u32 = i;
+        while (j < len) {
+            if (is_word_char((intent[j] as u32)) == 0) { break; }
+            j = j + 1;
+        }
+        if (j > i) {
+            if (is_noise(intent, i, j - i) == 0) {
+                var s : u32 = word_score(intent, i, j - i, name, path);
+                n = n + 1;
+                if (s > 0) { m = m + 1; }
+                if (s == 2) { m2 = m2 + 1; }
+            }
+            i = j;
+        }
+        if (j == i) { i = i + 1; }
+    }
+    return n * 10000 + m * 100 + m2;
+}
+
+/// 1 when n content words, m of them found and m2 of those in the name, cover the intent.
+pub fn covers(n: u32, m: u32, m2: u32) -> u32 {
+    if (n == 0) { return 0; }
+    if (m2 == 0) { return 0; }
+    if (n <= 3) {
+        if (m < n) { return 0; }
+        return 1;
+    }
+    if (3 * m < 2 * n) { return 0; }
+    if (2 * m2 < n) { return 0; }
+    return 1;
+}
+
+/// What one manifest row is for an intent: V_HIT, V_UNHEALTHY / V_UNTESTED when it covers the intent but is not served, V_NONE, or V_EMPTY.
+pub fn verdict(intent: string, name: string, path: string, tags: string) -> u32 {
+    var r : u32 = relevance(intent, name, path);
+    var n : u32 = r / 10000;
+    var m : u32 = (r / 100) % 100;
+    var m2 : u32 = r % 100;
+    if (n == 0) { return V_EMPTY; }
+    if (covers(n, m, m2) == 0) { return V_NONE; }
+    if (has(tags, "health/ok") == 0) { return V_UNHEALTHY; }
+    if (has(tags, "has/tests") == 0) { return V_UNTESTED; }
+    return V_HIT;
+}
+
+/// The number of tokens of a name.
+pub fn name_tokens(name: string) -> u32 {
+    var len : u32 = str_len(name);
+    var i : u32 = 0;
+    var t : u32 = 0;
+    var inside : u32 = 0;
+    while (i < len) {
+        var w : u32 = is_word_char((name[i] as u32));
+        if (w == 1) {
+            if (inside == 0) { t = t + 1; }
+        }
+        inside = w;
+        i = i + 1;
+    }
+    return t;
+}
+
+/// The rank of a hit (larger is better), 0 for a row that is not one.
+pub fn rank_key(intent: string, name: string, path: string, tags: string, lines: u32) -> u32 {
+    if (verdict(intent, name, path, tags) != V_HIT) { return 0; }
+    var r : u32 = relevance(intent, name, path);
+    var m : u32 = (r / 100) % 100;
+    var m2 : u32 = r % 100;
+    var score : u32 = 2 * m2 + m;
+    var nt : u32 = name_tokens(name);
+    if (nt > 99) { nt = 99; }
+    var ln : u32 = lines;
+    if (ln > 999) { ln = 999; }
+    return (score * 100 + (99 - nt)) * 1000 + (999 - ln) + 1;
+}
+
+test "hit: the words of the intent are in the file name" {
+    assert verdict("clocked counter", "clocked_counter", "specs/ternary/clocked_counter.t27", "health/ok has/tests") == V_HIT;
+    assert verdict("uart echo", "uart_echo", "specs/port/trinity/fpga/openxc7-synth/uart_echo.t27", "health/ok has/tests") == V_HIT;
+    assert verdict("fifo", "fifo", "specs/fpga/fifo.t27", "health/ok has/tests") == V_HIT;
+    assert verdict("Clocked Counter", "clocked_counter", "specs/ternary/clocked_counter.t27", "health/ok has/tests") == V_HIT;
+}
+
+test "hit: stems, plurals and abbreviations" {
+    assert verdict("counters", "counter", "x/counter.t27", "health/ok has/tests") == V_HIT;
+    assert verdict("a clocked register", "clock_reg", "x/clock_reg.t27", "health/ok has/tests") == V_HIT;
+    assert verdict("reset", "rst_gen", "x/rst_gen.t27", "health/ok has/tests") == V_HIT;
+    assert verdict("synchronous fifo", "sync_fifo", "x/sync_fifo.t27", "health/ok has/tests") == V_HIT;
+}
+
+test "hit: two thirds of the words are enough from four words up, when the name carries half of them" {
+    assert verdict("uart receiver fpga", "uart_receiver", "specs/fpga/uart_receiver.t27", "health/ok has/tests") == V_HIT;
+    assert verdict("uart receiver with baud rate", "uart_receiver", "specs/fpga/uart_receiver.t27", "health/ok has/tests") == V_NONE;
+    assert verdict("uart receiver baud rate", "uart_receiver_baud", "specs/fpga/uart_receiver_baud.t27", "health/ok has/tests") == V_HIT;
+    assert verdict("fpga uart receiver baud", "baud_gen", "specs/fpga/uart/baud_gen.t27", "health/ok has/tests") == V_NONE;
+}
+
+test "miss: the contract's own example is two specs, not one" {
+    assert verdict("clocked counter with sync reset", "clocked_counter", "specs/ternary/clocked_counter.t27", "health/ok has/tests") == V_NONE;
+    assert verdict("clocked counter with sync reset", "reset_sync", "specs/fpga/reset_sync.t27", "health/ok") == V_NONE;
+}
+
+test "miss: a neighbour that shares one word and a folder is not the spec" {
+    assert verdict("ternary full adder", "ternary_full_adder", "specs/ternary/ternary_full_adder.t27", "health/ok has/tests") == V_HIT;
+    assert verdict("ternary full adder", "ternary_ripple_adder", "specs/ternary/ternary_ripple_adder.t27", "health/ok has/tests") == V_NONE;
+    assert verdict("ternary full adder", "gft_add_rne", "specs/ternary/gft_add_rne.t27", "health/ok has/tests") == V_NONE;
+    assert verdict("ternary full adder", "gft_neuron_full", "specs/ternary/gft_neuron_full.t27", "health/ok has/tests") == V_NONE;
+}
+
+test "miss: below the threshold a related spec is not returned" {
+    assert verdict("uart receiver with parity and baud generator", "uart_echo", "specs/uart_echo.t27", "health/ok has/tests") == V_NONE;
+    assert verdict("pwm", "clocked_counter", "specs/ternary/clocked_counter.t27", "health/ok has/tests") == V_NONE;
+    assert verdict("sha-256", "shaping", "specs/queen/shaping.t27", "health/ok has/tests") == V_NONE;
+    assert verdict("sha256", "sha256", "specs/tri/crypto/sha256.t27", "health/ok has/tests") == V_HIT;
+}
+
+test "negative control: a word inside another word is not a match" {
+    assert verdict("count", "account_ledger", "specs/account_ledger.t27", "health/ok has/tests") == V_NONE;
+    assert verdict("io", "gpio_pad", "specs/gpio_pad.t27", "health/ok has/tests") == V_NONE;
+    assert verdict("io", "io_pad", "specs/io_pad.t27", "health/ok has/tests") == V_HIT;
+}
+
+test "negative control: a path match alone does not carry a spec" {
+    assert verdict("ternary", "adder", "specs/ternary/adder.t27", "health/ok has/tests") == V_NONE;
+    assert verdict("ternary adder", "adder", "specs/ternary/adder.t27", "health/ok has/tests") == V_HIT;
+}
+
+test "empty: an intent with no content word searches nothing" {
+    assert verdict("", "clocked_counter", "specs/clocked_counter.t27", "health/ok has/tests") == V_EMPTY;
+    assert verdict("write me a module with tests", "clocked_counter", "specs/clocked_counter.t27", "health/ok has/tests") == V_EMPTY;
+    assert verdict("8 x", "clocked_counter", "specs/clocked_counter.t27", "health/ok has/tests") == V_EMPTY;
+}
+
+test "relevant but not served: a failing or untested spec" {
+    assert verdict("clocked counter", "clocked_counter", "x/clocked_counter.t27", "health/fail has/tests") == V_UNHEALTHY;
+    assert verdict("clocked counter", "clocked_counter", "x/clocked_counter.t27", "health/warn has/tests") == V_UNHEALTHY;
+    assert verdict("clocked counter", "clocked_counter", "x/clocked_counter.t27", "health/ok has/functions") == V_UNTESTED;
+    assert verdict("sync reset", "reset_sync", "specs/fpga/reset_sync.t27", "health/ok has/functions") == V_UNTESTED;
+}
+
+test "stem_n and the word tests" {
+    assert stem_n("clocked", 0, 7) == 5;
+    assert stem_n("counter", 0, 7) == 5;
+    assert stem_n("counting", 0, 8) == 5;
+    assert stem_n("registers", 0, 9) == 8;
+    assert stem_n("address", 0, 7) == 7;
+    assert stem_n("uart", 0, 4) == 4;
+    assert stem_n("bed", 0, 3) == 3;
+    assert is_noise("with", 0, 4) == 1;
+    assert is_noise("uart", 0, 4) == 0;
+    assert is_noise("2048", 0, 4) == 0;
+    assert is_noise("u8", 0, 2) == 0;
+    assert is_noise("a", 0, 1) == 1;
+}
+
+test "rank_key: name over path, short name over long, small over large, and 0 for a non-hit" {
+    var exact : u32 = rank_key("clocked counter", "clocked_counter", "x/clocked_counter.t27", "health/ok has/tests", 38);
+    var longer : u32 = rank_key("clocked counter", "clocked_counter_with_enable_and_load", "x/y.t27", "health/ok has/tests", 38);
+    var path_only : u32 = rank_key("clocked counter", "counter", "x/clocked/counter.t27", "health/ok has/tests", 38);
+    var bigger : u32 = rank_key("clocked counter", "clocked_counter", "x/clocked_counter.t27", "health/ok has/tests", 380);
+    assert exact > longer;
+    assert exact > path_only;
+    assert exact > bigger;
+    assert exact > 0;
+    assert rank_key("clocked counter", "clocked_counter", "x/clocked_counter.t27", "health/fail has/tests", 38) == 0;
+    assert rank_key("pwm", "clocked_counter", "x/clocked_counter.t27", "health/ok has/tests", 38) == 0;
+}
+
+test "the notes and the limit" {
+    assert LIMIT == 5;
+    assert has(HIT_NOTE, "t27_spec") == 1;
+    assert has(MISS_NOTE, "0 vacuous") == 1;
+    assert has(EMPTY_NOTE, "counter") == 1;
+}