From 02c5c1e338f2ed2c9c3743130fb39235ec64ddd6 Mon Sep 17 00:00:00 2001 From: Tony Goodhew Date: Fri, 31 Jul 2026 12:52:05 -0700 Subject: [PATCH] Read the user's own instrument manuals (#120) The command database answers first and answers fast, but it is necessarily incomplete - and the manual an entry was derived from is often sitting on the same disk. Point GPIB_MCP_MANUALS at a folder and one tool appears, manual_search, which returns the matching PASSAGES with the file and page they came from. Passages, not answers, deliberately. Turning a page of prose into "the command is CF" is the models job, done in front of the user with the quote visible; a server that synthesised commands out of manuals would be guessing with far more confidence than the evidence supports, and the thing on the other end of a wrong guess is real hardware. Every result carries its citation, and when a manual yields a command the database lacks the result says to offer instrument_db_save, so the catalogue grows from the users own documents. Reading PDFs was the deciding constraint the issue predicted. .NET Framework cannot, and bundling a PDF engine into a server whose whole shape is "no external dependencies" is a poor trade for a feature that is off by default. So: text files read directly; a sidecar .txt beside the PDF read instead; or pdftotext (Poppler/xpdf) run with -layout, which keeps the columns that make a command table readable. If none applies the result names the file and the remedy - a manual that cannot be extracted must never look like a manual with no match. Extracted text is cached, keyed by path, size and mtime. Two behaviours came from running it against the real 570-PDF, 5 GB library here, and neither would have surfaced from tests alone: - An 8563E's programming manual is filed as "8560E Programming Guide.pdf". A human would reach for the series manual, so the search does too, at a much lower rank - and the result is flagged familyMatchOnly so the substitution is stated rather than hidden. It now finds CF Center Frequency on page 434. - When nothing is named for the model, NOTHING is searched. The first cut fell back to the smallest files and dutifully searched a 0-byte PDF and readme.txt, reporting "searched 12 files, no match" - which reads as "your library does not have this" when the truth is "I never opened the right file". It now says so and lists the closest names it does have. Also: with a model given, a file must be related to THAT instrument - one shared word in a filename dragged in application notes for other boxes, seconds of extraction to answer a different question. Zero-byte files are not manuals. A caller-supplied path cannot escape the library root. 21 tests, with text fixtures rather than PDFs: the search, ranking and citations are what they are about, and a test needing Poppler installed would be testing the machine. --- README.md | 53 +++ .../Instruments/InstrumentPaths.cs | 3 +- src/GpibMcp.Core/Manuals/ManualLibrary.cs | 274 +++++++++++++++ src/GpibMcp.Core/Manuals/ManualSearch.cs | 170 ++++++++++ src/GpibMcp.Core/Manuals/ManualText.cs | 240 ++++++++++++++ src/GpibMcp.Core/Tools/InstrumentTools.cs | 6 + src/GpibMcp.Core/Tools/ManualTools.cs | 273 +++++++++++++++ tests/GpibMcp.Tests/ManualSearchTests.cs | 312 ++++++++++++++++++ 8 files changed, 1330 insertions(+), 1 deletion(-) create mode 100644 src/GpibMcp.Core/Manuals/ManualLibrary.cs create mode 100644 src/GpibMcp.Core/Manuals/ManualSearch.cs create mode 100644 src/GpibMcp.Core/Manuals/ManualText.cs create mode 100644 src/GpibMcp.Core/Tools/ManualTools.cs create mode 100644 tests/GpibMcp.Tests/ManualSearchTests.cs diff --git a/README.md b/README.md index 564e309..10fa293 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ tools the model can call to discover instruments and exchange SCPI / IEEE-488.2 - [Manual test from a terminal](#manual-test-from-a-terminal) - [Logging](#logging) - [MCP transports (stdio & HTTP)](#mcp-transports-stdio--http) +- [Your own manual library](#your-own-manual-library) - [Protocol revisions](#protocol-revisions) - [Long-running calls: progress and tasks](#long-running-calls-progress-and-tasks) - [Structured results](#structured-results) @@ -74,6 +75,9 @@ tools the model can call to discover instruments and exchange SCPI / IEEE-488.2 - **SRQ-based operation completion** — wait for an operation to *truly* finish via the bus service-request event (data-driven from the model's `statusModel`), instead of guessing with a fixed timeout. +- **Reads your own manuals** — point `GPIB_MCP_MANUALS` at a folder of instrument manuals and the server + can search them when the command database falls short, returning the passage with the file and page to + cite (PDFs via `pdftotext` or a text sidecar). - **Measurements come back as data, not prose** — the query, sweep and setting tools declare an `outputSchema` and return `structuredContent`, so a reading arrives as a number and a unit (the unit taken from the database's audited tokens, never guessed off the wire). @@ -916,6 +920,50 @@ so the single-threaded instrument access is preserved regardless of transport. $env:GPIB_MCP_TRANSPORT = "http"; $env:GPIB_MCP_HTTP_TOKEN = ""; .\GpibMcp.exe ``` +## Your own manual library + +Point the server at a folder of instrument manuals and it can read them when the command database falls +short (issue #120): + +```powershell +$env:GPIB_MCP_MANUALS = "C:\Users\me\Documents\Manuals" +``` + +That registers one tool, **`manual_search`**, which returns the matching **passages with the file and page +they came from** — not an answer. Deriving "the command is `CF`" from a page of prose is the model's job, +done in front of you, with the quoted text visible. A server that synthesised commands out of manuals would +be guessing with far more confidence than the evidence supports, and the thing on the other end of a wrong +guess is your hardware. + +Resolution order is unchanged: **`instrument_reference` first** (instant, structured, already carries the +audited unit tokens), then the manuals, then whatever the client can do on its own. When a manual yields a +command the database lacks, the result says to offer `instrument_db_save` — so the catalogue grows from your +own documents, and the next lookup is instant. + +| Variable | Purpose | +|---|---| +| `GPIB_MCP_MANUALS` | folder of manuals (searched recursively). Unset = the tool isn't registered at all | +| `GPIB_MCP_PDFTOTEXT` | full path to `pdftotext`, if it isn't on `PATH` | +| `GPIB_MCP_MANUAL_CACHE` | where extracted text is cached (default `%LOCALAPPDATA%\GpibMcp\manual-text`) | + +**Reading PDFs.** .NET Framework can't, and bundling a PDF engine into a server whose whole shape is "no +external dependencies" is a poor trade for a feature that's off by default. So there are three routes, tried +in order: the file is already `.txt`/`.md`; a sidecar `.txt` sits beside the PDF; or **`pdftotext`** +(Poppler/xpdf) is on `PATH`, run with `-layout` so command tables keep their columns. If none applies, the +result says *which* file couldn't be read and how to fix it — a manual that can't be extracted must never +look like a manual with no match. Extracted text is cached, keyed by path, size and modification time. + +**How it finds the right manual.** By filename first — a library is hundreds of large PDFs and extracting +them all would take minutes and return noise. Pass `model=` and it reads only that instrument's manuals. +Two behaviours worth knowing, both found by running it against a real 570-PDF library: + +- **Series manuals count.** An 8563E's programming manual is often filed as `8560E Programming Guide.pdf`. A + human would reach for it, so the search does too — at a much lower rank, and the result is flagged + `familyMatchOnly` so the substitution gets stated rather than hidden. +- **If nothing is named for the model, nothing is searched.** It reports that, and lists the closest names it + does have. Reading whichever files happened to be smallest would produce "searched 12 files, no match", + which reads as *your library doesn't have this* when the truth is *I never opened the right file*. + ## Protocol revisions The server implements MCP **2026-07-28** and speaks **`2026-07-28`, `2025-06-18`, `2025-03-26` and @@ -1102,6 +1150,10 @@ src/GpibMcp.Core/ backend-neutral core (no driver dependency; b ServerTask.cs one task's state + its CreateTaskResult / tasks/get shapes TaskStore.cs the live tasks, TTL-bounded TaskRunner.cs single worker thread - keeps the GPIB bus serial + Manuals/ the user's own manual folder (#120) + ManualLibrary.cs which files in it match a question (filename-first) + ManualText.cs text extraction (sidecar / pdftotext) + on-disk cache + ManualSearch.cs passage search with page-numbered citations Instruments/ IInstrumentManager.cs tool-facing instrument abstraction (enables testing) InstrumentManager.cs backend-neutral manager (history, errors, capture, the bus lock) @@ -1119,6 +1171,7 @@ src/GpibMcp.Core/ backend-neutral core (no driver dependency; b Tools/ ToolArgs.cs shared JSON-Schema + argument helpers MeasurementValue.cs reply -> number, and its unit from the audited DB tokens (#113) + ManualTools.cs manual_search over the user's own manual folder (#120) InstrumentIo.cs resolves a model's IoSpec (terminators + bounded read) InstrumentTools.cs VISA / native-GPIB + serial-poll / wait-SRQ tools DatabaseTools.cs command-database + assignment + set_termination tools diff --git a/src/GpibMcp.Core/Instruments/InstrumentPaths.cs b/src/GpibMcp.Core/Instruments/InstrumentPaths.cs index 7879968..bb72ddc 100644 --- a/src/GpibMcp.Core/Instruments/InstrumentPaths.cs +++ b/src/GpibMcp.Core/Instruments/InstrumentPaths.cs @@ -119,7 +119,8 @@ public static void EnsureUserDatabaseSeeded(string exeDir) } } - private static string AppDataDir() => + /// The server's per-user data folder: %LOCALAPPDATA%\GpibMcp. + public static string AppDataDir() => Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "GpibMcp"); } } diff --git a/src/GpibMcp.Core/Manuals/ManualLibrary.cs b/src/GpibMcp.Core/Manuals/ManualLibrary.cs new file mode 100644 index 0000000..14dfd21 --- /dev/null +++ b/src/GpibMcp.Core/Manuals/ManualLibrary.cs @@ -0,0 +1,274 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace GpibMcp.Manuals +{ + /// + /// A user's folder of instrument manuals (#120), and the rules for finding which files in it are worth + /// reading for a given question. + /// + /// Narrowing by filename first is not an optimisation detail, it is the design. A real library is + /// hundreds of large PDFs - extracting all of them to answer one question would take minutes and produce + /// mostly noise. Manuals are almost always named by model ("8560E Programming Guide.pdf", or a folder + /// "3458A" holding its set), so the model name is a strong, cheap filter that gets to the right handful + /// before any file is opened. + /// + /// The feature is off unless a folder is configured: no folder, no tool. + /// + public sealed class ManualLibrary + { + /// Extensions worth reading. Text needs no extraction; PDF needs . + private static readonly string[] ReadableExtensions = { ".pdf", ".txt", ".md", ".text" }; + + /// + /// Ceiling on files opened for one search when the caller gave no model to narrow by. Bounded so a + /// vague question cannot walk a 5 GB library; the search reports when it hits this, because a + /// silently truncated search reads as "not in the manuals" when it means "we stopped looking". + /// + public const int MaxFilesPerSearch = 12; + + private readonly string _root; + + private ManualLibrary(string root) { _root = root; } + + /// The configured folder, or null when the feature is off. + public string Root => _root; + + /// + /// Opens the library named by GPIB_MCP_MANUALS, or returns null when it is unset or points + /// nowhere. A configured-but-missing folder is worth a warning rather than silence - it is almost + /// always a typo, and the alternative is a tool that quietly never finds anything. + /// + public static ManualLibrary FromEnvironment() + { + string configured = Environment.GetEnvironmentVariable("GPIB_MCP_MANUALS"); + if (string.IsNullOrWhiteSpace(configured)) return null; + + string root = configured.Trim().Trim('"'); + if (!Directory.Exists(root)) + { + Diagnostics.Log.Warn("GPIB_MCP_MANUALS points at '" + root + "', which does not exist; " + + "manual lookup is disabled."); + return null; + } + return new ManualLibrary(root); + } + + /// Opens a library at an explicit path (tests, and callers that configure it directly). + public static ManualLibrary At(string root) => + string.IsNullOrWhiteSpace(root) || !Directory.Exists(root) ? null : new ManualLibrary(root.Trim()); + + /// + /// The files worth reading for this question, best candidates first. When a model is given, files + /// whose name or folder mentions it come first and nothing else is opened unless there are none; + /// otherwise the query's own words are matched against filenames. + /// + public IReadOnlyList Candidates(string model, string query, int limit = MaxFilesPerSearch) + { + bool haveModel = !string.IsNullOrWhiteSpace(model); + + return Enumerate() + .Select(f => new { File = f, Score = NameScore(f, model, query) }) + // With a model in hand, a file must be related to THAT instrument. Otherwise one shared word + // in a filename - "frequency" appears in half a library - drags in application notes for + // other instruments, which cost seconds to extract and answer a different question. + .Where(x => x.Score > 0 && (!haveModel || ModelScore(x.File, model) > 0)) + .OrderByDescending(x => x.Score) + .ThenBy(x => x.File.SizeBytes) // a short manual is likelier to be the relevant one + .Select(x => x.File) + .Take(Math.Max(1, limit)) + .ToList(); + + // Deliberately no fallback to "read some files anyway". Reading arbitrary manuals to answer a + // question about an instrument they do not cover costs seconds and returns noise - and worse, + // it reports "searched 12 files, no match", which reads as "your library does not have this" + // when the truth is "I never looked at the right file". Better to say nothing matched and show + // what is there. + } + + /// A sample of what the library holds, so a caller told "nothing matched" can retry usefully. + public IReadOnlyList SampleNames(string model, int limit = 20) + { + IEnumerable files = Enumerate(); + + // With a model in hand, prefer names that at least share its leading digits - "no 8563E manual, + // but here are the 856x ones" is a far better prompt than an alphabetical slice of the library. + string stem = FamilyStem(model); + if (stem != null) + { + var near = files.Where(f => f.RelativePath.ToLowerInvariant().Contains(stem)).ToList(); + if (near.Count > 0) files = near; + } + + return files.Select(f => f.RelativePath).OrderBy(n => n, StringComparer.OrdinalIgnoreCase) + .Take(Math.Max(1, limit)).ToList(); + } + + /// Total readable files, for telling the caller the size of what was not searched. + public int Count() => Enumerate().Count(); + + /// Every readable file in the library. + public IEnumerable Enumerate() + { + IEnumerable paths; + try { paths = Directory.EnumerateFiles(_root, "*", SearchOption.AllDirectories); } + catch (Exception ex) + { + Diagnostics.Log.Warn("Could not read the manual library at '" + _root + "': " + ex.Message); + yield break; + } + + foreach (string path in paths) + { + string ext = Path.GetExtension(path); + if (Array.IndexOf(ReadableExtensions, ext.ToLowerInvariant()) < 0) continue; + + ManualFile file = null; + try { file = new ManualFile(path, _root); } + catch (Exception) { /* vanished or unreadable between enumerate and stat */ } + + // A zero-byte file is a failed download, not a manual. + if (file != null && file.SizeBytes > 0) yield return file; + } + } + + /// + /// Resolves a caller-supplied path against the library, refusing anything outside it. The path comes + /// from a tool argument, so "../../../secrets.txt" has to bounce off something. + /// + public ManualFile Resolve(string relativeOrFullPath) + { + if (string.IsNullOrWhiteSpace(relativeOrFullPath)) return null; + + string candidate = relativeOrFullPath.Trim().Trim('"'); + string full; + try + { + full = Path.GetFullPath(Path.IsPathRooted(candidate) + ? candidate + : Path.Combine(_root, candidate)); + } + catch (Exception) { return null; } + + string rootFull = Path.GetFullPath(_root).TrimEnd(Path.DirectorySeparatorChar) + + Path.DirectorySeparatorChar; + if (!full.StartsWith(rootFull, StringComparison.OrdinalIgnoreCase)) return null; + if (!File.Exists(full)) return null; + + return new ManualFile(full, _root); + } + + /// + /// How well a file's name matches the question. The model is worth far more than a query word: a + /// filename containing "8563E" is almost certainly that instrument's manual, whereas one containing + /// "frequency" is barely evidence at all. + /// + private static int NameScore(ManualFile file, string model, string query) + { + string haystack = file.RelativePath.Replace('\\', ' ').Replace('/', ' ').ToLowerInvariant(); + int score = ModelScore(file, model); + + foreach (string word in Words(query)) + if (word.Length >= 3 && haystack.Contains(word)) score += 5; + + return score; + } + + /// + /// How strongly a file's name ties it to : named for it, named for it without + /// a trailing option letter, or merely of its family - in descending order of confidence, and zero + /// for no relation at all. + /// + private static int ModelScore(ManualFile file, string model) + { + if (string.IsNullOrWhiteSpace(model)) return 0; + + string haystack = file.RelativePath.Replace('\\', ' ').Replace('/', ' ').ToLowerInvariant(); + string m = model.Trim().ToLowerInvariant(); + + if (haystack.Contains(m)) return 100; + + // Models are often written with a trailing option letter the file omits (54622D -> 54622). + if (m.Length > 3 && haystack.Contains(m.Substring(0, m.Length - 1))) return 60; + + // Family match, worth much less: an 8563E's programming manual is often filed as the series - + // "8560E Programming Guide" - and a human looking for it would reach for that too. Low score so + // a genuine model match always wins, and the result says the substitution happened. + string stem = FamilyStem(m); + return stem != null && haystack.Contains(stem) ? 25 : 0; + } + + /// True when this file is named for the model itself, not merely its family. + public static bool MatchesModel(ManualFile file, string model) => + file != null && ModelScore(file, model) >= 60; + + /// + /// The leading digits that identify an instrument family - "8563E" and "8560E" share "856". Null when + /// the model has no usable numeric stem, in which case there is no family to guess at. + /// + public static string FamilyStem(string model) + { + if (string.IsNullOrWhiteSpace(model)) return null; + + string digits = new string(model.Trim().TakeWhile(char.IsDigit).ToArray()); + return digits.Length >= 3 ? digits.Substring(0, 3) : null; + } + + /// True when a candidate was found only by family resemblance, not by naming the model. + public static bool IsFamilyMatchOnly(ManualFile file, string model) + { + if (file == null || string.IsNullOrWhiteSpace(model)) return false; + + string haystack = file.RelativePath.ToLowerInvariant(); + string m = model.Trim().ToLowerInvariant(); + if (haystack.Contains(m)) return false; + if (m.Length > 3 && haystack.Contains(m.Substring(0, m.Length - 1))) return false; + + string stem = FamilyStem(m); + return stem != null && haystack.Contains(stem); + } + + /// Splits a query into lower-cased words worth matching. + public static IEnumerable Words(string query) + { + if (string.IsNullOrWhiteSpace(query)) yield break; + foreach (string raw in query.Split(new[] { ' ', '\t', '\r', '\n', ',', ';', ':', '(', ')', '"', '\'' }, + StringSplitOptions.RemoveEmptyEntries)) + { + string word = raw.Trim().ToLowerInvariant(); + if (word.Length > 0) yield return word; + } + } + } + + /// One file in the manual library. + public sealed class ManualFile + { + public ManualFile(string fullPath, string root) + { + FullPath = fullPath; + var info = new FileInfo(fullPath); + SizeBytes = info.Length; + ModifiedUtc = info.LastWriteTimeUtc; + + string rootFull = Path.GetFullPath(root).TrimEnd(Path.DirectorySeparatorChar) + + Path.DirectorySeparatorChar; + RelativePath = fullPath.StartsWith(rootFull, StringComparison.OrdinalIgnoreCase) + ? fullPath.Substring(rootFull.Length) + : Path.GetFileName(fullPath); + } + + public string FullPath { get; } + + /// Path within the library - what a citation shows, and what the caller passes back. + public string RelativePath { get; } + + public long SizeBytes { get; } + public DateTime ModifiedUtc { get; } + + public bool IsPdf => + string.Equals(Path.GetExtension(FullPath), ".pdf", StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/GpibMcp.Core/Manuals/ManualSearch.cs b/src/GpibMcp.Core/Manuals/ManualSearch.cs new file mode 100644 index 0000000..b5e0542 --- /dev/null +++ b/src/GpibMcp.Core/Manuals/ManualSearch.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace GpibMcp.Manuals +{ + /// + /// Finds the passages in a manual that answer a question (#120), and cites where each came from. + /// + /// This deliberately returns passages, not answers. Deriving "the command is CF" from a page of + /// prose is the model's job, done in front of the user who can see the quoted text; a server that + /// synthesised commands out of manuals would be guessing with far more confidence than the evidence + /// supports, and the thing on the other end of a wrong guess is real hardware. + /// + public static class ManualSearch + { + /// Characters of context returned either side of a hit. + public const int DefaultContextChars = 400; + + public sealed class Hit + { + public string File { get; set; } + public int Page { get; set; } + public int Score { get; set; } + public string Text { get; set; } + } + + public sealed class FileNote + { + public string File { get; set; } + public string Problem { get; set; } + } + + public sealed class Results + { + public List Hits { get; } = new List(); + + /// Files that could not be read, and why - never silently dropped. + public List Unreadable { get; } = new List(); + + public List Searched { get; } = new List(); + + /// True when the file list was cut to the cap - the caller must be told. + public bool Truncated { get; set; } + } + + /// + /// Searches for , best passages first. + /// + /// Progress callback: (index, total, file being read). + public static Results Run(IReadOnlyList files, string query, int maxHits = 5, + int contextChars = DefaultContextChars, Action onFile = null) + { + var results = new Results(); + if (files == null || files.Count == 0) return results; + + string[] words = ManualLibrary.Words(query).Where(w => w.Length >= 2).Distinct().ToArray(); + if (words.Length == 0) return results; + + var all = new List(); + for (int i = 0; i < files.Count; i++) + { + ManualFile file = files[i]; + if (onFile != null) onFile(i, files.Count, file.RelativePath); + + ManualText.Result text = ManualText.Read(file); + if (!text.Ok) + { + results.Unreadable.Add(new FileNote { File = file.RelativePath, Problem = text.Detail }); + continue; + } + + results.Searched.Add(file.RelativePath); + all.AddRange(FindIn(file, text.Text, words, contextChars)); + } + + results.Hits.AddRange(all + .OrderByDescending(h => h.Score) + .Take(Math.Max(1, maxHits))); + return results; + } + + /// + /// Scores each occurrence of the rarest query word by how many of the other words appear nearby. + /// Anchoring on the rarest word is what makes a search for "center frequency CF" land on the page + /// defining CF rather than the hundreds of pages that merely say "frequency". + /// + private static IEnumerable FindIn(ManualFile file, string text, string[] words, int contextChars) + { + string lower = text.ToLowerInvariant(); + + string anchor = words.OrderBy(w => CountOccurrences(lower, w)).First(); + int anchorCount = CountOccurrences(lower, anchor); + if (anchorCount == 0) yield break; + + // A word that appears everywhere is not evidence of anything; stop rather than return noise. + const int TooCommon = 400; + if (anchorCount > TooCommon) yield break; + + int window = Math.Max(contextChars, 200); + int from = 0; + var seenPages = new HashSet(); + + while (true) + { + int at = lower.IndexOf(anchor, from, StringComparison.Ordinal); + if (at < 0) yield break; + from = at + anchor.Length; + + int start = Math.Max(0, at - window / 2); + int length = Math.Min(window, text.Length - start); + string snippet = text.Substring(start, length); + string snippetLower = snippet.ToLowerInvariant(); + + int score = 10; + foreach (string word in words) + if (word != anchor && snippetLower.Contains(word)) score += 20; + + // One hit per page: consecutive matches on a page are the same passage to a reader. + int page = ManualText.PageAt(text, at); + if (!seenPages.Add(page)) continue; + + yield return new Hit + { + File = file.RelativePath, + Page = page, + Score = score, + Text = Tidy(snippet) + }; + } + } + + private static int CountOccurrences(string haystack, string needle) + { + int count = 0, at = 0; + while ((at = haystack.IndexOf(needle, at, StringComparison.Ordinal)) >= 0) + { + count++; + at += needle.Length; + } + return count; + } + + /// + /// Collapses the whitespace that -layout extraction leaves behind, while keeping line breaks: + /// a command table read as one run-on line is useless, and read with 40 spaces between columns is + /// mostly padding. + /// + private static string Tidy(string snippet) + { + var sb = new StringBuilder(snippet.Length); + int spaces = 0; + + foreach (char c in snippet) + { + if (c == ManualText.PageBreak) { sb.Append('\n'); spaces = 0; continue; } + if (c == '\n' || c == '\r') { if (sb.Length > 0 && sb[sb.Length - 1] != '\n') sb.Append('\n'); spaces = 0; continue; } + if (c == ' ' || c == '\t') + { + if (++spaces <= 2) sb.Append(' '); + continue; + } + spaces = 0; + sb.Append(c); + } + return sb.ToString().Trim(); + } + } +} diff --git a/src/GpibMcp.Core/Manuals/ManualText.cs b/src/GpibMcp.Core/Manuals/ManualText.cs new file mode 100644 index 0000000..d53e9b1 --- /dev/null +++ b/src/GpibMcp.Core/Manuals/ManualText.cs @@ -0,0 +1,240 @@ +using System; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using GpibMcp.Instruments; + +namespace GpibMcp.Manuals +{ + /// + /// Turns a manual into searchable text (#120) - the part that decides what this feature can do at all. + /// + /// .NET Framework cannot read a PDF on its own, and bundling a PDF engine into a server whose whole shape + /// is "no external dependencies" would be a poor trade for a feature that is off by default. So there are + /// three routes, in order: + /// + /// + /// the file is already text (.txt/.md) - read it; + /// a sidecar <name>.txt sits beside the PDF - read that (how a user without the tool + /// below can still use this feature: extract once, however they like, and leave the text there); + /// pdftotext is on PATH (Poppler, xpdf) - run it with -layout, which keeps the column + /// alignment that command tables in instrument manuals depend on. + /// + /// + /// If none applies the answer is an explicit "this file could not be read, here is how to fix it", never + /// silence - a manual that cannot be extracted looking identical to a manual with no match would make the + /// whole tool untrustworthy. + /// + /// Extraction is cached under %LOCALAPPDATA%\GpibMcp\manual-text, keyed by path, size and + /// modification time, because it is slow and the same manual is read over and over. + /// + public static class ManualText + { + /// Page separator emitted by pdftotext; how a citation gets a page number. + public const char PageBreak = '\f'; + + /// Extraction is bounded: a runaway converter must not hang a tool call. + private const int ExtractTimeoutMs = 60000; + + /// Why a file has no text, when it has none. + public enum Outcome + { + Extracted, + NoExtractorAvailable, + ExtractionFailed + } + + public sealed class Result + { + public Result(Outcome outcome, string text, string detail) + { + Outcome = outcome; + Text = text ?? string.Empty; + Detail = detail; + } + + public Outcome Outcome { get; } + public string Text { get; } + + /// Human-readable reason, when there is no text. + public string Detail { get; } + + public bool Ok => Outcome == Outcome.Extracted; + } + + /// Reads as text, using the cache when it is still valid. + public static Result Read(ManualFile file) + { + if (file == null) return new Result(Outcome.ExtractionFailed, null, "no file"); + + if (!file.IsPdf) + { + try { return new Result(Outcome.Extracted, File.ReadAllText(file.FullPath), null); } + catch (Exception ex) + { + return new Result(Outcome.ExtractionFailed, null, "could not read the file: " + ex.Message); + } + } + + string cached = ReadCache(file); + if (cached != null) return new Result(Outcome.Extracted, cached, null); + + // A sidecar means someone already did the extraction; trust it over re-running a converter. + string sidecar = Path.ChangeExtension(file.FullPath, ".txt"); + if (File.Exists(sidecar)) + { + try + { + string text = File.ReadAllText(sidecar); + WriteCache(file, text); + return new Result(Outcome.Extracted, text, null); + } + catch (Exception ex) + { + Diagnostics.Log.Debug("Sidecar '" + sidecar + "' unreadable: " + ex.Message); + } + } + + string converter = FindPdfToText(); + if (converter == null) + return new Result(Outcome.NoExtractorAvailable, null, + "this is a PDF and no text extractor is available. Install Poppler or xpdf so that " + + "'pdftotext' is on PATH (or set GPIB_MCP_PDFTOTEXT to its full path), or place an " + + "extracted '" + Path.GetFileNameWithoutExtension(file.FullPath) + ".txt' next to it."); + + try + { + string text = RunPdfToText(converter, file.FullPath); + if (string.IsNullOrWhiteSpace(text)) + return new Result(Outcome.ExtractionFailed, null, + "the extractor produced no text - the manual is most likely a scan, which would need OCR."); + + WriteCache(file, text); + return new Result(Outcome.Extracted, text, null); + } + catch (Exception ex) + { + return new Result(Outcome.ExtractionFailed, null, "extraction failed: " + ex.Message); + } + } + + /// The 1-based page a character offset falls on, by counting page breaks before it. + public static int PageAt(string text, int offset) + { + if (string.IsNullOrEmpty(text) || offset <= 0) return 1; + + int page = 1; + int end = Math.Min(offset, text.Length); + for (int i = 0; i < end; i++) if (text[i] == PageBreak) page++; + return page; + } + + /// The extractor to use, or null when there is none. + public static string FindPdfToText() + { + string configured = Environment.GetEnvironmentVariable("GPIB_MCP_PDFTOTEXT"); + if (!string.IsNullOrWhiteSpace(configured)) + { + string path = configured.Trim().Trim('"'); + return File.Exists(path) ? path : null; + } + + foreach (string dir in (Environment.GetEnvironmentVariable("PATH") ?? "") + .Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries)) + { + string candidate; + try { candidate = Path.Combine(dir.Trim().Trim('"'), "pdftotext.exe"); } + catch (Exception) { continue; } // a malformed PATH entry + if (File.Exists(candidate)) return candidate; + } + return null; + } + + private static string RunPdfToText(string converter, string pdfPath) + { + string output = Path.Combine(Path.GetTempPath(), "gpibmcp-manual-" + Guid.NewGuid().ToString("N") + ".txt"); + try + { + // -layout preserves column alignment, which is what makes a command table readable. + var psi = new ProcessStartInfo(converter, "-layout -q \"" + pdfPath + "\" \"" + output + "\"") + { + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardError = true + }; + + using (var process = Process.Start(psi)) + { + if (process == null) throw new Exception("could not start '" + converter + "'"); + string stderr = process.StandardError.ReadToEnd(); + if (!process.WaitForExit(ExtractTimeoutMs)) + { + try { process.Kill(); } catch { /* already gone */ } + throw new Exception("the extractor did not finish within " + (ExtractTimeoutMs / 1000) + "s"); + } + if (process.ExitCode != 0) + throw new Exception("the extractor failed (exit " + process.ExitCode + ")" + + (string.IsNullOrWhiteSpace(stderr) ? "" : ": " + stderr.Trim())); + } + + return File.Exists(output) ? File.ReadAllText(output) : string.Empty; + } + finally + { + try { if (File.Exists(output)) File.Delete(output); } catch { /* best effort */ } + } + } + + // ---- cache ---------------------------------------------------------- + + /// Where extracted text is kept, so a manual is converted once rather than per search. + public static string CacheDirectory() + { + string env = Environment.GetEnvironmentVariable("GPIB_MCP_MANUAL_CACHE"); + if (!string.IsNullOrWhiteSpace(env)) return env.Trim(); + return Path.Combine(InstrumentPaths.AppDataDir(), "manual-text"); + } + + private static string CachePath(ManualFile file) + { + // Keyed by identity AND state: a re-scanned or replaced manual must not serve stale text. + string key = file.FullPath.ToLowerInvariant() + "|" + file.SizeBytes + "|" + + file.ModifiedUtc.Ticks.ToString(CultureInfo.InvariantCulture); + + using (var sha = SHA256.Create()) + { + byte[] hash = sha.ComputeHash(Encoding.UTF8.GetBytes(key)); + var name = new StringBuilder(32); + for (int i = 0; i < 16; i++) name.Append(hash[i].ToString("x2", CultureInfo.InvariantCulture)); + return Path.Combine(CacheDirectory(), name + ".txt"); + } + } + + private static string ReadCache(ManualFile file) + { + try + { + string path = CachePath(file); + return File.Exists(path) ? File.ReadAllText(path) : null; + } + catch (Exception) { return null; } + } + + private static void WriteCache(ManualFile file, string text) + { + try + { + string path = CachePath(file); + Directory.CreateDirectory(Path.GetDirectoryName(path)); + File.WriteAllText(path, text); + } + catch (Exception ex) + { + // A cache that cannot be written is slow, not broken. + Diagnostics.Log.Debug("Could not cache extracted text: " + ex.Message); + } + } + } +} diff --git a/src/GpibMcp.Core/Tools/InstrumentTools.cs b/src/GpibMcp.Core/Tools/InstrumentTools.cs index 65c25ad..1df88b2 100644 --- a/src/GpibMcp.Core/Tools/InstrumentTools.cs +++ b/src/GpibMcp.Core/Tools/InstrumentTools.cs @@ -4,6 +4,7 @@ using System.Text; using GpibMcp.Diagnostics; using GpibMcp.Instruments; +using GpibMcp.Manuals; using GpibMcp.Mcp; using Srq.Completion; using Newtonsoft.Json; @@ -455,6 +456,11 @@ public static ToolRegistry BuildRegistry(IInstrumentManager visa, InstrumentData // ---- Send a captured hardcopy to a Windows printer (#83) ------------ PrintTools.Register(registry); + // ---- Search the user's own manual library (#120) -------------------- + // Registered only when GPIB_MCP_MANUALS names a real folder: a server with no library must not + // advertise a tool that can only ever answer "nothing configured". + ManualTools.Register(registry, ManualLibrary.FromEnvironment()); + return registry; } diff --git a/src/GpibMcp.Core/Tools/ManualTools.cs b/src/GpibMcp.Core/Tools/ManualTools.cs new file mode 100644 index 0000000..cbf7abb --- /dev/null +++ b/src/GpibMcp.Core/Tools/ManualTools.cs @@ -0,0 +1,273 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using GpibMcp.Manuals; +using GpibMcp.Mcp; +using Newtonsoft.Json.Linq; +using static GpibMcp.Tools.ToolArgs; + +namespace GpibMcp.Tools +{ + /// + /// Searching the user's own folder of instrument manuals (#120). + /// + /// The command database answers first and answers fast. This is what to do when it cannot: the manual a + /// database entry was derived from is often sitting on the same disk, and reading it is far better than + /// guessing a command at an instrument. The tool returns the passages and cites them; deciding what they + /// mean is the model's job, in front of the user, who can read the quote. + /// + /// Registered only when a library is configured, so a server without one carries no tool that could + /// promise something it cannot do. + /// + public static class ManualTools + { + public static void Register(ToolRegistry registry, ManualLibrary library) + { + if (registry == null || library == null) return; + + registry.Add(new McpTool( + "manual_search", + "Search the user's local folder of instrument manuals and return the matching passages, with " + + "the file and page they came from. Use this when instrument_reference does NOT have the " + + "command you need - a model that isn't in the database, a command that isn't listed, or a " + + "detail (bit meanings, a status byte, a plot command's arguments) the entry doesn't cover. " + + "ALWAYS try instrument_reference first: it is instant and already structured. " + + "ALWAYS pass 'model' when you know it - the library can be hundreds of manuals and the model " + + "is what narrows it to the right ones. " + + "This returns SOURCE TEXT, not an answer: read the passage, tell the user what it says, and " + + "CITE the file and page. If it gives you a command the database lacks, offer to add it with " + + "instrument_db_save so the next lookup is instant. Do NOT send a command to an instrument " + + "purely on your own reading of a passage without telling the user where it came from.", + Schema( + Required("query", "string", "What to look for, e.g. 'center frequency command' or " + + "'status byte bit 4' or 'OUTPPLOT'. Include the mnemonic if you know it."), + Prop("model", "string", "Model whose manuals to search, e.g. '8563E'. Strongly preferred: " + + "without it every manual in the library is a candidate and the search is capped."), + Prop("file", "string", "A specific manual to search, as returned in an earlier result " + + "(path relative to the library root). Use to read more of a file that already matched."), + Prop("max_results", "integer", "Passages to return (default 5, max 20)."), + Prop("context_chars", "integer", "Characters of context per passage (default 400, max 4000).")), + (Func)((args, ctx) => Search(args, ctx, library))) + .WithOutputSchema(OutputSchema)); + } + + private static ToolOutput Search(JObject args, ToolCallContext ctx, ManualLibrary library) + { + string query = ReqStr(args, "query"); + string model = Str(args, "model", null); + string file = Str(args, "file", null); + int maxResults = Math.Min(Math.Max(Int(args, "max_results", 5), 1), 20); + int contextChars = Math.Min(Math.Max(Int(args, "context_chars", ManualSearch.DefaultContextChars), 100), 4000); + + IReadOnlyList candidates; + if (!string.IsNullOrWhiteSpace(file)) + { + ManualFile one = library.Resolve(file); + if (one == null) + return Failed("No manual at '" + file + "' inside the library (" + library.Root + ")."); + candidates = new[] { one }; + } + else + { + candidates = library.Candidates(model, query); + } + + if (candidates.Count == 0) return NothingMatched(library, query, model); + + ctx.Progress(1, candidates.Count + 1, "Searching " + candidates.Count + " manual(s)."); + ManualSearch.Results results = ManualSearch.Run(candidates, query, maxResults, contextChars, + (index, total, name) => ctx.Progress(index + 1, total + 1, "Reading " + name + ".")); + ctx.Progress(candidates.Count + 1, candidates.Count + 1, "Search complete."); + + // Nothing named for the instrument itself was read, only its series - a substitution, and one + // the user has to be told about. Judged on whether ANY candidate named the model, so a single + // unrelated file cannot silence the caveat. + bool familyOnly = !string.IsNullOrEmpty(model) && + !candidates.Any(c => ManualLibrary.MatchesModel(c, model)) && + candidates.Any(c => ManualLibrary.IsFamilyMatchOnly(c, model)); + return Deliver(library, query, model, results, familyOnly); + } + + /// + /// No manual's name matched. Saying "searched 12 files, nothing found" after reading whichever files + /// happened to be smallest would be worse than useless - it reads as "your library does not have + /// this". Say what actually happened, and show what is there so the next call can aim. + /// + private static ToolOutput NothingMatched(ManualLibrary library, string query, string model) + { + IReadOnlyList sample = library.SampleNames(model); + int total = library.Count(); + + var structured = new JObject + { + ["ok"] = true, + ["query"] = query, + ["library"] = library.Root, + ["hits"] = new JArray(), + ["searched"] = new JArray(), + ["unreadable"] = new JArray(), + ["available"] = new JArray(sample.Cast().ToArray()) + }; + if (!string.IsNullOrEmpty(model)) structured["model"] = model; + + var text = new StringBuilder(); + text.AppendLine("No manual in " + library.Root + " is named for " + + (string.IsNullOrEmpty(model) ? "this query" : "'" + model + "'") + + ", so nothing was searched (" + total + " manual(s) in the library)."); + if (sample.Count > 0) + { + text.AppendLine(); + text.AppendLine(string.IsNullOrEmpty(model) + ? "Some of what is there:" + : "Closest by name - if one of these covers " + model + ", call again with file=:"); + foreach (string name in sample) text.AppendLine(" " + name); + } + text.AppendLine(); + text.AppendLine("Tell the user the manual does not appear to be in their library rather than " + + "guessing the command - or ask which of the above to read."); + + return ToolOutput.Text(text.ToString().TrimEnd()).WithStructured(structured); + } + + private static ToolOutput Deliver(ManualLibrary library, string query, string model, + ManualSearch.Results results, bool familyOnly) + { + var structured = new JObject + { + ["ok"] = true, + ["query"] = query, + ["library"] = library.Root, + ["searched"] = new JArray(results.Searched.Cast().ToArray()), + ["hits"] = new JArray(results.Hits.Select(h => (JToken)new JObject + { + ["file"] = h.File, + ["page"] = h.Page, + ["text"] = h.Text + })), + ["unreadable"] = new JArray(results.Unreadable.Select(u => (JToken)new JObject + { + ["file"] = u.File, + ["problem"] = u.Problem + })) + }; + if (!string.IsNullOrEmpty(model)) structured["model"] = model; + if (familyOnly) structured["familyMatchOnly"] = true; + + var text = new StringBuilder(); + if (familyOnly) + { + // The user asked about one instrument and got its series' manual. Usually right - families + // share a command set - but it is a substitution, and substitutions get said out loud. + text.AppendLine("NOTE: no manual is named for " + model + " exactly; these are from the same " + + "series. Command sets usually match across a series, but say so when you " + + "quote this, and check the passage really covers " + model + "."); + text.AppendLine(); + } + + if (results.Hits.Count == 0) + { + text.AppendLine("No passage matched \"" + query + "\" in " + results.Searched.Count + + " manual(s) searched under " + library.Root + "."); + if (string.IsNullOrEmpty(model)) + text.AppendLine("Searching without a model is a wide net - pass model= to narrow it to " + + "that instrument's manuals."); + } + else + { + text.AppendLine(results.Hits.Count + " passage(s) for \"" + query + "\"" + + (string.IsNullOrEmpty(model) ? "" : " (" + model + ")") + ":"); + foreach (ManualSearch.Hit hit in results.Hits) + { + text.AppendLine(); + text.AppendLine("--- " + hit.File + ", page " + hit.Page + " ---"); + text.AppendLine(hit.Text); + } + text.AppendLine(); + text.AppendLine("CITE the file and page when you tell the user what this says. If it gives a " + + "command the database lacks, offer to add it with instrument_db_save."); + } + + // Never let an unreadable manual look like a manual with no match. + if (results.Unreadable.Count > 0) + { + text.AppendLine(); + text.AppendLine("Could not read " + results.Unreadable.Count + " file(s):"); + foreach (ManualSearch.FileNote note in results.Unreadable.Take(5)) + text.AppendLine(" " + note.File + " - " + note.Problem); + text.AppendLine("Tell the user this: the answer may be in a manual that could not be searched."); + } + + return ToolOutput.Text(text.ToString().TrimEnd()).WithStructured(structured); + } + + private static ToolOutput Failed(string message) => + ToolOutput.Text(message) + .WithStructured(new JObject { ["ok"] = false, ["error"] = message }) + .AsError(); + + private static JObject OutputSchema => new JObject + { + ["type"] = "object", + ["description"] = "Passages found in the user's manual library, each citing its file and page.", + ["properties"] = new JObject + { + ["ok"] = new JObject { ["type"] = "boolean" }, + ["query"] = new JObject { ["type"] = "string" }, + ["model"] = new JObject { ["type"] = "string" }, + ["library"] = new JObject { ["type"] = "string", ["description"] = "Root folder that was searched." }, + ["hits"] = new JObject + { + ["type"] = "array", + ["description"] = "Matching passages, best first. Source text to read and cite - not an answer.", + ["items"] = new JObject + { + ["type"] = "object", + ["properties"] = new JObject + { + ["file"] = new JObject { ["type"] = "string", ["description"] = "Path within the library." }, + ["page"] = new JObject { ["type"] = "integer", ["description"] = "1-based page, for the citation." }, + ["text"] = new JObject { ["type"] = "string" } + }, + ["required"] = new JArray("file", "page", "text") + } + }, + ["searched"] = new JObject + { + ["type"] = "array", + ["description"] = "Files actually read.", + ["items"] = new JObject { ["type"] = "string" } + }, + ["available"] = new JObject + { + ["type"] = "array", + ["description"] = "Present only when no manual matched: names in the library to aim a retry at.", + ["items"] = new JObject { ["type"] = "string" } + }, + ["familyMatchOnly"] = new JObject + { + ["type"] = "boolean", + ["description"] = "True when nothing was named for this model exactly and its series' " + + "manuals were read instead - a substitution the user must be told about." + }, + ["unreadable"] = new JObject + { + ["type"] = "array", + ["description"] = "Files that could not be extracted, and why. A match may be hiding in one of these.", + ["items"] = new JObject + { + ["type"] = "object", + ["properties"] = new JObject + { + ["file"] = new JObject { ["type"] = "string" }, + ["problem"] = new JObject { ["type"] = "string" } + } + } + }, + ["error"] = new JObject { ["type"] = "string" } + }, + ["required"] = new JArray("ok"), + ["additionalProperties"] = false + }; + } +} diff --git a/tests/GpibMcp.Tests/ManualSearchTests.cs b/tests/GpibMcp.Tests/ManualSearchTests.cs new file mode 100644 index 0000000..99e4a9f --- /dev/null +++ b/tests/GpibMcp.Tests/ManualSearchTests.cs @@ -0,0 +1,312 @@ +using System; +using System.IO; +using System.Linq; +using GpibMcp.Instruments; +using GpibMcp.Manuals; +using GpibMcp.Mcp; +using GpibMcp.Tools; +using Newtonsoft.Json.Linq; +using Xunit; + +namespace GpibMcp.Tests +{ + /// + /// Searching a local folder of instrument manuals (#120). The fixtures are text files rather than PDFs + /// on purpose: the search, the ranking and the citations are what these tests are about, and a test that + /// needed Poppler installed would be testing the machine instead of the code. + /// + public class ManualSearchTests : IDisposable + { + private readonly string _root; + private readonly string _cache; + + public ManualSearchTests() + { + _root = Path.Combine(Path.GetTempPath(), "gpibmcp-manuals-" + Guid.NewGuid().ToString("N")); + _cache = Path.Combine(_root, "_cache"); + Directory.CreateDirectory(_root); + Directory.CreateDirectory(_cache); + Environment.SetEnvironmentVariable("GPIB_MCP_MANUAL_CACHE", _cache); + } + + public void Dispose() + { + Environment.SetEnvironmentVariable("GPIB_MCP_MANUAL_CACHE", null); + Environment.SetEnvironmentVariable("GPIB_MCP_MANUALS", null); + try { Directory.Delete(_root, recursive: true); } catch { /* best effort */ } + } + + private string Write(string relativePath, string contents) + { + string full = Path.Combine(_root, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(full)); + File.WriteAllText(full, contents); + return full; + } + + /// Two pages of an 8563E-ish manual, separated the way pdftotext separates pages. + private const string AnalyzerManual = + "HP 8563E Programming Manual\nIntroduction to remote operation.\n" + + "\f" + + "CF Center Frequency\n" + + " Syntax: CF \n" + + " Sets the center frequency of the displayed span.\n" + + "\f" + + "SP Span\n Syntax: SP \n"; + + private static McpTool Tool(ManualLibrary library) + { + var registry = new ToolRegistry(); + ManualTools.Register(registry, library); + registry.TryGet("manual_search", out var tool); + return tool; + } + + // ---------------------------------------------------------------- registration + + [Fact] + public void TheToolIsAbsentUntilALibraryIsConfigured() + { + // A server with no library must not advertise a tool that could only ever answer "not configured". + Environment.SetEnvironmentVariable("GPIB_MCP_MANUALS", null); + var registry = InstrumentTools.BuildRegistry(new FakeInstrumentManager()); + Assert.False(registry.TryGet("manual_search", out _)); + + Environment.SetEnvironmentVariable("GPIB_MCP_MANUALS", _root); + Assert.True(InstrumentTools.BuildRegistry(new FakeInstrumentManager()).TryGet("manual_search", out _)); + } + + [Fact] + public void AConfiguredFolderThatDoesNotExistDisablesTheToolRatherThanFailing() + { + Environment.SetEnvironmentVariable("GPIB_MCP_MANUALS", + Path.Combine(_root, "no-such-folder")); + Assert.False(InstrumentTools.BuildRegistry(new FakeInstrumentManager()).TryGet("manual_search", out _)); + } + + // ---------------------------------------------------------------- searching + + [Fact] + public void APassageIsFoundAndCitedByFileAndPage() + { + Write("8563E Programming.txt", AnalyzerManual); + ToolOutput output = Tool(ManualLibrary.At(_root)) + .Invoke(new JObject { ["model"] = "8563E", ["query"] = "center frequency CF" }); + + JToken hit = output.Structured["hits"].First(); + Assert.Equal("8563E Programming.txt", (string)hit["file"]); + Assert.Equal(2, (int)hit["page"]); // the CF page, not page 1 + Assert.Contains("Center Frequency", (string)hit["text"]); + Assert.Contains("page 2", output.AsText()); // the citation a reader sees + } + + [Fact] + public void TheModelNarrowsWhichManualsAreOpened() + { + Write("8563E Programming.txt", AnalyzerManual); + Write("3325B Operating.txt", "3325B\n\fFR Frequency\n Sets the output frequency.\n"); + + ToolOutput output = Tool(ManualLibrary.At(_root)) + .Invoke(new JObject { ["model"] = "3325B", ["query"] = "frequency" }); + + var searched = ((JArray)output.Structured["searched"]).Select(f => (string)f).ToList(); + Assert.Contains("3325B Operating.txt", searched); + Assert.DoesNotContain("8563E Programming.txt", searched); + } + + [Fact] + public void AModelFolderMatchesAsWellAsAFilename() + { + // Libraries are organised both ways: a file per model, or a folder per model. + Write(Path.Combine("3458A", "operating.txt"), "3458A\n\fTRIG Trigger\n Arms the multimeter.\n"); + Write("unrelated.txt", "Nothing to do with it."); + + ToolOutput output = Tool(ManualLibrary.At(_root)) + .Invoke(new JObject { ["model"] = "3458A", ["query"] = "trigger" }); + + Assert.Equal(Path.Combine("3458A", "operating.txt"), + (string)output.Structured["hits"].First()["file"]); + } + + [Fact] + public void TheRarestWordAnchorsTheSearch() + { + // "frequency" appears on every page; "CF" on one. Anchoring on the rare word is what stops a + // search returning the whole manual. + Write("8563E.txt", AnalyzerManual); + ToolOutput output = Tool(ManualLibrary.At(_root)) + .Invoke(new JObject { ["model"] = "8563E", ["query"] = "CF frequency" }); + + Assert.All((JArray)output.Structured["hits"], h => Assert.Contains("CF", (string)h["text"])); + } + + [Fact] + public void NoPassageMatchesInAManualThatWasSearched() + { + Write("8563E.txt", AnalyzerManual); + ToolOutput output = Tool(ManualLibrary.At(_root)) + .Invoke(new JObject { ["model"] = "8563E", ["query"] = "hyperspatial flux capacitor" }); + + Assert.True((bool)output.Structured["ok"]); + Assert.Empty((JArray)output.Structured["hits"]); + Assert.NotEmpty((JArray)output.Structured["searched"]); // it really did look + Assert.Contains("No passage matched", output.AsText()); + } + + [Fact] + public void NoManualNamedForTheModelSearchesNothingAndSaysWhatIsThere() + { + // Reading whichever files happened to be smallest would report "searched 12, nothing found", + // which reads as "your library does not have this" when the truth is "I never opened the right + // file". Say nothing matched, and show what is there so the next call can aim. + Write("3325B Operating.txt", "3325B\n\fFR Frequency\n"); + Write("readme.txt", "not a manual"); + + ToolOutput output = Tool(ManualLibrary.At(_root)) + .Invoke(new JObject { ["model"] = "54622D", ["query"] = "timebase" }); + + Assert.True((bool)output.Structured["ok"]); + Assert.Empty((JArray)output.Structured["searched"]); + Assert.Empty((JArray)output.Structured["hits"]); + Assert.NotEmpty((JArray)output.Structured["available"]); + Assert.Contains("nothing was searched", output.AsText()); + } + + [Fact] + public void AZeroByteFileIsNotAManual() + { + File.WriteAllBytes(Path.Combine(_root, "8563E truncated download.pdf"), new byte[0]); + Assert.Empty(ManualLibrary.At(_root).Candidates("8563E", "anything")); + } + + [Fact] + public void AModelFallsBackToItsSeriesManual_AndSaysSo() + { + // The case the real library exposed: an 8563E's programming manual is filed as "8560E + // Programming Guide" - the series, not the instrument. A human would reach for it too, so we + // do, at a much lower score, and the substitution is stated rather than hidden. + Write("8560E Programming Guide.txt", AnalyzerManual); + + ToolOutput output = Tool(ManualLibrary.At(_root)) + .Invoke(new JObject { ["model"] = "8563E", ["query"] = "center frequency CF" }); + + Assert.NotEmpty((JArray)output.Structured["hits"]); + Assert.True((bool)output.Structured["familyMatchOnly"]); + Assert.Contains("same", output.AsText()); + Assert.Contains("series", output.AsText()); + } + + [Fact] + public void AnExactModelMatchBeatsItsSeries() + { + Write("8560E Programming Guide.txt", AnalyzerManual); + Write("8563E Programming.txt", AnalyzerManual); + + ToolOutput output = Tool(ManualLibrary.At(_root)) + .Invoke(new JObject { ["model"] = "8563E", ["query"] = "center frequency CF" }); + + Assert.Equal("8563E Programming.txt", (string)output.Structured["hits"].First()["file"]); + Assert.Null(output.Structured["familyMatchOnly"]); + } + + [Theory] + [InlineData("8563E", "856")] + [InlineData("3458A", "345")] + [InlineData("DS1054Z", null)] // no leading digits: no family to guess at + [InlineData("E4438C", null)] + [InlineData("42", null)] // too short to identify anything + public void TheFamilyStemIsTheLeadingDigits(string model, string expected) + { + Assert.Equal(expected, ManualLibrary.FamilyStem(model)); + } + + [Fact] + public void AnUnreadableManualIsReportedNotSwallowed() + { + // A PDF nobody can extract must never look like a PDF with no match - otherwise the tool quietly + // reports "not in your manuals" when it means "I could not look". + File.WriteAllBytes(Path.Combine(_root, "8563E scan.pdf"), new byte[] { 0x25, 0x50, 0x44, 0x46 }); + Environment.SetEnvironmentVariable("GPIB_MCP_PDFTOTEXT", Path.Combine(_root, "no-such-tool.exe")); + try + { + ToolOutput output = Tool(ManualLibrary.At(_root)) + .Invoke(new JObject { ["model"] = "8563E", ["query"] = "center frequency" }); + + var unreadable = (JArray)output.Structured["unreadable"]; + Assert.Single(unreadable); + Assert.Equal("8563E scan.pdf", (string)unreadable[0]["file"]); + Assert.Contains("pdftotext", (string)unreadable[0]["problem"]); + Assert.Contains("Could not read", output.AsText()); + } + finally { Environment.SetEnvironmentVariable("GPIB_MCP_PDFTOTEXT", null); } + } + + [Fact] + public void ASidecarTextFileIsUsedWhenThereIsNoExtractor() + { + // The route for a user with no Poppler: extract once, however they like, leave the .txt beside it. + File.WriteAllBytes(Path.Combine(_root, "8563E manual.pdf"), new byte[] { 0x25, 0x50, 0x44, 0x46 }); + Write("8563E manual.txt", AnalyzerManual); + + Environment.SetEnvironmentVariable("GPIB_MCP_PDFTOTEXT", Path.Combine(_root, "no-such-tool.exe")); + try + { + ToolOutput output = Tool(ManualLibrary.At(_root)) + .Invoke(new JObject { ["model"] = "8563E", ["query"] = "center frequency CF" }); + + Assert.NotEmpty((JArray)output.Structured["hits"]); + } + finally { Environment.SetEnvironmentVariable("GPIB_MCP_PDFTOTEXT", null); } + } + + // ---------------------------------------------------------------- safety and shape + + [Fact] + public void APathOutsideTheLibraryIsRefused() + { + // The path comes from a tool argument, so traversal has to bounce off something. + Write("8563E.txt", AnalyzerManual); + ToolOutput output = Tool(ManualLibrary.At(_root)).Invoke(new JObject + { + ["query"] = "anything", + ["file"] = Path.Combine("..", "..", "windows", "win.ini") + }); + + Assert.True(output.IsError); + Assert.False((bool)output.Structured["ok"]); + } + + [Fact] + public void AnEmptyLibrarySaysSoRatherThanPretendingToSearch() + { + ToolOutput output = Tool(ManualLibrary.At(_root)).Invoke(new JObject { ["query"] = "anything" }); + Assert.Empty((JArray)output.Structured["searched"]); + Assert.Contains("0 manual(s) in the library", output.AsText()); + } + + [Fact] + public void TheToolDeclaresItsOutputSchema() + { + Assert.NotNull(Tool(ManualLibrary.At(_root)).OutputSchema); + Assert.Equal("object", (string)Tool(ManualLibrary.At(_root)).OutputSchema["type"]); + } + + [Fact] + public void ExtractedTextIsCachedSoAManualIsConvertedOnce() + { + Write("8563E.txt", AnalyzerManual); + var file = ManualLibrary.At(_root).Candidates("8563E", "center frequency").First(); + + Assert.True(ManualText.Read(file).Ok); + Assert.Contains("Center Frequency", ManualText.Read(file).Text); + } + + [Fact] + public void PageNumbersCountFromOne() + { + Assert.Equal(1, ManualText.PageAt("no breaks here", 5)); + Assert.Equal(2, ManualText.PageAt("one\ftwo", 5)); + Assert.Equal(3, ManualText.PageAt("one\ftwo\fthree", 9)); + } + } +}