From af2b3c9577a2f25ad55491c424c9cd371915c10a Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Tue, 6 Oct 2026 18:11:22 -0300 Subject: [PATCH 1/2] Improve timeout errors on the `llm` command Signed-off-by: Juan Cruz Viotti --- docs/llm.markdown | 3 -- src/command_llm.cc | 16 ++++++++- src/error.h | 25 ++++++++++++++ test/CMakeLists.txt | 1 + test/ci/fail_llm_timeout.sh | 65 +++++++++++++++++++++++++++++++++++++ 5 files changed, 106 insertions(+), 4 deletions(-) create mode 100755 test/ci/fail_llm_timeout.sh diff --git a/docs/llm.markdown b/docs/llm.markdown index c4e1e999..158e0566 100644 --- a/docs/llm.markdown +++ b/docs/llm.markdown @@ -73,9 +73,6 @@ jsonschema llm path/to/schema.json \ --header "Authorization: Bearer $DOUBLEWORD_API_KEY" ``` -Doubleword can sometimes be slow to serve a request, independently of the -model, which is what the generous default `--timeout/-T` leaves room for. - ### OpenAI ```sh diff --git a/src/command_llm.cc b/src/command_llm.cc index 071619af..63a47a08 100644 --- a/src/command_llm.cc +++ b/src/command_llm.cc @@ -777,7 +777,21 @@ auto sourcemeta::jsonschema::llm(const sourcemeta::core::Options &options) // Sent once and never retried. A completion is not idempotent and it costs // money, so a request that fails is reported rather than repeated - const auto response{request.send()}; + const auto started{std::chrono::steady_clock::now()}; + sourcemeta::core::HTTPResponse response; + try { + response = request.send(); + } catch (const sourcemeta::core::HTTPError &) { + // What the backend says about a failure is its own business, so whether the + // time ran out is read off the clock rather than off the message + if (std::chrono::steady_clock::now() - started >= timeout) { + throw LLMTimeoutError{"The completion request did not answer within " + + std::to_string(timeout.count()) + " seconds", + std::string{url}}; + } + + throw; + } LOG_VERBOSE(options) << "Received HTTP " << response.status.code << "\n"; diff --git a/src/error.h b/src/error.h index caa23af1..e23f8223 100644 --- a/src/error.h +++ b/src/error.h @@ -418,6 +418,22 @@ class MissingOptionError : public std::runtime_error { std::string example_; }; +// A request that ran out of time before an answer arrived. Waiting longer is +// the one thing the caller can do about it, so what it took is part of what is +// said +class LLMTimeoutError : public std::runtime_error { +public: + LLMTimeoutError(const std::string &message, std::string url) + : std::runtime_error{message}, url_{std::move(url)} {} + + [[nodiscard]] auto url() const noexcept -> const std::string & { + return this->url_; + } + +private: + std::string url_; +}; + class OptionConflictError : public std::runtime_error { public: OptionConflictError(const std::string &message) @@ -2283,6 +2299,15 @@ inline auto try_catch(const sourcemeta::core::Options &options, const auto is_json{options.contains("json")}; print_exception(is_json, error); return EXIT_OTHER_INPUT_ERROR; + } catch (const LLMTimeoutError &error) { + const auto is_json{options.contains("json")}; + print_exception(is_json, error); + if (!is_json) { + std::cerr << "\nPass --timeout/-T with a larger number of seconds to " + "wait longer\n"; + } + + return EXIT_UNEXPECTED_ERROR; } catch (const sourcemeta::core::HTTPStatusError &error) { const auto is_json{options.contains("json")}; print_exception(is_json, error); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 583ee81a..5e823e9f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1614,6 +1614,7 @@ add_jsonschema_test_unix_ci(pass_llm_request_response) add_jsonschema_test_unix_ci(fail_llm_response_invalid) add_jsonschema_test_unix_ci(fail_llm_response_rejected) add_jsonschema_test_unix_ci(fail_llm_response_unreadable) +add_jsonschema_test_unix_ci(fail_llm_timeout) add_jsonschema_test_unix_ci(precommit_lint) if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND NOT JSONSCHEMA_USE_SYSTEM_CURL) diff --git a/test/ci/fail_llm_timeout.sh b/test/ci/fail_llm_timeout.sh new file mode 100755 index 00000000..202f485d --- /dev/null +++ b/test/ci/fail_llm_timeout.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +PORT=5899 + +# A server that accepts the connection and never answers, which is the one +# failure the caller can do something about by waiting longer +cat << 'EOF' > "$TMP/server.js" +const http = require('http'); +const server = http.createServer(() => {}); +server.listen(parseInt(process.argv[2], 10)); +EOF + +node "$TMP/server.js" "$PORT" & +SERVER_PID="$!" + +clean() { + kill "$SERVER_PID" 2>/dev/null || true + rm -rf "$TMP" +} +trap clean EXIT + +sleep 2 + +cat << 'EOF' > "$TMP/schema.json" +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string" +} +EOF + +"$1" llm "$TMP/schema.json" --ask hello \ + --url "http://localhost:${PORT}/v1/chat/completions" --model my-model \ + --timeout 2 \ + > "$TMP/output.txt" 2>&1 && EXIT_CODE="$?" || EXIT_CODE="$?" +# Unexpected error +test "$EXIT_CODE" = "1" + +cat << EOF > "$TMP/expected.txt" +error: The completion request did not answer within 2 seconds + at url http://localhost:${PORT}/v1/chat/completions + +Pass --timeout/-T with a larger number of seconds to wait longer +EOF + +diff "$TMP/output.txt" "$TMP/expected.txt" + +"$1" llm "$TMP/schema.json" --ask hello \ + --url "http://localhost:${PORT}/v1/chat/completions" --model my-model \ + --timeout 2 --json \ + > "$TMP/output_json.txt" 2>&1 && EXIT_CODE="$?" || EXIT_CODE="$?" +# Unexpected error +test "$EXIT_CODE" = "1" + +cat << EOF > "$TMP/expected_json.txt" +{ + "error": "The completion request did not answer within 2 seconds", + "url": "http://localhost:${PORT}/v1/chat/completions" +} +EOF + +diff "$TMP/output_json.txt" "$TMP/expected_json.txt" From 285bb5686693f9367506d1bb41d219c65fd324ab Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Tue, 6 Oct 2026 18:19:51 -0300 Subject: [PATCH 2/2] More Signed-off-by: Juan Cruz Viotti --- src/command_llm.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/command_llm.cc b/src/command_llm.cc index 63a47a08..a18df16b 100644 --- a/src/command_llm.cc +++ b/src/command_llm.cc @@ -784,6 +784,9 @@ auto sourcemeta::jsonschema::llm(const sourcemeta::core::Options &options) } catch (const sourcemeta::core::HTTPError &) { // What the backend says about a failure is its own business, so whether the // time ran out is read off the clock rather than off the message + // TODO: Have Core raise an exception of its own for a request that ran out + // of time, so that this is detected from the failure rather than inferred + // from how long it took if (std::chrono::steady_clock::now() - started >= timeout) { throw LLMTimeoutError{"The completion request did not answer within " + std::to_string(timeout.count()) + " seconds",