diff --git a/docs/llm.markdown b/docs/llm.markdown index c4e1e999..158e0566 100644 --- a/docs/llm.markdown +++ b/docs/llm.markdown @@ -73,9 +73,6 @@ jsonschema llm path/to/schema.json \ --header "Authorization: Bearer $DOUBLEWORD_API_KEY" ``` -Doubleword can sometimes be slow to serve a request, independently of the -model, which is what the generous default `--timeout/-T` leaves room for. - ### OpenAI ```sh diff --git a/src/command_llm.cc b/src/command_llm.cc index 071619af..a18df16b 100644 --- a/src/command_llm.cc +++ b/src/command_llm.cc @@ -777,7 +777,24 @@ auto sourcemeta::jsonschema::llm(const sourcemeta::core::Options &options) // Sent once and never retried. A completion is not idempotent and it costs // money, so a request that fails is reported rather than repeated - const auto response{request.send()}; + const auto started{std::chrono::steady_clock::now()}; + sourcemeta::core::HTTPResponse response; + try { + response = request.send(); + } catch (const sourcemeta::core::HTTPError &) { + // What the backend says about a failure is its own business, so whether the + // time ran out is read off the clock rather than off the message + // TODO: Have Core raise an exception of its own for a request that ran out + // of time, so that this is detected from the failure rather than inferred + // from how long it took + if (std::chrono::steady_clock::now() - started >= timeout) { + throw LLMTimeoutError{"The completion request did not answer within " + + std::to_string(timeout.count()) + " seconds", + std::string{url}}; + } + + throw; + } LOG_VERBOSE(options) << "Received HTTP " << response.status.code << "\n"; diff --git a/src/error.h b/src/error.h index caa23af1..e23f8223 100644 --- a/src/error.h +++ b/src/error.h @@ -418,6 +418,22 @@ class MissingOptionError : public std::runtime_error { std::string example_; }; +// A request that ran out of time before an answer arrived. Waiting longer is +// the one thing the caller can do about it, so what it took is part of what is +// said +class LLMTimeoutError : public std::runtime_error { +public: + LLMTimeoutError(const std::string &message, std::string url) + : std::runtime_error{message}, url_{std::move(url)} {} + + [[nodiscard]] auto url() const noexcept -> const std::string & { + return this->url_; + } + +private: + std::string url_; +}; + class OptionConflictError : public std::runtime_error { public: OptionConflictError(const std::string &message) @@ -2283,6 +2299,15 @@ inline auto try_catch(const sourcemeta::core::Options &options, const auto is_json{options.contains("json")}; print_exception(is_json, error); return EXIT_OTHER_INPUT_ERROR; + } catch (const LLMTimeoutError &error) { + const auto is_json{options.contains("json")}; + print_exception(is_json, error); + if (!is_json) { + std::cerr << "\nPass --timeout/-T with a larger number of seconds to " + "wait longer\n"; + } + + return EXIT_UNEXPECTED_ERROR; } catch (const sourcemeta::core::HTTPStatusError &error) { const auto is_json{options.contains("json")}; print_exception(is_json, error); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 583ee81a..5e823e9f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1614,6 +1614,7 @@ add_jsonschema_test_unix_ci(pass_llm_request_response) add_jsonschema_test_unix_ci(fail_llm_response_invalid) add_jsonschema_test_unix_ci(fail_llm_response_rejected) add_jsonschema_test_unix_ci(fail_llm_response_unreadable) +add_jsonschema_test_unix_ci(fail_llm_timeout) add_jsonschema_test_unix_ci(precommit_lint) if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND NOT JSONSCHEMA_USE_SYSTEM_CURL) diff --git a/test/ci/fail_llm_timeout.sh b/test/ci/fail_llm_timeout.sh new file mode 100755 index 00000000..202f485d --- /dev/null +++ b/test/ci/fail_llm_timeout.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +PORT=5899 + +# A server that accepts the connection and never answers, which is the one +# failure the caller can do something about by waiting longer +cat << 'EOF' > "$TMP/server.js" +const http = require('http'); +const server = http.createServer(() => {}); +server.listen(parseInt(process.argv[2], 10)); +EOF + +node "$TMP/server.js" "$PORT" & +SERVER_PID="$!" + +clean() { + kill "$SERVER_PID" 2>/dev/null || true + rm -rf "$TMP" +} +trap clean EXIT + +sleep 2 + +cat << 'EOF' > "$TMP/schema.json" +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string" +} +EOF + +"$1" llm "$TMP/schema.json" --ask hello \ + --url "http://localhost:${PORT}/v1/chat/completions" --model my-model \ + --timeout 2 \ + > "$TMP/output.txt" 2>&1 && EXIT_CODE="$?" || EXIT_CODE="$?" +# Unexpected error +test "$EXIT_CODE" = "1" + +cat << EOF > "$TMP/expected.txt" +error: The completion request did not answer within 2 seconds + at url http://localhost:${PORT}/v1/chat/completions + +Pass --timeout/-T with a larger number of seconds to wait longer +EOF + +diff "$TMP/output.txt" "$TMP/expected.txt" + +"$1" llm "$TMP/schema.json" --ask hello \ + --url "http://localhost:${PORT}/v1/chat/completions" --model my-model \ + --timeout 2 --json \ + > "$TMP/output_json.txt" 2>&1 && EXIT_CODE="$?" || EXIT_CODE="$?" +# Unexpected error +test "$EXIT_CODE" = "1" + +cat << EOF > "$TMP/expected_json.txt" +{ + "error": "The completion request did not answer within 2 seconds", + "url": "http://localhost:${PORT}/v1/chat/completions" +} +EOF + +diff "$TMP/output_json.txt" "$TMP/expected_json.txt"