From fb35a09b07221351a8f4c1f1b96b8494d9d5026e Mon Sep 17 00:00:00 2001 From: Mohammad Saddam Hossain Date: Mon, 19 Oct 2026 07:14:12 +0530 Subject: [PATCH] feat(ai): configurable AI provider (anthropic, gemini, openai-compatible) Failure analysis and the run summary no longer hardcode the Anthropic API. ai.provider selects the backend, ai.baseUrl overrides the endpoint, and ai.model defaults per provider. Default behaviour is unchanged. Provider tests run against a local fake endpoint, which also gives the AI call path an HTTP seam. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 2 + docs-site/docs/changelog.md | 2 + docs-site/docs/reporting/html-report.md | 19 +++ .../seleniumboot/ai/AiFailureAnalyzer.java | 70 +++++------ .../java/com/seleniumboot/ai/AiProvider.java | 115 ++++++++++++++++++ .../com/seleniumboot/ai/AiRunSummarizer.java | 2 +- .../config/SeleniumBootConfig.java | 13 +- .../com/seleniumboot/ai/AiProviderTest.java | 111 +++++++++++++++++ 8 files changed, 290 insertions(+), 44 deletions(-) create mode 100644 src/main/java/com/seleniumboot/ai/AiProvider.java create mode 100644 src/test/java/com/seleniumboot/ai/AiProviderTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index d07ee38..452732c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Unreleased +- **AI provider is configurable** — `ai.provider` selects the backend for failure analysis and the run summary: `anthropic` (default, unchanged), `gemini`, or `openai` (any OpenAI-compatible `/chat/completions` endpoint — OpenAI, Azure, Groq, local Ollama via `ai.baseUrl`). `ai.model` now defaults per provider. Existing configs keep working with no change. + - **AI run summary (opt-in)** — `ai.runSummary: true` adds one short paragraph to the HTML dashboard summarising the run's failures: which same-cause groups account for most of them, which look like product bugs versus test or environment problems, and what to check first. One API call per run, independent of `ai.failureAnalysis`; only grouped error messages and test names are sent (see the data note in the HTML report docs). Absent when disabled, when nothing failed, or on any error; the suite result is never affected. The text is exported as `aiRunSummary` in the metrics JSON. - **HTML report: pass-rate and duration trend charts** — the dashboard now shows two line charts under the donut and percentile row, built from the last 20 runs in the metrics history (the same `flakiness.historyDir` / `-Dseleniumboot.history.dir` location the Flakiness Radar reads). They appear once there are at least two runs of history and stay hidden otherwise. (#160) diff --git a/docs-site/docs/changelog.md b/docs-site/docs/changelog.md index 3756d2a..a28340d 100644 --- a/docs-site/docs/changelog.md +++ b/docs-site/docs/changelog.md @@ -11,6 +11,8 @@ All notable changes to Selenium Boot are documented here. ## [Unreleased] +- **AI provider is configurable** — `ai.provider` selects the backend for failure analysis and the run summary: `anthropic` (default, unchanged), `gemini`, or `openai` (any OpenAI-compatible `/chat/completions` endpoint — OpenAI, Azure, Groq, local Ollama via `ai.baseUrl`). `ai.model` now defaults per provider. Existing configs keep working with no change. + - **AI run summary (opt-in)** — `ai.runSummary: true` adds one short paragraph to the HTML dashboard summarising the run's failures: which same-cause groups account for most of them, which look like product bugs versus test or environment problems, and what to check first. One API call per run, independent of `ai.failureAnalysis`; only grouped error messages and test names are sent (see the data note in the HTML report docs). Absent when disabled, when nothing failed, or on any error; the suite result is never affected. The text is exported as `aiRunSummary` in the metrics JSON. - **HTML report: pass-rate and duration trend charts** — the dashboard now shows two line charts under the donut and percentile row, built from the last 20 runs in the metrics history (the same `flakiness.historyDir` / `-Dseleniumboot.history.dir` location the Flakiness Radar reads). They appear once there are at least two runs of history and stay hidden otherwise. (#160) diff --git a/docs-site/docs/reporting/html-report.md b/docs-site/docs/reporting/html-report.md index 7e475a1..9330fb5 100644 --- a/docs-site/docs/reporting/html-report.md +++ b/docs-site/docs/reporting/html-report.md @@ -107,6 +107,25 @@ ai: apiKey: ${CLAUDE_API_KEY} ``` +### Choosing an AI provider + +The AI features (`ai.failureAnalysis` and `ai.runSummary`) are not tied to one vendor. Set `ai.provider`: + +| `ai.provider` | Backend | Default `ai.model` | +|---|---|---| +| `anthropic` (default) | Claude Messages API | `claude-haiku-4-5-20251001` | +| `gemini` | Google Gemini `generateContent` | `gemini-2.5-flash` | +| `openai` | Any OpenAI-compatible `/chat/completions` endpoint | `gpt-4o-mini` | + +```yaml +ai: + runSummary: true + provider: gemini + apiKey: ${GEMINI_API_KEY} +``` + +Use `ai.baseUrl` to point `openai` at another compatible server, for example a local Ollama (`http://localhost:11434/v1`) or Azure OpenAI. The same data-minimisation rules apply whichever provider you pick. + The request contains only the run totals and, for the largest same-cause groups (at most 8 groups, 5 test names each), the rule-based category, the error message (first 300 characters) and the test names. It does not send stack traces, steps, page content, or screenshot/trace files. The text is stored in the metrics JSON as `aiRunSummary`. --- diff --git a/src/main/java/com/seleniumboot/ai/AiFailureAnalyzer.java b/src/main/java/com/seleniumboot/ai/AiFailureAnalyzer.java index 007f85a..eec7da4 100644 --- a/src/main/java/com/seleniumboot/ai/AiFailureAnalyzer.java +++ b/src/main/java/com/seleniumboot/ai/AiFailureAnalyzer.java @@ -20,7 +20,7 @@ import java.util.stream.Collectors; /** - * Calls the Claude API to generate a plain-English failure analysis for a failed test. + * Calls the configured LLM (Claude by default; see {@link AiProvider}) to generate a plain-English failure analysis for a failed test. * *

Activated when {@code ai.failureAnalysis: true} and {@code ai.apiKey} are set in * {@code selenium-boot.yml}. The analysis is stored in the test metrics and surfaced in the @@ -37,8 +37,10 @@ * # selenium-boot.yml * ai: * failureAnalysis: true + * provider: anthropic # anthropic | gemini | openai * apiKey: ${CLAUDE_API_KEY} - * model: claude-haiku-4-5-20251001 + * model: claude-haiku-4-5-20251001 # optional; defaults per provider + * baseUrl: # optional endpoint override (OpenAI-compatible servers) * timeoutSeconds: 20 * */ @@ -46,8 +48,6 @@ public final class AiFailureAnalyzer { private static final Logger LOG = Logger.getLogger(AiFailureAnalyzer.class.getName()); - private static final String API_URL = "https://api.anthropic.com/v1/messages"; - private static final String API_VERSION = "2023-06-01"; /** Analysis per failure signature, so one cause costs one API call. */ private static final ConcurrentHashMap> BY_SIGNATURE = @@ -99,8 +99,7 @@ public static void analyze(String testId, String pageUrl, String pageTitle) { } else { String result = null; try { - result = callApi(apiKey, aiCfg.getModel(), buildPrompt(timing, pageUrl, pageTitle), - aiCfg.getTimeoutSeconds()); + result = callApi(aiCfg, apiKey, buildPrompt(timing, pageUrl, pageTitle)); } finally { if (result == null || result.isBlank()) BY_SIGNATURE.remove(signature, mine); mine.complete(result); @@ -185,32 +184,29 @@ static String buildPrompt( // HTTP call to Claude API // ------------------------------------------------------------------ - static String callApi(String apiKey, String model, String prompt, int timeoutSeconds) { + static String callApi(SeleniumBootConfig.Ai cfg, String apiKey, String prompt) { + AiProvider provider = AiProvider.parse(cfg.getProvider()); + if (provider == null) { + LOG.warning("[AiFailureAnalyzer] Unknown ai.provider '" + cfg.getProvider() + + "' (expected anthropic, gemini or openai). Skipping AI call."); + return null; + } try { - String body = buildRequestBody(model, prompt); - + String model = provider.modelOrDefault(cfg.getModel()); + int timeout = cfg.getTimeoutSeconds(); HttpClient client = HttpClient.newBuilder() - .connectTimeout(Duration.ofSeconds(timeoutSeconds)) + .connectTimeout(Duration.ofSeconds(timeout)) .build(); - - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(API_URL)) - .header("x-api-key", apiKey) - .header("anthropic-version", API_VERSION) - .header("content-type", "application/json") - .timeout(Duration.ofSeconds(timeoutSeconds)) - .POST(HttpRequest.BodyPublishers.ofString(body)) - .build(); - - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + HttpResponse response = client.send( + provider.request(cfg.getBaseUrl(), model, apiKey, prompt, timeout), + HttpResponse.BodyHandlers.ofString()); if (response.statusCode() == 200) { - return extractContent(response.body()); - } else { - LOG.warning("[AiFailureAnalyzer] API returned HTTP " + response.statusCode() - + ": " + response.body().substring(0, Math.min(200, response.body().length()))); - return null; + return provider.extract(response.body()); } + LOG.warning("[AiFailureAnalyzer] " + provider + " API returned HTTP " + response.statusCode() + + ": " + response.body().substring(0, Math.min(200, response.body().length()))); + return null; } catch (Exception e) { LOG.warning("[AiFailureAnalyzer] HTTP call failed: " + e.getMessage()); return null; @@ -221,25 +217,15 @@ static String callApi(String apiKey, String model, String prompt, int timeoutSec // JSON helpers (no extra dependency — hand-rolled) // ------------------------------------------------------------------ - private static String buildRequestBody(String model, String prompt) { - String escaped = prompt - .replace("\\", "\\\\") - .replace("\"", "\\\"") - .replace("\n", "\\n") - .replace("\r", "") - .replace("\t", "\\t"); - return "{" - + "\"model\":\"" + model + "\"," - + "\"max_tokens\":512," - + "\"messages\":[{\"role\":\"user\",\"content\":\"" + escaped + "\"}]" - + "}"; - } - /** Extracts {@code content[0].text} from a Claude API response JSON string. */ public static String extractContent(String json) { + return extractField(json, "text"); + } + + /** Value of the first string field called {@code key} (Claude/Gemini: {@code text}; OpenAI: {@code content}). */ + static String extractField(String json, String key) { // Simple extraction — avoids adding Jackson dependency to the hot path - // Pattern: "text": "..." (first occurrence in content array) - int textIdx = json.indexOf("\"text\""); + int textIdx = json.indexOf("\"" + key + "\""); if (textIdx < 0) return null; int colon = json.indexOf(':', textIdx); if (colon < 0) return null; diff --git a/src/main/java/com/seleniumboot/ai/AiProvider.java b/src/main/java/com/seleniumboot/ai/AiProvider.java new file mode 100644 index 0000000..0a89398 --- /dev/null +++ b/src/main/java/com/seleniumboot/ai/AiProvider.java @@ -0,0 +1,115 @@ +package com.seleniumboot.ai; + +import java.net.URI; +import java.net.http.HttpRequest; +import java.time.Duration; + +/** + * The LLM backends the AI features can talk to. Selected with {@code ai.provider}; the request + * shape, auth header and response path are the only things that differ. + * + *

+ */ +enum AiProvider { + + ANTHROPIC("https://api.anthropic.com/v1", "claude-haiku-4-5-20251001"), + GEMINI("https://generativelanguage.googleapis.com/v1beta", "gemini-2.5-flash"), + OPENAI("https://api.openai.com/v1", "gpt-4o-mini"); + + final String defaultBaseUrl; + final String defaultModel; + + AiProvider(String defaultBaseUrl, String defaultModel) { + this.defaultBaseUrl = defaultBaseUrl; + this.defaultModel = defaultModel; + } + + /** Parses {@code ai.provider}; blank means Anthropic. Returns {@code null} for an unknown name. */ + static AiProvider parse(String name) { + if (name == null || name.isBlank()) return ANTHROPIC; + String n = name.strip().toLowerCase(); + if (n.equals("claude")) return ANTHROPIC; + if (n.equals("google")) return GEMINI; + for (AiProvider p : values()) { + if (p.name().equalsIgnoreCase(n)) return p; + } + return null; + } + + /** Model to use: the configured one, or this provider's default when none is set. */ + String modelOrDefault(String configured) { + return configured == null || configured.isBlank() ? defaultModel : configured; + } + + /** Full endpoint URL for {@code model}; {@code baseUrl} overrides the provider default. */ + String endpoint(String baseUrl, String model) { + String base = baseUrl == null || baseUrl.isBlank() ? defaultBaseUrl : baseUrl.strip(); + if (base.endsWith("/")) base = base.substring(0, base.length() - 1); + switch (this) { + case GEMINI: return base + "/models/" + model + ":generateContent"; + case OPENAI: return base + "/chat/completions"; + default: return base + "/messages"; + } + } + + HttpRequest request(String baseUrl, String model, String apiKey, String prompt, int timeoutSeconds) { + HttpRequest.Builder b = HttpRequest.newBuilder() + .uri(URI.create(endpoint(baseUrl, model))) + .header("content-type", "application/json") + .timeout(Duration.ofSeconds(timeoutSeconds)); + switch (this) { + case GEMINI: + b.header("x-goog-api-key", apiKey); + break; + case OPENAI: + b.header("authorization", "Bearer " + apiKey); + break; + default: + b.header("x-api-key", apiKey).header("anthropic-version", "2023-06-01"); + } + return b.POST(HttpRequest.BodyPublishers.ofString(body(model, prompt))).build(); + } + + String body(String model, String prompt) { + String p = "\"" + escape(prompt) + "\""; + switch (this) { + case GEMINI: + // Gemini 2.5 spends output tokens on thinking, so leave headroom for the answer. + return "{\"contents\":[{\"parts\":[{\"text\":" + p + "}]}]," + + "\"generationConfig\":{\"maxOutputTokens\":2048}}"; + case OPENAI: + return "{\"model\":\"" + escape(model) + "\",\"max_tokens\":512," + + "\"messages\":[{\"role\":\"user\",\"content\":" + p + "}]}"; + default: + return "{\"model\":\"" + escape(model) + "\",\"max_tokens\":512," + + "\"messages\":[{\"role\":\"user\",\"content\":" + p + "}]}"; + } + } + + /** The generated text from a successful response body, or {@code null} when absent. */ + String extract(String json) { + return AiFailureAnalyzer.extractField(json, this == OPENAI ? "content" : "text"); + } + + private static String escape(String s) { + StringBuilder sb = new StringBuilder(s.length() + 16); + for (char c : s.toCharArray()) { + switch (c) { + case '\\': sb.append("\\\\"); break; + case '"': sb.append("\\\""); break; + case '\n': sb.append("\\n"); break; + case '\t': sb.append("\\t"); break; + case '\r': break; + default: + if (c < 0x20) sb.append(String.format("\\u%04x", (int) c)); + else sb.append(c); + } + } + return sb.toString(); + } +} diff --git a/src/main/java/com/seleniumboot/ai/AiRunSummarizer.java b/src/main/java/com/seleniumboot/ai/AiRunSummarizer.java index e0c4c12..ee925f1 100644 --- a/src/main/java/com/seleniumboot/ai/AiRunSummarizer.java +++ b/src/main/java/com/seleniumboot/ai/AiRunSummarizer.java @@ -40,7 +40,7 @@ public static String summarize(Collection timings) { } String prompt = buildPrompt(timings); if (prompt == null) return null; - String result = AiFailureAnalyzer.callApi(apiKey, cfg.getModel(), prompt, cfg.getTimeoutSeconds()); + String result = AiFailureAnalyzer.callApi(cfg, apiKey, prompt); return result == null || result.isBlank() ? null : result.strip(); } catch (Exception e) { LOG.warning("[AiRunSummarizer] Run summary failed (non-critical): " + e.getMessage()); diff --git a/src/main/java/com/seleniumboot/config/SeleniumBootConfig.java b/src/main/java/com/seleniumboot/config/SeleniumBootConfig.java index 33ed728..d494c23 100644 --- a/src/main/java/com/seleniumboot/config/SeleniumBootConfig.java +++ b/src/main/java/com/seleniumboot/config/SeleniumBootConfig.java @@ -465,7 +465,9 @@ public static final class Ai { private boolean failureAnalysis = false; private boolean runSummary = false; private String apiKey = null; - private String model = "claude-haiku-4-5-20251001"; + private String provider = "anthropic"; + private String model = null; + private String baseUrl = null; private int timeoutSeconds = 20; public boolean isFailureAnalysis() { return failureAnalysis; } @@ -477,9 +479,18 @@ public static final class Ai { public String getApiKey() { return apiKey; } public void setApiKey(String v) { this.apiKey = v; } + /** {@code anthropic} (default), {@code gemini} or {@code openai} (any OpenAI-compatible endpoint). */ + public String getProvider() { return provider; } + public void setProvider(String v) { this.provider = v; } + + /** Model name; {@code null} means the provider's default. */ public String getModel() { return model; } public void setModel(String v) { this.model = v; } + /** Optional endpoint override, e.g. a local Ollama or Azure OpenAI URL. */ + public String getBaseUrl() { return baseUrl; } + public void setBaseUrl(String v) { this.baseUrl = v; } + public int getTimeoutSeconds() { return timeoutSeconds; } public void setTimeoutSeconds(int v) { this.timeoutSeconds = v; } } diff --git a/src/test/java/com/seleniumboot/ai/AiProviderTest.java b/src/test/java/com/seleniumboot/ai/AiProviderTest.java new file mode 100644 index 0000000..1a14d5d --- /dev/null +++ b/src/test/java/com/seleniumboot/ai/AiProviderTest.java @@ -0,0 +1,111 @@ +package com.seleniumboot.ai; + +import com.seleniumboot.config.SeleniumBootConfig; +import com.sun.net.httpserver.HttpServer; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicReference; + +/** Drives each AI provider against a local fake endpoint — no network, no real key. */ +public class AiProviderTest { + + private HttpServer server; + private final AtomicReference path = new AtomicReference<>(); + private final AtomicReference body = new AtomicReference<>(); + private final AtomicReference headers = new AtomicReference<>(); + private volatile int status = 200; + private volatile String reply = "{}"; + + @BeforeMethod + public void start() throws Exception { + status = 200; + reply = "{}"; + path.set(null); + body.set(null); + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", ex -> { + path.set(ex.getRequestURI().getPath()); + headers.set(ex.getRequestHeaders()); + body.set(new String(ex.getRequestBody().readAllBytes(), StandardCharsets.UTF_8)); + byte[] out = reply.getBytes(StandardCharsets.UTF_8); + ex.sendResponseHeaders(status, out.length); + ex.getResponseBody().write(out); + ex.close(); + }); + server.start(); + } + + @AfterMethod + public void stop() { + server.stop(0); + } + + private SeleniumBootConfig.Ai cfg(String provider, String model) { + SeleniumBootConfig.Ai ai = new SeleniumBootConfig.Ai(); + ai.setProvider(provider); + ai.setModel(model); + ai.setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort() + "/api"); + ai.setTimeoutSeconds(5); + return ai; + } + + @Test + public void anthropicIsTheDefaultAndUsesItsHeadersAndModel() { + reply = "{\"content\":[{\"type\":\"text\",\"text\":\"claude says hi\"}]}"; + SeleniumBootConfig.Ai ai = cfg(null, null); + + Assert.assertEquals(AiFailureAnalyzer.callApi(ai, "k1", "hello \"world\"\nline2"), "claude says hi"); + Assert.assertEquals(path.get(), "/api/messages"); + Assert.assertEquals(headers.get().getFirst("x-api-key"), "k1"); + Assert.assertNotNull(headers.get().getFirst("anthropic-version")); + Assert.assertTrue(body.get().contains("\"model\":\"claude-haiku-4-5-20251001\"")); + Assert.assertTrue(body.get().contains("hello \\\"world\\\"\\nline2")); + } + + @Test + public void geminiUsesGenerateContentAndKeyHeader() { + reply = "{\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"gemini says hi\"}]}}]}"; + SeleniumBootConfig.Ai ai = cfg("gemini", "gemini-2.5-flash"); + + Assert.assertEquals(AiFailureAnalyzer.callApi(ai, "g1", "ping"), "gemini says hi"); + Assert.assertEquals(path.get(), "/api/models/gemini-2.5-flash:generateContent"); + Assert.assertEquals(headers.get().getFirst("x-goog-api-key"), "g1"); + Assert.assertTrue(body.get().contains("\"parts\":[{\"text\":\"ping\"}]")); + } + + @Test + public void openAiCompatibleUsesChatCompletionsAndBearerAuth() { + reply = "{\"choices\":[{\"message\":{\"role\":\"assistant\",\"content\":\"gpt says hi\"}}]}"; + SeleniumBootConfig.Ai ai = cfg("openai", null); + + Assert.assertEquals(AiFailureAnalyzer.callApi(ai, "o1", "ping"), "gpt says hi"); + Assert.assertEquals(path.get(), "/api/chat/completions"); + Assert.assertEquals(headers.get().getFirst("authorization"), "Bearer o1"); + Assert.assertTrue(body.get().contains("\"model\":\"gpt-4o-mini\"")); + } + + @Test + public void providerNameIsCaseInsensitiveAndAliased() { + reply = "{\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"ok\"}]}}]}"; + Assert.assertEquals(AiFailureAnalyzer.callApi(cfg("Google", "m"), "k", "p"), "ok"); + Assert.assertEquals(path.get(), "/api/models/m:generateContent"); + } + + @Test + public void unknownProviderIsSkippedWithoutCallingOut() { + Assert.assertNull(AiFailureAnalyzer.callApi(cfg("bogus", null), "k", "p")); + Assert.assertNull(path.get()); + } + + @Test + public void httpErrorReturnsNull() { + status = 401; + reply = "{\"error\":\"bad key\"}"; + Assert.assertNull(AiFailureAnalyzer.callApi(cfg("anthropic", null), "k", "p")); + } +}