diff --git a/CHANGELOG.md b/CHANGELOG.md index d07ee38..452732c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Unreleased +- **AI provider is configurable** — `ai.provider` selects the backend for failure analysis and the run summary: `anthropic` (default, unchanged), `gemini`, or `openai` (any OpenAI-compatible `/chat/completions` endpoint — OpenAI, Azure, Groq, local Ollama via `ai.baseUrl`). `ai.model` now defaults per provider. Existing configs keep working with no change. + - **AI run summary (opt-in)** — `ai.runSummary: true` adds one short paragraph to the HTML dashboard summarising the run's failures: which same-cause groups account for most of them, which look like product bugs versus test or environment problems, and what to check first. One API call per run, independent of `ai.failureAnalysis`; only grouped error messages and test names are sent (see the data note in the HTML report docs). Absent when disabled, when nothing failed, or on any error; the suite result is never affected. The text is exported as `aiRunSummary` in the metrics JSON. - **HTML report: pass-rate and duration trend charts** — the dashboard now shows two line charts under the donut and percentile row, built from the last 20 runs in the metrics history (the same `flakiness.historyDir` / `-Dseleniumboot.history.dir` location the Flakiness Radar reads). They appear once there are at least two runs of history and stay hidden otherwise. (#160) diff --git a/docs-site/docs/changelog.md b/docs-site/docs/changelog.md index 3756d2a..a28340d 100644 --- a/docs-site/docs/changelog.md +++ b/docs-site/docs/changelog.md @@ -11,6 +11,8 @@ All notable changes to Selenium Boot are documented here. ## [Unreleased] +- **AI provider is configurable** — `ai.provider` selects the backend for failure analysis and the run summary: `anthropic` (default, unchanged), `gemini`, or `openai` (any OpenAI-compatible `/chat/completions` endpoint — OpenAI, Azure, Groq, local Ollama via `ai.baseUrl`). `ai.model` now defaults per provider. Existing configs keep working with no change. + - **AI run summary (opt-in)** — `ai.runSummary: true` adds one short paragraph to the HTML dashboard summarising the run's failures: which same-cause groups account for most of them, which look like product bugs versus test or environment problems, and what to check first. One API call per run, independent of `ai.failureAnalysis`; only grouped error messages and test names are sent (see the data note in the HTML report docs). Absent when disabled, when nothing failed, or on any error; the suite result is never affected. The text is exported as `aiRunSummary` in the metrics JSON. - **HTML report: pass-rate and duration trend charts** — the dashboard now shows two line charts under the donut and percentile row, built from the last 20 runs in the metrics history (the same `flakiness.historyDir` / `-Dseleniumboot.history.dir` location the Flakiness Radar reads). They appear once there are at least two runs of history and stay hidden otherwise. (#160) diff --git a/docs-site/docs/reporting/html-report.md b/docs-site/docs/reporting/html-report.md index 7e475a1..9330fb5 100644 --- a/docs-site/docs/reporting/html-report.md +++ b/docs-site/docs/reporting/html-report.md @@ -107,6 +107,25 @@ ai: apiKey: ${CLAUDE_API_KEY} ``` +### Choosing an AI provider + +The AI features (`ai.failureAnalysis` and `ai.runSummary`) are not tied to one vendor. Set `ai.provider`: + +| `ai.provider` | Backend | Default `ai.model` | +|---|---|---| +| `anthropic` (default) | Claude Messages API | `claude-haiku-4-5-20251001` | +| `gemini` | Google Gemini `generateContent` | `gemini-2.5-flash` | +| `openai` | Any OpenAI-compatible `/chat/completions` endpoint | `gpt-4o-mini` | + +```yaml +ai: + runSummary: true + provider: gemini + apiKey: ${GEMINI_API_KEY} +``` + +Use `ai.baseUrl` to point `openai` at another compatible server, for example a local Ollama (`http://localhost:11434/v1`) or Azure OpenAI. The same data-minimisation rules apply whichever provider you pick. + The request contains only the run totals and, for the largest same-cause groups (at most 8 groups, 5 test names each), the rule-based category, the error message (first 300 characters) and the test names. It does not send stack traces, steps, page content, or screenshot/trace files. The text is stored in the metrics JSON as `aiRunSummary`. --- diff --git a/src/main/java/com/seleniumboot/ai/AiFailureAnalyzer.java b/src/main/java/com/seleniumboot/ai/AiFailureAnalyzer.java index 007f85a..eec7da4 100644 --- a/src/main/java/com/seleniumboot/ai/AiFailureAnalyzer.java +++ b/src/main/java/com/seleniumboot/ai/AiFailureAnalyzer.java @@ -20,7 +20,7 @@ import java.util.stream.Collectors; /** - * Calls the Claude API to generate a plain-English failure analysis for a failed test. + * Calls the configured LLM (Claude by default; see {@link AiProvider}) to generate a plain-English failure analysis for a failed test. * *

Activated when {@code ai.failureAnalysis: true} and {@code ai.apiKey} are set in * {@code selenium-boot.yml}. The analysis is stored in the test metrics and surfaced in the @@ -37,8 +37,10 @@ * # selenium-boot.yml * ai: * failureAnalysis: true + * provider: anthropic # anthropic | gemini | openai * apiKey: ${CLAUDE_API_KEY} - * model: claude-haiku-4-5-20251001 + * model: claude-haiku-4-5-20251001 # optional; defaults per provider + * baseUrl: # optional endpoint override (OpenAI-compatible servers) * timeoutSeconds: 20 * */ @@ -46,8 +48,6 @@ public final class AiFailureAnalyzer { private static final Logger LOG = Logger.getLogger(AiFailureAnalyzer.class.getName()); - private static final String API_URL = "https://api.anthropic.com/v1/messages"; - private static final String API_VERSION = "2023-06-01"; /** Analysis per failure signature, so one cause costs one API call. */ private static final ConcurrentHashMap> BY_SIGNATURE = @@ -99,8 +99,7 @@ public static void analyze(String testId, String pageUrl, String pageTitle) { } else { String result = null; try { - result = callApi(apiKey, aiCfg.getModel(), buildPrompt(timing, pageUrl, pageTitle), - aiCfg.getTimeoutSeconds()); + result = callApi(aiCfg, apiKey, buildPrompt(timing, pageUrl, pageTitle)); } finally { if (result == null || result.isBlank()) BY_SIGNATURE.remove(signature, mine); mine.complete(result); @@ -185,32 +184,29 @@ static String buildPrompt( // HTTP call to Claude API // ------------------------------------------------------------------ - static String callApi(String apiKey, String model, String prompt, int timeoutSeconds) { + static String callApi(SeleniumBootConfig.Ai cfg, String apiKey, String prompt) { + AiProvider provider = AiProvider.parse(cfg.getProvider()); + if (provider == null) { + LOG.warning("[AiFailureAnalyzer] Unknown ai.provider '" + cfg.getProvider() + + "' (expected anthropic, gemini or openai). Skipping AI call."); + return null; + } try { - String body = buildRequestBody(model, prompt); - + String model = provider.modelOrDefault(cfg.getModel()); + int timeout = cfg.getTimeoutSeconds(); HttpClient client = HttpClient.newBuilder() - .connectTimeout(Duration.ofSeconds(timeoutSeconds)) + .connectTimeout(Duration.ofSeconds(timeout)) .build(); - - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(API_URL)) - .header("x-api-key", apiKey) - .header("anthropic-version", API_VERSION) - .header("content-type", "application/json") - .timeout(Duration.ofSeconds(timeoutSeconds)) - .POST(HttpRequest.BodyPublishers.ofString(body)) - .build(); - - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + HttpResponse response = client.send( + provider.request(cfg.getBaseUrl(), model, apiKey, prompt, timeout), + HttpResponse.BodyHandlers.ofString()); if (response.statusCode() == 200) { - return extractContent(response.body()); - } else { - LOG.warning("[AiFailureAnalyzer] API returned HTTP " + response.statusCode() - + ": " + response.body().substring(0, Math.min(200, response.body().length()))); - return null; + return provider.extract(response.body()); } + LOG.warning("[AiFailureAnalyzer] " + provider + " API returned HTTP " + response.statusCode() + + ": " + response.body().substring(0, Math.min(200, response.body().length()))); + return null; } catch (Exception e) { LOG.warning("[AiFailureAnalyzer] HTTP call failed: " + e.getMessage()); return null; @@ -221,25 +217,15 @@ static String callApi(String apiKey, String model, String prompt, int timeoutSec // JSON helpers (no extra dependency — hand-rolled) // ------------------------------------------------------------------ - private static String buildRequestBody(String model, String prompt) { - String escaped = prompt - .replace("\\", "\\\\") - .replace("\"", "\\\"") - .replace("\n", "\\n") - .replace("\r", "") - .replace("\t", "\\t"); - return "{" - + "\"model\":\"" + model + "\"," - + "\"max_tokens\":512," - + "\"messages\":[{\"role\":\"user\",\"content\":\"" + escaped + "\"}]" - + "}"; - } - /** Extracts {@code content[0].text} from a Claude API response JSON string. */ public static String extractContent(String json) { + return extractField(json, "text"); + } + + /** Value of the first string field called {@code key} (Claude/Gemini: {@code text}; OpenAI: {@code content}). */ + static String extractField(String json, String key) { // Simple extraction — avoids adding Jackson dependency to the hot path - // Pattern: "text": "..." (first occurrence in content array) - int textIdx = json.indexOf("\"text\""); + int textIdx = json.indexOf("\"" + key + "\""); if (textIdx < 0) return null; int colon = json.indexOf(':', textIdx); if (colon < 0) return null; diff --git a/src/main/java/com/seleniumboot/ai/AiProvider.java b/src/main/java/com/seleniumboot/ai/AiProvider.java new file mode 100644 index 0000000..0a89398 --- /dev/null +++ b/src/main/java/com/seleniumboot/ai/AiProvider.java @@ -0,0 +1,115 @@ +package com.seleniumboot.ai; + +import java.net.URI; +import java.net.http.HttpRequest; +import java.time.Duration; + +/** + * The LLM backends the AI features can talk to. Selected with {@code ai.provider}; the request + * shape, auth header and response path are the only things that differ. + * + *

+ */ +enum AiProvider { + + ANTHROPIC("https://api.anthropic.com/v1", "claude-haiku-4-5-20251001"), + GEMINI("https://generativelanguage.googleapis.com/v1beta", "gemini-2.5-flash"), + OPENAI("https://api.openai.com/v1", "gpt-4o-mini"); + + final String defaultBaseUrl; + final String defaultModel; + + AiProvider(String defaultBaseUrl, String defaultModel) { + this.defaultBaseUrl = defaultBaseUrl; + this.defaultModel = defaultModel; + } + + /** Parses {@code ai.provider}; blank means Anthropic. Returns {@code null} for an unknown name. */ + static AiProvider parse(String name) { + if (name == null || name.isBlank()) return ANTHROPIC; + String n = name.strip().toLowerCase(); + if (n.equals("claude")) return ANTHROPIC; + if (n.equals("google")) return GEMINI; + for (AiProvider p : values()) { + if (p.name().equalsIgnoreCase(n)) return p; + } + return null; + } + + /** Model to use: the configured one, or this provider's default when none is set. */ + String modelOrDefault(String configured) { + return configured == null || configured.isBlank() ? defaultModel : configured; + } + + /** Full endpoint URL for {@code model}; {@code baseUrl} overrides the provider default. */ + String endpoint(String baseUrl, String model) { + String base = baseUrl == null || baseUrl.isBlank() ? defaultBaseUrl : baseUrl.strip(); + if (base.endsWith("/")) base = base.substring(0, base.length() - 1); + switch (this) { + case GEMINI: return base + "/models/" + model + ":generateContent"; + case OPENAI: return base + "/chat/completions"; + default: return base + "/messages"; + } + } + + HttpRequest request(String baseUrl, String model, String apiKey, String prompt, int timeoutSeconds) { + HttpRequest.Builder b = HttpRequest.newBuilder() + .uri(URI.create(endpoint(baseUrl, model))) + .header("content-type", "application/json") + .timeout(Duration.ofSeconds(timeoutSeconds)); + switch (this) { + case GEMINI: + b.header("x-goog-api-key", apiKey); + break; + case OPENAI: + b.header("authorization", "Bearer " + apiKey); + break; + default: + b.header("x-api-key", apiKey).header("anthropic-version", "2023-06-01"); + } + return b.POST(HttpRequest.BodyPublishers.ofString(body(model, prompt))).build(); + } + + String body(String model, String prompt) { + String p = "\"" + escape(prompt) + "\""; + switch (this) { + case GEMINI: + // Gemini 2.5 spends output tokens on thinking, so leave headroom for the answer. + return "{\"contents\":[{\"parts\":[{\"text\":" + p + "}]}]," + + "\"generationConfig\":{\"maxOutputTokens\":2048}}"; + case OPENAI: + return "{\"model\":\"" + escape(model) + "\",\"max_tokens\":512," + + "\"messages\":[{\"role\":\"user\",\"content\":" + p + "}]}"; + default: + return "{\"model\":\"" + escape(model) + "\",\"max_tokens\":512," + + "\"messages\":[{\"role\":\"user\",\"content\":" + p + "}]}"; + } + } + + /** The generated text from a successful response body, or {@code null} when absent. */ + String extract(String json) { + return AiFailureAnalyzer.extractField(json, this == OPENAI ? "content" : "text"); + } + + private static String escape(String s) { + StringBuilder sb = new StringBuilder(s.length() + 16); + for (char c : s.toCharArray()) { + switch (c) { + case '\\': sb.append("\\\\"); break; + case '"': sb.append("\\\""); break; + case '\n': sb.append("\\n"); break; + case '\t': sb.append("\\t"); break; + case '\r': break; + default: + if (c < 0x20) sb.append(String.format("\\u%04x", (int) c)); + else sb.append(c); + } + } + return sb.toString(); + } +} diff --git a/src/main/java/com/seleniumboot/ai/AiRunSummarizer.java b/src/main/java/com/seleniumboot/ai/AiRunSummarizer.java index e0c4c12..ee925f1 100644 --- a/src/main/java/com/seleniumboot/ai/AiRunSummarizer.java +++ b/src/main/java/com/seleniumboot/ai/AiRunSummarizer.java @@ -40,7 +40,7 @@ public static String summarize(Collection timings) { } String prompt = buildPrompt(timings); if (prompt == null) return null; - String result = AiFailureAnalyzer.callApi(apiKey, cfg.getModel(), prompt, cfg.getTimeoutSeconds()); + String result = AiFailureAnalyzer.callApi(cfg, apiKey, prompt); return result == null || result.isBlank() ? null : result.strip(); } catch (Exception e) { LOG.warning("[AiRunSummarizer] Run summary failed (non-critical): " + e.getMessage()); diff --git a/src/main/java/com/seleniumboot/config/SeleniumBootConfig.java b/src/main/java/com/seleniumboot/config/SeleniumBootConfig.java index 33ed728..d494c23 100644 --- a/src/main/java/com/seleniumboot/config/SeleniumBootConfig.java +++ b/src/main/java/com/seleniumboot/config/SeleniumBootConfig.java @@ -465,7 +465,9 @@ public static final class Ai { private boolean failureAnalysis = false; private boolean runSummary = false; private String apiKey = null; - private String model = "claude-haiku-4-5-20251001"; + private String provider = "anthropic"; + private String model = null; + private String baseUrl = null; private int timeoutSeconds = 20; public boolean isFailureAnalysis() { return failureAnalysis; } @@ -477,9 +479,18 @@ public static final class Ai { public String getApiKey() { return apiKey; } public void setApiKey(String v) { this.apiKey = v; } + /** {@code anthropic} (default), {@code gemini} or {@code openai} (any OpenAI-compatible endpoint). */ + public String getProvider() { return provider; } + public void setProvider(String v) { this.provider = v; } + + /** Model name; {@code null} means the provider's default. */ public String getModel() { return model; } public void setModel(String v) { this.model = v; } + /** Optional endpoint override, e.g. a local Ollama or Azure OpenAI URL. */ + public String getBaseUrl() { return baseUrl; } + public void setBaseUrl(String v) { this.baseUrl = v; } + public int getTimeoutSeconds() { return timeoutSeconds; } public void setTimeoutSeconds(int v) { this.timeoutSeconds = v; } } diff --git a/src/test/java/com/seleniumboot/ai/AiProviderTest.java b/src/test/java/com/seleniumboot/ai/AiProviderTest.java new file mode 100644 index 0000000..1a14d5d --- /dev/null +++ b/src/test/java/com/seleniumboot/ai/AiProviderTest.java @@ -0,0 +1,111 @@ +package com.seleniumboot.ai; + +import com.seleniumboot.config.SeleniumBootConfig; +import com.sun.net.httpserver.HttpServer; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicReference; + +/** Drives each AI provider against a local fake endpoint — no network, no real key. */ +public class AiProviderTest { + + private HttpServer server; + private final AtomicReference path = new AtomicReference<>(); + private final AtomicReference body = new AtomicReference<>(); + private final AtomicReference headers = new AtomicReference<>(); + private volatile int status = 200; + private volatile String reply = "{}"; + + @BeforeMethod + public void start() throws Exception { + status = 200; + reply = "{}"; + path.set(null); + body.set(null); + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", ex -> { + path.set(ex.getRequestURI().getPath()); + headers.set(ex.getRequestHeaders()); + body.set(new String(ex.getRequestBody().readAllBytes(), StandardCharsets.UTF_8)); + byte[] out = reply.getBytes(StandardCharsets.UTF_8); + ex.sendResponseHeaders(status, out.length); + ex.getResponseBody().write(out); + ex.close(); + }); + server.start(); + } + + @AfterMethod + public void stop() { + server.stop(0); + } + + private SeleniumBootConfig.Ai cfg(String provider, String model) { + SeleniumBootConfig.Ai ai = new SeleniumBootConfig.Ai(); + ai.setProvider(provider); + ai.setModel(model); + ai.setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort() + "/api"); + ai.setTimeoutSeconds(5); + return ai; + } + + @Test + public void anthropicIsTheDefaultAndUsesItsHeadersAndModel() { + reply = "{\"content\":[{\"type\":\"text\",\"text\":\"claude says hi\"}]}"; + SeleniumBootConfig.Ai ai = cfg(null, null); + + Assert.assertEquals(AiFailureAnalyzer.callApi(ai, "k1", "hello \"world\"\nline2"), "claude says hi"); + Assert.assertEquals(path.get(), "/api/messages"); + Assert.assertEquals(headers.get().getFirst("x-api-key"), "k1"); + Assert.assertNotNull(headers.get().getFirst("anthropic-version")); + Assert.assertTrue(body.get().contains("\"model\":\"claude-haiku-4-5-20251001\"")); + Assert.assertTrue(body.get().contains("hello \\\"world\\\"\\nline2")); + } + + @Test + public void geminiUsesGenerateContentAndKeyHeader() { + reply = "{\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"gemini says hi\"}]}}]}"; + SeleniumBootConfig.Ai ai = cfg("gemini", "gemini-2.5-flash"); + + Assert.assertEquals(AiFailureAnalyzer.callApi(ai, "g1", "ping"), "gemini says hi"); + Assert.assertEquals(path.get(), "/api/models/gemini-2.5-flash:generateContent"); + Assert.assertEquals(headers.get().getFirst("x-goog-api-key"), "g1"); + Assert.assertTrue(body.get().contains("\"parts\":[{\"text\":\"ping\"}]")); + } + + @Test + public void openAiCompatibleUsesChatCompletionsAndBearerAuth() { + reply = "{\"choices\":[{\"message\":{\"role\":\"assistant\",\"content\":\"gpt says hi\"}}]}"; + SeleniumBootConfig.Ai ai = cfg("openai", null); + + Assert.assertEquals(AiFailureAnalyzer.callApi(ai, "o1", "ping"), "gpt says hi"); + Assert.assertEquals(path.get(), "/api/chat/completions"); + Assert.assertEquals(headers.get().getFirst("authorization"), "Bearer o1"); + Assert.assertTrue(body.get().contains("\"model\":\"gpt-4o-mini\"")); + } + + @Test + public void providerNameIsCaseInsensitiveAndAliased() { + reply = "{\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"ok\"}]}}]}"; + Assert.assertEquals(AiFailureAnalyzer.callApi(cfg("Google", "m"), "k", "p"), "ok"); + Assert.assertEquals(path.get(), "/api/models/m:generateContent"); + } + + @Test + public void unknownProviderIsSkippedWithoutCallingOut() { + Assert.assertNull(AiFailureAnalyzer.callApi(cfg("bogus", null), "k", "p")); + Assert.assertNull(path.get()); + } + + @Test + public void httpErrorReturnsNull() { + status = 401; + reply = "{\"error\":\"bad key\"}"; + Assert.assertNull(AiFailureAnalyzer.callApi(cfg("anthropic", null), "k", "p")); + } +}