diff --git a/docs-site/docs/cucumber.md b/docs-site/docs/cucumber.md index ced49bc..555ecae 100644 --- a/docs-site/docs/cucumber.md +++ b/docs-site/docs/cucumber.md @@ -13,25 +13,60 @@ Selenium Boot integrates with Cucumber 7 out of the box. Driver lifecycle, the H ## Setup -Add Cucumber to your project alongside Selenium Boot: +Add Cucumber to your project alongside Selenium Boot. + +### Option A: JUnit Platform Engine (Recommended) + +```xml title="pom.xml" + + io.github.seleniumboot + selenium-boot + 3.8.0 + + + + io.cucumber + cucumber-java + 8.0.3 + test + + + + io.cucumber + cucumber-junit-platform-engine + 8.0.3 + test + + + + org.junit.platform + junit-platform-suite + 6.1.3 + test + +``` + +### Option B: TestNG (Legacy / Backwards Compatible) ```xml title="pom.xml" io.github.seleniumboot selenium-boot - 1.11.0 + 3.8.0 io.cucumber cucumber-java - 7.20.1 + 8.0.3 + test io.cucumber cucumber-testng - 7.20.1 + 8.0.3 + test ``` @@ -59,7 +94,38 @@ src/ ## Runner class -Annotate with `@CucumberOptions` and extend `BaseCucumberTest`. No other code needed: +### Option A: JUnit Platform Suite (`@Suite` — Recommended) + +Use JUnit Platform's standard `@Suite` runner with `cucumber-junit-platform-engine`. No base class is needed: + +```java +package com.yourcompany.bdd; + +import org.junit.platform.suite.api.ConfigurationParameter; +import org.junit.platform.suite.api.IncludeEngines; +import org.junit.platform.suite.api.SelectClasspathResource; +import org.junit.platform.suite.api.Suite; + +import static io.cucumber.junit.platform.engine.Constants.GLUE_PROPERTY_NAME; +import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME; + +@Suite +@IncludeEngines("cucumber") +@SelectClasspathResource("features") +@ConfigurationParameter(key = GLUE_PROPERTY_NAME, value = "com.yourcompany.bdd.steps, com.seleniumboot.cucumber") +@ConfigurationParameter(key = PLUGIN_PROPERTY_NAME, value = "pretty, com.seleniumboot.cucumber.CucumberStepLogger") +public class CucumberSuiteTest {} +``` + +Alternatively, configure glue and plugins in `src/test/resources/junit-platform.properties`: +```properties title="junit-platform.properties" +cucumber.glue=com.yourcompany.bdd.steps, com.seleniumboot.cucumber +cucumber.plugin=pretty, com.seleniumboot.cucumber.CucumberStepLogger +``` + +### Option B: TestNG Runner (`BaseCucumberTest` — Deprecated in 3.9.0) + +Annotate with `@CucumberOptions` and extend `BaseCucumberTest`: ```java @CucumberOptions( @@ -70,6 +136,10 @@ Annotate with `@CucumberOptions` and extend `BaseCucumberTest`. No other code ne public class CucumberRunner extends BaseCucumberTest {} ``` +:::note Deprecation Notice +`BaseCucumberTest` is deprecated as of Selenium Boot 3.9.0 because upstream `AbstractTestNGCucumberTests` has been deprecated for removal in Cucumber 8.x. `BaseCucumberTest` continues to function for all TestNG suites throughout the 3.x series and will be removed in 4.0.0. +::: + `"com.seleniumboot.cucumber"` in `glue` is required — it tells Cucumber where to find `CucumberHooks`, which manages the driver lifecycle. `CucumberStepLogger` in `plugin` streams Gherkin step names into the Selenium Boot HTML report step timeline. diff --git a/src/main/java/com/seleniumboot/cucumber/CucumberHooks.java b/src/main/java/com/seleniumboot/cucumber/CucumberHooks.java index bc653e0..9631900 100644 --- a/src/main/java/com/seleniumboot/cucumber/CucumberHooks.java +++ b/src/main/java/com/seleniumboot/cucumber/CucumberHooks.java @@ -216,7 +216,7 @@ private static void applyRetryTag(Scenario scenario) { CucumberRetryContext.clear(); } - private static void checkCucumberQuarantine(Scenario scenario) { + static void checkCucumberQuarantine(Scenario scenario) { try { com.seleniumboot.config.SeleniumBootConfig.Quarantine cfg = SeleniumBootContext.getConfig().getQuarantine(); @@ -228,7 +228,7 @@ private static void checkCucumberQuarantine(Scenario scenario) { for (String t : scenario.getSourceTagNames()) { String normalized = t.startsWith("@") ? t.substring(1) : t; if (normalized.equalsIgnoreCase(configuredTag)) { - throw new org.testng.SkipException( + abortQuarantinedScenario( "[Quarantined] " + scenario.getName() + " — @" + configuredTag + " tag present" ); } @@ -240,16 +240,32 @@ private static void checkCucumberQuarantine(Scenario scenario) { java.util.Collection tags = scenario.getSourceTagNames(); if (QuarantineLoader.isQuarantinedScenario(tags, featureUri, scenarioName)) { - throw new org.testng.SkipException( + abortQuarantinedScenario( "[Quarantined] " + scenarioName + " — " + QuarantineLoader.getScenarioReason(tags, featureUri, scenarioName) ); } - } catch (org.testng.SkipException e) { + } catch (org.testng.SkipException | org.opentest4j.TestAbortedException e) { throw e; } catch (Exception ignored) {} } + private static void abortQuarantinedScenario(String reason) { + if (isTestNgExecution()) { + throw new org.testng.SkipException(reason); + } else { + throw new org.opentest4j.TestAbortedException(reason); + } + } + + private static boolean isTestNgExecution() { + try { + return org.testng.Reporter.getCurrentTestResult() != null; + } catch (Throwable ignored) { + return false; + } + } + private void safeQuitDriver() { try { if (DriverManager.shouldQuitAfterTest()) DriverManager.quitDriver(); diff --git a/src/main/java/com/seleniumboot/junit5/SeleniumBootLauncherListener.java b/src/main/java/com/seleniumboot/junit5/SeleniumBootLauncherListener.java index 4745bcf..c5d4716 100644 --- a/src/main/java/com/seleniumboot/junit5/SeleniumBootLauncherListener.java +++ b/src/main/java/com/seleniumboot/junit5/SeleniumBootLauncherListener.java @@ -9,6 +9,8 @@ import com.seleniumboot.internal.SeleniumBootContext; import com.seleniumboot.lifecycle.FrameworkBootstrap; import com.seleniumboot.metrics.ExecutionMetrics; +import com.seleniumboot.precondition.ApiHealthChecker; +import com.seleniumboot.precondition.PreConditionRegistry; import com.seleniumboot.reporting.JUnitXmlReporter; import com.seleniumboot.precondition.PreConditionRunner; import com.seleniumboot.reporting.ReportAdapterRegistry; @@ -16,6 +18,8 @@ import org.junit.platform.launcher.TestExecutionListener; import org.junit.platform.launcher.TestPlan; +import java.util.concurrent.atomic.AtomicBoolean; + /** * JUnit Platform launcher listener that generates Selenium Boot reports after * the entire test plan finishes — the JUnit 5 equivalent of @@ -30,6 +34,8 @@ @SeleniumBootApi(since = "1.9.0") public class SeleniumBootLauncherListener implements TestExecutionListener { + private static final AtomicBoolean suiteStarted = new AtomicBoolean(false); + /** * Fires suite-start hooks once per test plan — the JUnit 5 equivalent of * {@code SuiteExecutionListener.onStart()}. Bootstrap is idempotent, so this is @@ -38,9 +44,13 @@ public class SeleniumBootLauncherListener implements TestExecutionListener { */ @Override public void testPlanExecutionStarted(TestPlan testPlan) { + if (!suiteStarted.compareAndSet(false, true)) return; try { FrameworkBootstrap.initialize(); + ApiHealthChecker.clearCache(); + PreConditionRegistry.loadAll(); HookRegistry.onSuiteStart(); + TestManagementReporter.getInstance().onSuiteStart(); } catch (Exception e) { System.err.println("[Selenium Boot] Suite start hooks skipped: " + e.getMessage()); } @@ -70,6 +80,8 @@ public void testPlanExecutionFinished(TestPlan testPlan) { throw e; // re-throw CI gate failures (pass-rate / flakiness) } catch (Exception e) { System.err.println("[Selenium Boot] Report generation failed: " + e.getMessage()); + } finally { + suiteStarted.set(false); } } } diff --git a/src/main/java/com/seleniumboot/testmanagement/TestManagementReporter.java b/src/main/java/com/seleniumboot/testmanagement/TestManagementReporter.java index a76391a..252dd96 100644 --- a/src/main/java/com/seleniumboot/testmanagement/TestManagementReporter.java +++ b/src/main/java/com/seleniumboot/testmanagement/TestManagementReporter.java @@ -53,6 +53,7 @@ public final class TestManagementReporter { private int activeTestRailRunId = 0; private boolean testRailEnabled = false; private boolean xrayEnabled = false; + private final java.util.concurrent.atomic.AtomicBoolean suiteStarted = new java.util.concurrent.atomic.AtomicBoolean(false); private TestManagementReporter() {} @@ -67,6 +68,7 @@ public static TestManagementReporter getInstance() { * and optionally create a TestRail run. */ public void onSuiteStart() { + if (!suiteStarted.compareAndSet(false, true)) return; SeleniumBootConfig.TestManagement cfg = loadConfig(); if (cfg == null) return; @@ -145,6 +147,7 @@ public void onSuiteEnd() { activeTestRailRunId = 0; testRailClient = null; xrayClient = null; + suiteStarted.set(false); } // ── internals ───────────────────────────────────────────────────────────── diff --git a/src/test/java/com/seleniumboot/unit/CucumberQuarantineTest.java b/src/test/java/com/seleniumboot/unit/CucumberQuarantineTest.java new file mode 100644 index 0000000..77ee066 --- /dev/null +++ b/src/test/java/com/seleniumboot/unit/CucumberQuarantineTest.java @@ -0,0 +1,213 @@ +package com.seleniumboot.unit; + +import com.seleniumboot.cucumber.CucumberHooks; +import com.seleniumboot.internal.SeleniumBootContext; +import com.seleniumboot.quarantine.QuarantineLoader; +import io.cucumber.java.Before; +import io.cucumber.java.Scenario; +import io.cucumber.java.en.Given; +import org.junit.platform.launcher.Launcher; +import org.junit.platform.launcher.LauncherDiscoveryRequest; +import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; +import org.junit.platform.launcher.core.LauncherFactory; +import org.junit.platform.launcher.listeners.SummaryGeneratingListener; +import org.junit.platform.launcher.listeners.TestExecutionSummary; +import org.opentest4j.TestAbortedException; +import org.testng.ITestResult; +import org.testng.Reporter; +import org.testng.SkipException; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.io.File; +import java.io.FileWriter; +import java.lang.reflect.Method; +import java.net.URI; +import java.util.Collections; +import java.util.List; + +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectFile; +import static org.mockito.Mockito.*; +import static org.testng.Assert.*; + +/** + * Unit tests verifying quarantine skip behavior for Cucumber scenarios under both + * TestNG and JUnit Platform. + */ +public class CucumberQuarantineTest { + + private Method checkQuarantineMethod; + private File tempQuarantineFile; + + @BeforeMethod + public void setup() throws Exception { + checkQuarantineMethod = CucumberHooks.class.getDeclaredMethod("checkCucumberQuarantine", Scenario.class); + checkQuarantineMethod.setAccessible(true); + QuarantineLoader.reload(); + } + + @AfterMethod + public void cleanup() { + Reporter.setCurrentTestResult(null); + System.clearProperty("selenium.boot.quarantine"); + QuarantineLoader.reload(); + if (tempQuarantineFile != null) { + tempQuarantineFile.delete(); + } + } + + // ── TestNG Environment (Reporter.getCurrentTestResult() != null) ─────────── + + @Test + public void checkQuarantine_underTestNg_throwsSkipException_whenTagged() { + ITestResult mockResult = mock(ITestResult.class); + Reporter.setCurrentTestResult(mockResult); + + Scenario scenario = mock(Scenario.class); + when(scenario.getName()).thenReturn("Quarantined Scenario"); + when(scenario.getSourceTagNames()).thenReturn(List.of("@quarantine")); + + try { + checkQuarantineMethod.invoke(null, scenario); + fail("Expected SkipException under TestNG"); + } catch (Exception e) { + assertTrue(e.getCause() instanceof SkipException, + "Expected SkipException under TestNG, but got: " + e.getCause()); + assertTrue(e.getCause().getMessage().contains("[Quarantined]")); + } + } + + @Test + public void checkQuarantine_underTestNg_throwsSkipException_whenInYaml() throws Exception { + ITestResult mockResult = mock(ITestResult.class); + Reporter.setCurrentTestResult(mockResult); + + createQuarantineYaml("quarantine:\n - login.feature\n"); + + Scenario scenario = mock(Scenario.class); + when(scenario.getName()).thenReturn("Login with 2FA"); + when(scenario.getUri()).thenReturn(URI.create("classpath:features/login.feature")); + when(scenario.getSourceTagNames()).thenReturn(Collections.emptyList()); + + try { + checkQuarantineMethod.invoke(null, scenario); + fail("Expected SkipException under TestNG"); + } catch (Exception e) { + assertTrue(e.getCause() instanceof SkipException, + "Expected SkipException under TestNG, but got: " + e.getCause()); + assertTrue(e.getCause().getMessage().contains("[Quarantined]")); + } + } + + // ── JUnit Platform Environment (Reporter.getCurrentTestResult() == null) ── + + @Test + public void checkQuarantine_underJUnitPlatform_throwsTestAbortedException_whenTagged() { + Reporter.setCurrentTestResult(null); // Ensure null to simulate JUnit Platform execution + + Scenario scenario = mock(Scenario.class); + when(scenario.getName()).thenReturn("Quarantined Scenario"); + when(scenario.getSourceTagNames()).thenReturn(List.of("@quarantine")); + + try { + checkQuarantineMethod.invoke(null, scenario); + fail("Expected TestAbortedException under JUnit Platform"); + } catch (Exception e) { + assertTrue(e.getCause() instanceof TestAbortedException, + "Expected TestAbortedException under JUnit Platform, but got: " + e.getCause()); + assertTrue(e.getCause().getMessage().contains("[Quarantined]")); + } + } + + @Test + public void checkQuarantine_underJUnitPlatform_throwsTestAbortedException_whenInYaml() throws Exception { + Reporter.setCurrentTestResult(null); + + createQuarantineYaml("quarantine:\n - test: checkout.feature\n reason: \"Payment outage\"\n"); + + Scenario scenario = mock(Scenario.class); + when(scenario.getName()).thenReturn("Checkout with card"); + when(scenario.getUri()).thenReturn(URI.create("classpath:features/checkout.feature")); + when(scenario.getSourceTagNames()).thenReturn(Collections.emptyList()); + + try { + checkQuarantineMethod.invoke(null, scenario); + fail("Expected TestAbortedException under JUnit Platform"); + } catch (Exception e) { + assertTrue(e.getCause() instanceof TestAbortedException, + "Expected TestAbortedException under JUnit Platform, but got: " + e.getCause()); + assertTrue(e.getCause().getMessage().contains("Payment outage")); + } + } + + // ── Non-quarantined scenario ─────────────────────────────────────────────── + + @Test + public void checkQuarantine_notQuarantined_doesNotThrow() throws Exception { + Scenario scenario = mock(Scenario.class); + when(scenario.getName()).thenReturn("Normal Scenario"); + when(scenario.getSourceTagNames()).thenReturn(List.of("@smoke")); + when(scenario.getUri()).thenReturn(URI.create("classpath:features/smoke.feature")); + + // Must not throw any exception + checkQuarantineMethod.invoke(null, scenario); + } + + // ── End-to-end Cucumber JUnit Platform Engine execution ──────────────────── + + public static class QuarantineStepDefinitions { + @Before("@quarantine") + public void beforeScenario(Scenario scenario) { + CucumberHooks hooks = new CucumberHooks(); + // In a real run, Cucumber invokes hooks; here we directly invoke quarantine check + try { + Method m = CucumberHooks.class.getDeclaredMethod("checkCucumberQuarantine", Scenario.class); + m.setAccessible(true); + m.invoke(null, scenario); + } catch (Exception e) { + if (e.getCause() instanceof RuntimeException re) throw re; + throw new RuntimeException(e); + } + } + + @Given("a step in quarantined scenario") + public void a_step() {} + } + + @Test + public void cucumberJUnitPlatformEngine_reportsQuarantinedScenarioAsAborted() throws Exception { + Reporter.setCurrentTestResult(null); + + File tempFeature = File.createTempFile("quarantine_e2e_", ".feature"); + tempFeature.deleteOnExit(); + try (FileWriter w = new FileWriter(tempFeature)) { + w.write("@quarantine\nFeature: Quarantine E2E\n Scenario: Quarantined\n Given a step in quarantined scenario\n"); + } + + LauncherDiscoveryRequest request = LauncherDiscoveryRequestBuilder.request() + .selectors(selectFile(tempFeature)) + .configurationParameter("cucumber.glue", "com.seleniumboot.unit") + .build(); + + Launcher launcher = LauncherFactory.create(); + SummaryGeneratingListener listener = new SummaryGeneratingListener(); + launcher.registerTestExecutionListeners(listener); + launcher.execute(request); + + TestExecutionSummary summary = listener.getSummary(); + assertEquals(summary.getTestsAbortedCount(), 1, "Quarantined scenario must be recorded as ABORTED on JUnit Platform"); + assertEquals(summary.getTestsFailedCount(), 0, "Quarantined scenario must NOT be recorded as FAILED on JUnit Platform"); + } + + private void createQuarantineYaml(String content) throws Exception { + tempQuarantineFile = File.createTempFile("quarantine-test-", ".yml"); + tempQuarantineFile.deleteOnExit(); + try (FileWriter fw = new FileWriter(tempQuarantineFile)) { + fw.write(content); + } + System.setProperty("selenium.boot.quarantine", tempQuarantineFile.getAbsolutePath()); + QuarantineLoader.reload(); + } +} + diff --git a/src/test/java/com/seleniumboot/unit/SeleniumBootLauncherListenerTest.java b/src/test/java/com/seleniumboot/unit/SeleniumBootLauncherListenerTest.java index f3e4cbf..2a50265 100644 --- a/src/test/java/com/seleniumboot/unit/SeleniumBootLauncherListenerTest.java +++ b/src/test/java/com/seleniumboot/unit/SeleniumBootLauncherListenerTest.java @@ -3,13 +3,17 @@ import com.seleniumboot.hooks.ExecutionHook; import com.seleniumboot.hooks.HookRegistry; import com.seleniumboot.junit5.SeleniumBootLauncherListener; +import com.seleniumboot.precondition.BaseConditions; +import com.seleniumboot.precondition.PreConditionRegistry; +import com.seleniumboot.testmanagement.TestManagementReporter; import org.testng.annotations.AfterMethod; import org.testng.annotations.Test; import java.lang.reflect.Field; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; -import static org.testng.Assert.assertEquals; +import static org.testng.Assert.*; /** * Unit tests for {@link SeleniumBootLauncherListener} — the JUnit Platform @@ -22,6 +26,12 @@ public void resetRegistry() throws Exception { Field f = HookRegistry.class.getDeclaredField("hooks"); f.setAccessible(true); ((List) f.get(null)).clear(); + + Field suiteStartedField = SeleniumBootLauncherListener.class.getDeclaredField("suiteStarted"); + suiteStartedField.setAccessible(true); + ((AtomicBoolean) suiteStartedField.get(null)).set(false); + + TestManagementReporter.getInstance().onSuiteEnd(); } @Test @@ -34,6 +44,43 @@ public void testPlanExecutionStarted_firesSuiteStartHookOnce() { assertEquals(hook.suiteStarts, 1, "onSuiteStart must fire once per test plan"); } + @Test + public void testPlanExecutionStarted_isIdempotentOnRepeatedCalls() { + CountingHook hook = new CountingHook(); + HookRegistry.register(hook); + + SeleniumBootLauncherListener listener = new SeleniumBootLauncherListener(); + listener.testPlanExecutionStarted(null); + listener.testPlanExecutionStarted(null); + listener.testPlanExecutionStarted(null); + + assertEquals(hook.suiteStarts, 1, "onSuiteStart must not double-fire when testPlanExecutionStarted is called repeatedly"); + } + + @Test + public void testPlanExecutionFinished_resetsGuardForNextRun() { + CountingHook hook = new CountingHook(); + HookRegistry.register(hook); + + SeleniumBootLauncherListener listener = new SeleniumBootLauncherListener(); + listener.testPlanExecutionStarted(null); + assertEquals(hook.suiteStarts, 1); + + listener.testPlanExecutionFinished(null); + + // Next run should fire onSuiteStart again + listener.testPlanExecutionStarted(null); + assertEquals(hook.suiteStarts, 2, "After testPlanExecutionFinished, a new test plan should fire onSuiteStart again"); + } + + @Test + public void testPlanExecutionStarted_loadsPreConditions() { + new SeleniumBootLauncherListener().testPlanExecutionStarted(null); + + // PreConditionRegistry.loadAll() is safe and idempotent + assertNotNull(PreConditionRegistry.getProviders()); + } + private static final class CountingHook implements ExecutionHook { int suiteStarts;