diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ea4f603..d5c20a9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: - name: Setup PHP uses: shivammathur/setup-php@v2 with: - php-version: '8.2' + php-version: '8.3' tools: composer - name: Install dependencies diff --git a/cli.php b/cli.php index 7e3b5ea..939921e 100644 --- a/cli.php +++ b/cli.php @@ -13,12 +13,15 @@ * argv[3] -> case (optional) * argv[4] -> -report=html (optional) * argv[5] -> -output="./output/out.html" (optional), if no path is specified it will be saved in the root + * JUnit example -> php cli.php tests -report=junit reports/test-results.xml * */ $output = '.'. DIRECTORY_SEPARATOR; $report = 'html'; $output_index = 0; $report_index = 0; +$junit_output = 'test-results.xml'; +$junit_output_index = 0; $test_start_time = 0; $test_end_time = 0; $test_time = 0; @@ -33,6 +36,13 @@ $report = end($type_out); $argc--; $report_index = $i; + + if ($report == 'junit' && isset($argv[$i + 1]) && substr($argv[$i + 1], 0, 1) != '-') + { + $junit_output = $argv[$i + 1]; + $junit_output_index = $i + 1; + $argc--; + } } if ($search == '-output') @@ -47,6 +57,7 @@ // remove output and report parameters, so the rest are in the same order for the execution (suite, class, function) if ($output_index > 0) unset($argv[$output_index]); if ($report_index > 0) unset($argv[$report_index]); +if ($junit_output_index > 0) unset($argv[$junit_output_index]); $argv = array_values($argv); if ($argc < 2) @@ -105,7 +116,20 @@ } else if ($report == 'junit') { - $exit_code = $run->generate_junit_xml(); + $builder = new \CaboLabs\Debbie\JunitXmlBuilder(); + $generated = $builder->generateFromReports( + $run->get_reports(), + $junit_output + ); + + if (!$generated) + { + $exit_code = 1; + } + else + { + $exit_code = $builder->hasFailures() ? 1 : 0; + } } else { diff --git a/src/JunitXmlBuilder.php b/src/JunitXmlBuilder.php index 7b71bb6..905dbb1 100644 --- a/src/JunitXmlBuilder.php +++ b/src/JunitXmlBuilder.php @@ -5,19 +5,15 @@ /** * Builder class for creating JUnit XML test result files */ -class JUnitXMLBuilder +class JunitXmlBuilder { - private $testSuites = []; - private $startTime; + const ROOT_REPORT_NAME = 'Amplify - Tests Report'; - public function __construct() - { - $this->startTime = microtime(true); - } + private $testSuites = []; /** * Create a new test suite - * + * * @param string $name The name of the test suite (typically the test class name) * @return TestSuite */ @@ -28,18 +24,154 @@ public function addTestSuite($name) return $suite; } + /** + * * + * Historical context: + * before this method existed, DebbieRun transformed + * the reports manually before passing them to the builder. Now this + * function centralizes that conversion in JunitXmlBuilder, allowing + * DebbieRun to provide its original reports without losing information. + * Previously, DebbieRun::generate_junit_xml() only used part of the + * available information: it kept the first failure or error, ignored OK + * assertions, could discard additional failures, and did not always pass + * all error details and traces. This function now preserves all received + * assertions, failures, errors, exceptions, parameters, traces, and test + * output when converting the reports. + * + * Translate DebbieRun reports into the builder's internal format. + * + * This method converts the complete reports returned by DebbieRun into + * TestSuite and TestCase objects, without changing or duplicating report + * transformation logic in DebbieRun itself. + * + * @param array $reports Reports returned by DebbieRun::get_reports() + * @return self + */ + public function addReports(array $reports) + { + $suitesByNamespace = []; + + foreach ($reports as $suiteReports) { + foreach ($suiteReports as $className => $tests) { + $namespace = $this->extractNamespace($className); + + if (!isset($suitesByNamespace[$namespace])) { + $suitesByNamespace[$namespace] = $this->addTestSuite($namespace); + } + $suite = $suitesByNamespace[$namespace]; + + foreach ($tests as $testName => $report) { + $testCase = $suite->addTestCase($testName, $className); + + foreach (isset($report['asserts']) ? $report['asserts'] : [] as $assert) { + $type = isset($assert['type']) ? $assert['type'] : ''; + $message = $this->buildAssertionMessage( + $testName, + isset($assert['msg']) ? $assert['msg'] : '' + ); + + if ($type === 'OK') { + $testCase->addAssertion(); + } elseif ($type === 'FAIL') { + $testCase->addFailure($message, 'AssertionFailedError', $assert); + } elseif ($type === 'ERROR' || $type === 'EXCEPTION') { + $errorType = $type === 'EXCEPTION' ? 'Exception' : 'RuntimeError'; + $testCase->addError($message, $errorType, $assert); + } elseif ($type === 'SKIPPED') { + $testCase->setSkipped($message ?: 'Test skipped'); + } + } + + if (!empty($report['output'])) { + $testCase->addSystemOut($report['output']); + } + + $testCase->finish(); + } + } + } + + return $this; + } + + /** + * Prefix an assertion message with its test function name. + * + * @param string $testName Test function name + * @param string $message Assertion message + * @return string + */ + private function buildAssertionMessage($testName, $message) + { + return '[' . $testName . '] ' . $message; + } + + /** + * Get the namespace. + * + * @param string $className Fully qualified class name + * @return string + */ + private function extractNamespace($className) + { + $pos = strrpos($className, '\\'); + return $pos !== false ? substr($className, 0, $pos) : $className; + } + + /** + * Clear suites previously added to the builder. + * + * @return self + */ + public function reset() + { + $this->testSuites = []; + return $this; + } + + /** + * Convert DebbieRun reports and save the resulting JUnit XML. + * + * @param array $reports Reports returned by DebbieRun::get_reports() + * @param string $filepath Path where to save the XML file + * @return bool True on success, false on failure + */ + public function generateFromReports(array $reports, $filepath) + { + $this->reset(); + $this->addReports($reports); + return $this->saveToFile($filepath); + } + + /** + * Check whether a failure or error was recorded. + * + * @return bool + */ + public function hasFailures() + { + foreach ($this->testSuites as $suite) { + $stats = $suite->getStatistics(); + if ($stats['failures'] > 0 || $stats['errors'] > 0) { + return true; + } + } + + return false; + } + /** * Generate the JUnit XML string - * + * * @return string The complete JUnit XML */ public function toXML() { - $totalTime = microtime(true) - $this->startTime; $totalTests = 0; $totalFailures = 0; $totalErrors = 0; $totalSkipped = 0; + $totalTime = 0; // Calculate totals foreach ($this->testSuites as $suite) { @@ -48,12 +180,14 @@ public function toXML() $totalFailures += $stats['failures']; $totalErrors += $stats['errors']; $totalSkipped += $stats['skipped']; + $totalTime += $stats['time']; } $xml = new \DOMDocument('1.0', 'UTF-8'); $xml->formatOutput = true; $testsuites = $xml->createElement('testsuites'); + $testsuites->setAttribute('name', self::ROOT_REPORT_NAME); $testsuites->setAttribute('tests', $totalTests); $testsuites->setAttribute('failures', $totalFailures); $testsuites->setAttribute('errors', $totalErrors); @@ -72,13 +206,17 @@ public function toXML() /** * Save the JUnit XML to a file - * + * * @param string $filepath Path to save the XML file * @return bool True on success, false on failure */ public function saveToFile($filepath) { $xml = $this->toXML(); + $directory = dirname($filepath); + if (!is_dir($directory) && !mkdir($directory, 0755, true)) { + return false; + } return file_put_contents($filepath, $xml) !== false; } } @@ -90,30 +228,34 @@ class TestSuite { private $name; private $testCases = []; - private $startTime; + /** + * Create a test suite. + * + * @param string $name Test suite name + */ public function __construct($name) { $this->name = $name; - $this->startTime = microtime(true); } /** * Add a test case to this suite - * + * * @param string $name The test case name (typically the test method name) + * @param string|null $classname The fully qualified test class name (defaults to the suite name) * @return TestCase */ - public function addTestCase($name) + public function addTestCase($name, $classname = null) { - $testCase = new TestCase($name, $this->name); + $testCase = new TestCase($name, $classname !== null ? $classname : $this->name); $this->testCases[] = $testCase; return $testCase; } /** * Get statistics for this test suite - * + * * @return array Array with keys: tests, failures, errors, skipped */ public function getStatistics() @@ -123,6 +265,7 @@ public function getStatistics() 'failures' => 0, 'errors' => 0, 'skipped' => 0, + 'assertions' => 0, 'time' => 0 ]; @@ -136,6 +279,7 @@ public function getStatistics() if ($testCase->isSkipped()) { $stats['skipped']++; } + $stats['assertions'] += $testCase->getAssertionCount(); $stats['time'] += $testCase->getTime(); } @@ -144,21 +288,34 @@ public function getStatistics() /** * Convert this test suite to a DOMElement - * + * * @param \DOMDocument $doc The document to create elements in * @return \DOMElement */ public function toXMLElement(\DOMDocument $doc) { $stats = $this->getStatistics(); - + $element = $doc->createElement('testsuite'); $element->setAttribute('name', $this->name); $element->setAttribute('tests', $stats['tests']); $element->setAttribute('failures', $stats['failures']); $element->setAttribute('errors', $stats['errors']); $element->setAttribute('skipped', $stats['skipped']); + $element->setAttribute('assertions', $stats['assertions']); $element->setAttribute('time', number_format($stats['time'], 6, '.', '')); + $element->setAttribute('timestamp', gmdate('c')); + + $properties = $doc->createElement('properties'); + $phpProperty = $doc->createElement('property'); + $phpProperty->setAttribute('name', 'php.version'); + $phpProperty->setAttribute('value', phpversion()); + $properties->appendChild($phpProperty); + $osProperty = $doc->createElement('property'); + $osProperty->setAttribute('name', 'os'); + $osProperty->setAttribute('value', php_uname('s')); + $properties->appendChild($osProperty); + $element->appendChild($properties); foreach ($this->testCases as $testCase) { $testCaseElement = $testCase->toXMLElement($doc); @@ -177,13 +334,19 @@ class TestCase private $name; private $classname; private $time; - private $failure = null; - private $error = null; + private $failures = []; + private $errors = []; private $skipped = false; private $startTime; private $systemOut = ''; - private $systemErr = ''; + private $assertions = 0; + /** + * Create a test case. + * + * @param string $name Test case name + * @param string $classname Test class name + */ public function __construct($name, $classname) { $this->name = $name; @@ -197,12 +360,15 @@ public function __construct($name, $classname) */ public function finish() { - $this->time = microtime(true) - $this->startTime; + if ($this->time === 0) { + $this->time = microtime(true) - $this->startTime; + } + return $this; } /** * Add a failure to this test case - * + * * @param string $message The failure message * @param string $type The failure type (default: 'AssertionFailure') * @param string $details Additional details/stack trace @@ -210,7 +376,8 @@ public function finish() */ public function addFailure($message, $type = 'AssertionFailure', $details = '') { - $this->failure = [ + $this->assertions++; + $this->failures[] = [ 'message' => $message, 'type' => $type, 'details' => $details @@ -220,7 +387,7 @@ public function addFailure($message, $type = 'AssertionFailure', $details = '') /** * Add an error to this test case - * + * * @param string $message The error message * @param string $type The error type (default: 'RuntimeError') * @param string $details Additional details/stack trace @@ -228,7 +395,8 @@ public function addFailure($message, $type = 'AssertionFailure', $details = '') */ public function addError($message, $type = 'RuntimeError', $details = '') { - $this->error = [ + $this->assertions++; + $this->errors[] = [ 'message' => $message, 'type' => $type, 'details' => $details @@ -236,9 +404,20 @@ public function addError($message, $type = 'RuntimeError', $details = '') return $this; } + /** + * Record a successful assertion. + * + * @return self + */ + public function addAssertion() + { + $this->assertions++; + return $this; + } + /** * Mark this test case as skipped - * + * * @param string $message Optional skip message * @return self */ @@ -250,7 +429,7 @@ public function setSkipped($message = 'Test skipped') /** * Add standard output to this test case - * + * * @param string $output The output text * @return self */ @@ -260,41 +439,29 @@ public function addSystemOut($output) return $this; } - /** - * Add error output to this test case - * - * @param string $output The error output text - * @return self - */ - public function addSystemErr($output) - { - $this->systemErr .= $output; - return $this; - } - /** * Check if this test case has a failure - * + * * @return bool */ public function hasFailure() { - return $this->failure !== null; + return !empty($this->failures); } /** * Check if this test case has an error - * + * * @return bool */ public function hasError() { - return $this->error !== null; + return !empty($this->errors); } /** * Check if this test case is skipped - * + * * @return bool */ public function isSkipped() @@ -304,45 +471,59 @@ public function isSkipped() /** * Get the execution time - * + * * @return float */ public function getTime() { + $this->finish(); return $this->time; } + /** + * Get the number of recorded assertion results. + * + * @return int + */ + public function getAssertionCount() + { + return $this->assertions; + } + /** * Convert this test case to a DOMElement - * + * * @param \DOMDocument $doc The document to create elements in * @return \DOMElement */ public function toXMLElement(\DOMDocument $doc) { + $this->finish(); + $element = $doc->createElement('testcase'); $element->setAttribute('name', $this->name); $element->setAttribute('classname', $this->classname); + $element->setAttribute('assertions', $this->getAssertionCount()); $element->setAttribute('time', number_format($this->time, 6, '.', '')); - // Add failure element if present - if ($this->failure !== null) { + // Add every failure recorded for this test case. + foreach ($this->failures as $failure) { $failureElement = $doc->createElement('failure'); - $failureElement->setAttribute('message', $this->failure['message']); - $failureElement->setAttribute('type', $this->failure['type']); - if (!empty($this->failure['details'])) { - $failureElement->appendChild($doc->createTextNode($this->failure['details'])); + $failureElement->setAttribute('message', $failure['message']); + $failureElement->setAttribute('type', $failure['type']); + if (!empty($failure['details'])) { + $failureElement->appendChild($this->createSafeCDATA($doc, $this->formatDetails($failure['details']))); } $element->appendChild($failureElement); } - // Add error element if present - if ($this->error !== null) { + // Add every error recorded for this test case. + foreach ($this->errors as $error) { $errorElement = $doc->createElement('error'); - $errorElement->setAttribute('message', $this->error['message']); - $errorElement->setAttribute('type', $this->error['type']); - if (!empty($this->error['details'])) { - $errorElement->appendChild($doc->createTextNode($this->error['details'])); + $errorElement->setAttribute('message', $error['message']); + $errorElement->setAttribute('type', $error['type']); + if (!empty($error['details'])) { + $errorElement->appendChild($this->createSafeCDATA($doc, $this->formatDetails($error['details']))); } $element->appendChild($errorElement); } @@ -357,17 +538,75 @@ public function toXMLElement(\DOMDocument $doc) // Add system-out if present if (!empty($this->systemOut)) { $systemOutElement = $doc->createElement('system-out'); - $systemOutElement->appendChild($doc->createCDATASection($this->systemOut)); + $systemOutElement->appendChild($this->createSafeCDATA($doc, $this->systemOut)); $element->appendChild($systemOutElement); } - // Add system-err if present - if (!empty($this->systemErr)) { - $systemErrElement = $doc->createElement('system-err'); - $systemErrElement->appendChild($doc->createCDATASection($this->systemErr)); - $element->appendChild($systemErrElement); + return $element; + } + + /** + * Create a CDATA fragment that remains valid when content contains ]]>. + * + * @param \DOMDocument $doc XML document + * @param string $content Content to wrap in CDATA + * @return \DOMDocumentFragment + */ + private function createSafeCDATA(\DOMDocument $doc, $content) + { + $parts = explode(']]>', (string) $content); + $fragment = $doc->createDocumentFragment(); + $lastPart = count($parts) - 1; + + foreach ($parts as $index => $part) { + if ($index === 0) { + $part .= $index < $lastPart ? ']]' : ''; + } else { + $part = '>' . $part . ($index < $lastPart ? ']]' : ''); + } + + $fragment->appendChild($doc->createCDATASection($part)); } - return $element; + return $fragment; + } + + /** + * Convert assertion details or stack frames to readable text. + * + * @param mixed $details Assertion details + * @return string + */ + private function formatDetails($details) + { + if (is_array($details) && (isset($details['trace']) || isset($details['params']) || isset($details['msg']))) { + $formatted = isset($details['msg']) ? 'Assertion: ' . $details['msg'] : ''; + if (!empty($details['params'])) { + $formatted .= "\n\nParameters:\n" . $details['params']; + } + if (!empty($details['trace'])) { + $formatted .= "\n\nStack Trace:\n" . $this->formatDetails($details['trace']); + } + return $formatted; + } + + if (is_array($details)) { + $lines = []; + foreach ($details as $frame) { + if (is_array($frame)) { + $file = isset($frame['file']) ? $frame['file'] : 'unknown'; + $line = isset($frame['line']) ? $frame['line'] : '?'; + $class = isset($frame['class']) ? $frame['class'] : ''; + $function = isset($frame['function']) ? $frame['function'] : 'unknown'; + $call = $class ? $class . '::' . $function : $function; + $lines[] = 'at ' . $call . '() in ' . $file . ':' . $line; + } else { + $lines[] = $frame; + } + } + return implode("\n", $lines); + } + + return $details; } } diff --git a/test-results.xml b/test-results.xml index c5630f9..1307d5a 100644 --- a/test-results.xml +++ b/test-results.xml @@ -1,143 +1,249 @@ - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + - - + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - - + - - + + - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - - - + - - - + + + - - - - - - - - + + + + + + - - - - - - - - + + - - - - - - + + - - + + + + - - - - - - - - - - - - - - - - - - - - - - + + + + + + - - - - + + - +