test(verify): compare the charts the two paths drew

build.sbt

@@ -228,8 +228,8 @@
     Test / forkOptions := (Test / forkOptions).value
       .withWorkingDirectory((ThisBuild / baseDirectory).value),
     Test / testGrouping := (Test / definedTests).value.map { suite =>
       Tests.Group(suite.name, Seq(suite), Tests.SubProcess((Test / forkOptions).value))
     }
   )
 
 lazy val WorkflowOperator = (project in file("common/workflow-operator")).settings(commonModuleSettingsWithVendored).dependsOn(WorkflowCore)
@@ -239,6 +239,7 @@
   .dependsOn(WorkflowOperator)
 lazy val WorkflowCompilingService = (project in file("workflow-compiling-service"))
   .dependsOn(WorkflowCompiler, Auth, Config, Resource)
+  .dependsOn(WorkflowOperator % "test->test") // reuse PythonWorkerPool in verify tests
   .settings(commonModuleSettings)
   .settings(
     dependencyOverrides ++= Seq(

workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationHtmlComparator.scala

@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.texera.amber.translator.verify
+
+import org.apache.texera.amber.util.JSONUtils.objectMapper
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.{Files, Path}
+import scala.jdk.CollectionConverters._
+import scala.util.matching.Regex
+
+object VisualizationHtmlComparator {
+
+  /** A pandas Styler namespaces its CSS with a uuid drawn per Styler instance, so
+    * the same table rendered twice differs in every `id=` and every selector even
+    * though the markup is identical. The uuid carries no information about the
+    * table, it only keeps two tables on one page from colliding, so it is
+    * normalized away before comparing.
+    *
+    * It is normalized only where the markup uses it, never in what the table
+    * says. A Styler writes it in the `#` selectors inside its `<style>` element
+    * and in the `id` attribute of the table and its cells, so the rewrite visits
+    * those two places and leaves text content alone. A cell reading `#T_dead` is
+    * the table's own value and still has to match the other side, and the
+    * `_row0_col0` suffix stays, so a real difference still fails.
+    */
+  private val StyleElement = "(?s)<style\\b.*?</style>".r
+  private val Tag = "<[^>]*>".r
+  private val SelectorUuid = "(?<=#)T_[0-9a-f]+".r
+  private val IdUuid = """(?<=id=")T_[0-9a-f]+""".r
+
+  /** The standalone script writes its page with Python's text mode, which on Windows
+    * turns every newline into CRLF, while the runtime path carries the same markup
+    * through JSONL untouched and so keeps LF. The line ending is the platform writing
+    * the file rather than anything the operator chose, so it is normalized away too.
+    */
+  private val LineEnding = "\r\n|\r".r
+
+  private def normalize(html: String): String = {
+    val page = LineEnding.replaceAllIn(html, "\n")
+    val styled = StyleElement.replaceAllIn(page, within(SelectorUuid))
+    Tag.replaceAllIn(styled, within(IdUuid))
+  }
+
+  /** The same page with `uuid` normalized inside the region that matched. */
+  private def within(uuid: Regex): Regex.Match => String =
+    region => Regex.quoteReplacement(uuid.replaceAllIn(region.matched, "T_uuid"))
+
+  /** Every chart the runtime path drew, against every chart the standalone one
+    * wrote.
+    *
+    * A visualization operator emits a row per chart, and the exported script
+    * writes one page, so the sides are compared as sequences rather than by
+    * reading a row off each: a run whose second chart the other path never drew
+    * is a disagreement, and looking only at the first row would call it a match.
+    */
+  def assertEqual(actualVisualizationJsonl: Path, expectedHtmlFile: Path): Unit = {
+    val actual = readActualHtml(actualVisualizationJsonl)
+    val expected =
+      Seq(new String(Files.readAllBytes(expectedHtmlFile), StandardCharsets.UTF_8))
+
+    if (actual.map(normalize) != expected.map(normalize)) {
+      throw new VisualizationHtmlMismatchException(
+        actual = actualVisualizationJsonl,
+        expected = expectedHtmlFile,
+        actualHtml = actual,
+        expectedHtml = expected
+      )
+    }
+  }
+
+  private def readActualHtml(path: Path): Seq[String] = {
+    val lines =
+      Files.readAllLines(path, StandardCharsets.UTF_8).asScala.filter(_.trim.nonEmpty).toSeq
+    if (lines.isEmpty) throw new AssertionError(s"$path is empty")
+
+    lines.zipWithIndex.map {
+      case (line, row) =>
+        val htmlNode = objectMapper.readTree(line).get("html-content")
+        if (htmlNode == null || htmlNode.isNull) {
+          throw new AssertionError(s"$path row ${row + 1} has no html-content field")
+        }
+        htmlNode.asText()
+    }
+  }
+}
+
+final class VisualizationHtmlMismatchException(
+    val actual: Path,
+    val expected: Path,
+    val actualHtml: Seq[String],
+    val expectedHtml: Seq[String]
+) extends RuntimeException(
+      s"""Visualization HTML mismatch:
+         |  actual:   $actual (${actualHtml.size} drawn)
+         |  expected: $expected (${expectedHtml.size} drawn)
+         |--- actual html ---
+         |${actualHtml.mkString("\n--- and ---\n")}
+         |--- expected html ---
+         |${expectedHtml.mkString("\n--- and ---\n")}""".stripMargin
+    )

workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationHtmlComparatorSpec.scala

@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.texera.amber.translator.verify
+
+import org.apache.texera.amber.util.JSONUtils.objectMapper
+import org.scalatest.flatspec.AnyFlatSpec
+import org.scalatest.matchers.should.Matchers
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.{Files, Path}
+
+// Untagged: the comparator reads two files and compares strings, with no Python
+// to shell out to, so this belongs in the plain unit job.
+class VisualizationHtmlComparatorSpec extends AnyFlatSpec with Matchers {
+
+  private val dir: Path = Files.createTempDirectory("visualization-html-comparator-spec-")
+
+  /** The runtime path's side: a JSONL row per chart, each in `html-content`. */
+  private def writeActual(name: String, html: String*): Path = {
+    val p = dir.resolve(name)
+    val rows = html.map { page =>
+      val node = objectMapper.createObjectNode()
+      node.put("html-content", page)
+      objectMapper.writeValueAsString(node)
+    }
+    Files.write(p, rows.mkString("\n").getBytes(StandardCharsets.UTF_8))
+    p
+  }
+
+  /** The standalone path's side: the markup as the script wrote it to a file. */
+  private def writeExpected(name: String, html: String): Path = {
+    val p = dir.resolve(name)
+    Files.write(p, html.getBytes(StandardCharsets.UTF_8))
+    p
+  }
+
+  private val page = "<html>\n<body>\n<p>chart</p>\n</body>\n</html>\n"
+
+  "VisualizationHtmlComparator" should "accept two sides that differ only in line endings" in {
+    val actual = writeActual("crlf-actual.jsonl", page)
+    val expected = writeExpected("crlf-expected.html", page.replace("\n", "\r\n"))
+    noException should be thrownBy VisualizationHtmlComparator.assertEqual(actual, expected)
+  }
+
+  /** The two places a Styler writes its uuid: the CSS selector and the `id` attribute. */
+  private def styledCell(uuid: String, value: String) =
+    s"""<style>#${uuid}_row0_col0 { color: red; }</style>""" +
+      s"""<td id="${uuid}_row0_col0">$value</td>"""
+
+  it should "accept two sides that differ only in a Styler uuid" in {
+    val actual = writeActual("styler-actual.jsonl", styledCell("T_a1b2c3", "1"))
+    val expected = writeExpected("styler-expected.html", styledCell("T_9f8e7d", "1"))
+    noException should be thrownBy VisualizationHtmlComparator.assertEqual(actual, expected)
+  }
+
+  // A Styler does not escape a cell, so a value can read exactly like a uuid,
+  // `#` and all. What the table says is compared wherever it lands.
+  Seq("T_dead" -> "T_beef", "#T_dead" -> "#T_beef").foreach {
+    case (oneValue, otherValue) =>
+      it should s"still reject two sides whose cell reads $oneValue against $otherValue" in {
+        val name = oneValue.filter(_.isLetterOrDigit)
+        val actual = writeActual(s"cell-$name-actual.jsonl", styledCell("T_a1b2c3", oneValue))
+        val expected =
+          writeExpected(s"cell-$name-expected.html", styledCell("T_a1b2c3", otherValue))
+        a[VisualizationHtmlMismatchException] should be thrownBy VisualizationHtmlComparator
+          .assertEqual(actual, expected)
+      }
+  }
+
+  // The runtime path draws a chart per row it emits, and the exported script
+  // writes one page. A run that drew a second chart the other path never drew is
+  // a disagreement, and it is the one a comparison that reads the first row and
+  // stops would call a match.
+  it should "reject a run that drew a chart the other path did not" in {
+    val actual = writeActual("extra-actual.jsonl", page, page.replace("chart", "second chart"))
+    val expected = writeExpected("extra-expected.html", page)
+    val thrown = the[VisualizationHtmlMismatchException] thrownBy VisualizationHtmlComparator
+      .assertEqual(actual, expected)
+    thrown.actualHtml should have length 2
+    thrown.expectedHtml should have length 1
+    // The rows are named in the message, so the one nobody compared is readable
+    // from the failure rather than only from the file.
+    thrown.getMessage should include("second chart")
+  }
+
+  it should "still reject markup that differs in more than its line endings" in {
+    val actual = writeActual("differ-actual.jsonl", page)
+    val expected =
+      writeExpected("differ-expected.html", page.replace("chart", "table").replace("\n", "\r\n"))
+    a[VisualizationHtmlMismatchException] should be thrownBy VisualizationHtmlComparator
+      .assertEqual(actual, expected)
+  }
+}

workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/VisualizationJsonComparator.scala

@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.texera.amber.translator.verify
+
+import com.typesafe.scalalogging.LazyLogging
+import org.apache.texera.amber.util.JSONUtils.objectMapper
+import org.apache.texera.amber.util.python.PythonWorkerPool
+
+import java.nio.file.{Files, Path, StandardCopyOption}
+import scala.collection.mutable.ArrayBuffer
+import scala.sys.process._
+
+/**
+  * Compares the Plotly figure the two paths render, via `compare.py`'s
+  * `--plotly` mode.
+  *
+  * Shares that script — and therefore [[Comparator]]'s pool — rather than
+  * carrying one of its own: a worker is bound to the script it was launched
+  * with, so a separate script would mean a separate pool of interpreters for
+  * what is the same job, comparing one operator's two outputs. One comparison
+  * pool serves both output shapes.
+  */
+object VisualizationJsonComparator extends LazyLogging {
+
+  private val ScriptResourcePath = "/python/compare.py"
+
+  def assertEqual(
+      actualVisualizationJsonl: Path,
+      expectedPlotlyJson: Path,
+      pythonExe: String = resolvePython()
+  ): Unit = {
+    val (exit, stdout, stderr) =
+      compare(actualVisualizationJsonl, expectedPlotlyJson, pythonExe)
+    if (exit != 0) {
+      throw new VisualizationJsonMismatchException(
+        actual = actualVisualizationJsonl,
+        expected = expectedPlotlyJson,
+        exitCode = exit,
+        stdout = stdout,
+        stderr = stderr
+      )
+    }
+  }
+
+  // Pooled worker first, one-shot CLI as the fallback and as the behavior
+  // selected by TEXERA_TEST_PYTHON_WORKER=0. Both run the same
+  // `_run_plotly_comparison`, so results are identical.
+  private def compare(actual: Path, expected: Path, pythonExe: String): (Int, String, String) = {
+    if (PythonWorkerPool.enabled) {
+      try {
+        val req = objectMapper.createObjectNode()
+        req.put("kind", "plotly")
+        req.put("actual", actual.toString)
+        req.put("expected", expected.toString)
+        val o = PythonWorkerPool.run(ScriptResourcePath, Seq("--serve"), pythonExe, req)
+        return (o.exit, o.stdout, o.stderr)
+      } catch {
+        case e: PythonWorkerPool.WorkerDiedException =>
+          logger.warn(
+            s"Comparator worker unavailable; falling back to one-shot CLI: ${e.getMessage}"
+          )
+      }
+    }
+    runCli(actual, expected, pythonExe)
+  }
+
+  private def runCli(actual: Path, expected: Path, pythonExe: String): (Int, String, String) = {
+    val scriptPath = extractScript()
+    val outBuf = ArrayBuffer.empty[String]
+    val errBuf = ArrayBuffer.empty[String]
+    val processLogger = ProcessLogger(line => outBuf += line, line => errBuf += line)
+
+    val exit = Process(
+      Seq(pythonExe, scriptPath.toString, "--plotly", actual.toString, expected.toString)
+    ).!(processLogger)
+    (exit, outBuf.mkString("\n"), errBuf.mkString("\n"))
+  }
+
+  private def extractScript(): Path = {
+    val stream = getClass.getResourceAsStream(ScriptResourcePath)
+    require(stream != null, s"compare.py not found at $ScriptResourcePath")
+    try {
+      val tmp = Files.createTempFile("compare-", ".py")
+      Files.copy(stream, tmp, StandardCopyOption.REPLACE_EXISTING)
+      tmp.toFile.deleteOnExit()
+      tmp
+    } finally stream.close()
+  }
+
+  private def resolvePython(): String =
+    sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty).getOrElse("python3.12")
+}
+
+final class VisualizationJsonMismatchException(
+    val actual: Path,
+    val expected: Path,
+    val exitCode: Int,
+    val stdout: String,
+    val stderr: String
+) extends RuntimeException(
+      s"""Visualization JSON mismatch (compare.py --plotly exit $exitCode):
+         |  actual:   $actual
+         |  expected: $expected
+         |--- stderr ---
+         |$stderr""".stripMargin
+    )