diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDesc.scala index 3f6fe276724..dfb7c2c1d51 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDesc.scala @@ -22,16 +22,19 @@ package org.apache.texera.amber.operator.visualization.ImageViz import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity -import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder import javax.validation.constraints.NotNull -class ImageVisualizerOpDesc extends PythonOperatorDescriptor { +class ImageVisualizerOpDesc extends PythonOperatorDescriptor with StandaloneCodeGenerator { @JsonProperty(required = true) @JsonSchemaTitle("image content column") @@ -98,4 +101,58 @@ class ImageVisualizerOpDesc extends PythonOperatorDescriptor { finalCode.encode } + // Output is an HTML visualization, not a tabular DataFrame. + // The translator skips it in the leaf-DataFrame print block. + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = generateStandaloneCode(Map.empty) + + /** The executor hands `b64encode` whatever the column holds, so a column of + * text ends in the reason page: `b64encode` refuses a `str`. The exported + * script reads the same column out of JSONL, where a BINARY column arrives as + * the base64 text of its bytes, which is already what the tag wants. + * + * So the text is read as an image only where the column was declared BINARY. + * A column of any other type falls through to the same `b64encode` call and + * fails there, which is the page the run showed. + */ + override def generateStandaloneCode(inputSchemas: Map[PortIdentity, Schema]): String = { + val carriesBytes = inputSchemas + .get(operatorInfo.inputPorts.head.id) + .flatMap(_.getAttributes.find(_.getName == binaryContent)) + .forall(_.getType == AttributeType.BINARY) + val textIsTheImage = + if (carriesBytes) "isinstance(binary_image_data, str)" else "False" + s"""import base64 + | + |LT = chr(60) + |GT = chr(62) + | + |def encode_image_to_html(binary_image_data): + | try: + | if $textIsTheImage: + | encoded_image_str = binary_image_data + | else: + | encoded_image_data = base64.b64encode(binary_image_data) + | encoded_image_str = encoded_image_data.decode("utf-8") + | return ( + | f'{LT}img src="data:image;base64,{encoded_image_str}" alt="Image" ' + | f'style="max-width: 100vw; max-height: 90vh; width: auto; height: auto;"{GT}' + | ) + | except Exception: + | return ( + | f'{LT}h1{GT}Image is not available.{LT}/h1{GT}' + | f'{LT}p{GT}Reason: Binary input is not valid{LT}/p{GT}' + | ) + | + |all_images_html = f"{LT}div{GT}" + "".join( + | encode_image_to_html(binary_image_data) + | for binary_image_data in in1df[${pyStringLiteral(binaryContent)}] + |) + f"{LT}/div{GT}" + | + |with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(all_images_html) + |print("Image visualizer saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDesc.scala index f9aa6bcd2b4..4c94d322f78 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDesc.scala @@ -22,15 +22,18 @@ package org.apache.texera.amber.operator.visualization.figureFactoryTable import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity -import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder import javax.validation.constraints.{DecimalMin, NotEmpty} -class FigureFactoryTableOpDesc extends PythonOperatorDescriptor { +class FigureFactoryTableOpDesc extends PythonOperatorDescriptor with StandaloneCodeGenerator { @JsonProperty(required = false) @JsonSchemaTitle("Font Size") @@ -44,7 +47,8 @@ class FigureFactoryTableOpDesc extends PythonOperatorDescriptor { @JsonPropertyDescription("Font color of the Figure Factory Table") @JsonSchemaInject(json = """ { - "pattern": "^\\s*$|^\\s*#(?:\\s*[0-9a-fA-F]){3}(?:(?:\\s*[0-9a-fA-F]){3})?\\s*$|^\\s*(?:[rR]\\s*[gG]\\s*[bB]|[hH]\\s*[sS]\\s*[lL]|[hH]\\s*[sS]\\s*[vV])(?:\\s*[aA])?\\s*\\(\\s*(?:\\s*[0-9.])+(?:\\s*%)?(?:\\s*,(?:\\s*[0-9.])+(?:\\s*%)?){2,3}\\s*\\)\\s*$|^\\s*[vV]\\s*[aA]\\s*[rR]\\s*\\(\\s*-\\s*-[^)]*\\)\\s*$|^\\s*[a-zA-Z][a-zA-Z\\s]*$" + "pattern": "^\\s*$|^\\s*#(?:\\s*[0-9a-fA-F]){3}(?:(?:\\s*[0-9a-fA-F]){3})?\\s*$|^\\s*(?:[rR]\\s*[gG]\\s*[bB]|[hH]\\s*[sS]\\s*[lL]|[hH]\\s*[sS]\\s*[vV])(?:\\s*[aA])?\\s*\\(\\s*(?:\\s*[0-9.])+(?:\\s*%)?(?:\\s*,(?:\\s*[0-9.])+(?:\\s*%)?){2,3}\\s*\\)\\s*$|^\\s*[vV]\\s*[aA]\\s*[rR]\\s*\\(\\s*-\\s*-[^)]*\\)\\s*$|^\\s*[a-zA-Z][a-zA-Z\\s]*$", + "examples": ["red"] } """) var fontColor: EncodableString = "#000000" @@ -147,4 +151,39 @@ class FigureFactoryTableOpDesc extends PythonOperatorDescriptor { .add("html-content", AttributeType.STRING) Map(operatorInfo.outputPorts.head.id -> outputSchema) } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val attributes = columns.map(c => pyStringLiteral(c.attributeName)).mkString(", ") + s"""import plotly.figure_factory as ff + | + |def render_error(error_msg): + | return "
Reason is: {}
".format( + | error_msg + | ) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |else: + | table = in1df.dropna(subset=[$attributes]) + | if table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("value column contains only non-positive numbers or nulls.")) + | else: + | filtered_table = table[[$attributes]] + | headers = filtered_table.columns.tolist() + | cell_values = [filtered_table[col].tolist() for col in headers] + | data = [headers] + list(map(list, zip(*cell_values))) + | fig = ff.create_table(data, height_constant=$rowHeight, font_colors=[${pyStringLiteral( + fontColor + )}]) + | for i in range(len(fig.layout.annotations)): + | fig.layout.annotations[i].font.size = $fontSize + | fig.update_layout(margin=dict(l=0, r=0, b=0, t=0)) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Figure factory table saved to " + outputHtml)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/htmlviz/HtmlVizOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/htmlviz/HtmlVizOpDesc.scala index 4775a123d5a..4735fbab53b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/htmlviz/HtmlVizOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/htmlviz/HtmlVizOpDesc.scala @@ -25,9 +25,10 @@ import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper import javax.validation.constraints.NotNull @@ -36,10 +37,11 @@ import javax.validation.constraints.NotNull * HTML Visualization operator to render any given HTML code * This is the description of the operator */ -class HtmlVizOpDesc extends LogicalOp { +class HtmlVizOpDesc extends LogicalOp with StandaloneCodeGenerator { @JsonProperty(required = true) @JsonSchemaTitle("HTML content") @AutofillAttributeName + @SampleColumn("short_text") @NotNull(message = "HTML content cannot be empty") var htmlContentAttrName: String = "" @@ -74,4 +76,10 @@ class HtmlVizOpDesc extends LogicalOp { OperatorGroupConstants.VISUALIZATION_MEDIA_GROUP ) + // Mirrors HtmlVizOpExec: emit one row per input row whose single + // "html-content" column is the value of htmlContentAttrName, passed through + // unconverted (the exec does not coerce either). + override def generateStandaloneCode(): String = + s"""out1df = pd.DataFrame({"html-content": in1df[${pyStringLiteral(htmlContentAttrName)}]})""" + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/nestedTable/NestedTableOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/nestedTable/NestedTableOpDesc.scala index b32cd9740ac..7da846c1bfc 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/nestedTable/NestedTableOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/nestedTable/NestedTableOpDesc.scala @@ -22,15 +22,16 @@ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext import org.apache.texera.amber.core.workflow.PortIdentity -import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import java.util import javax.validation.constraints.NotEmpty import scala.jdk.CollectionConverters.ListHasAsScala -class NestedTableOpDesc extends PythonOperatorDescriptor { +class NestedTableOpDesc extends PythonOperatorDescriptor with StandaloneCodeGenerator { @JsonPropertyDescription( "List of columns to include in the nested table chart and their subgroup" @@ -137,4 +138,80 @@ class NestedTableOpDesc extends PythonOperatorDescriptor { |""" finalcode.encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val sortedColumns = includedColumns.asScala.sortBy(_.attributeGroup) + + val multiIndexTuples = sortedColumns + .map { config => + val name = + if (config.newName != null && config.newName.nonEmpty) config.newName + else config.originalName + s"""(${pyStringLiteral(config.attributeGroup)}, ${pyStringLiteral(name)})""" + } + .mkString(",\n ") + + val rowValues = sortedColumns + .map(config => s"""row[${pyStringLiteral(config.originalName)}]""") + .mkString(", ") + + s"""import pandas as pd + |import numpy as np + | + |def render_error(error_msg): + | return '''Reason is: {}
+ | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |else: + | columns = pd.MultiIndex.from_tuples([ + | $multiIndexTuples + | ]) + | + | data = [] + | for _, row in in1df.iterrows(): + | data.append([ + | $rowValues + | ]) + | + | df = pd.DataFrame(data, columns=columns) + | + | styles = [ + | {'selector': 'th', 'props': [('background-color', '#f2f2f2'), + | ('color', 'black'), + | ('font-weight', 'bold'), + | ('border', '1px solid #ddd'), + | ('padding', '8px'), + | ('text-align', 'center')]}, + | {'selector': 'td', 'props': [('border', '1px solid #ddd'), + | ('padding', '8px'), + | ('text-align', 'center')]}, + | {'selector': 'caption', 'props': [('caption-side', 'top'), + | ('font-size', '16pt'), + | ('font-weight', 'bold'), + | ('text-align', 'left'), + | ('padding', '10px')]}, + | {'selector': '.row_heading', 'props': [('text-align', 'left'), + | ('font-weight', 'normal')]}, + | {'selector': '.blank.level0', 'props': [('display', 'none')]} + | ] + | + | styled_table = ( + | df.style + | .set_table_styles(styles) + | .format(precision=2, na_rep="") + | .set_table_attributes('class="dataframe"') + | .hide(axis="index") + | ) + | + | html = styled_table.to_html() + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(html) + | print("Nested table saved to " + outputHtml)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/parallelCoordinatesPlot/ParallelCoordinatesPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/parallelCoordinatesPlot/ParallelCoordinatesPlotOpDesc.scala index 4af91c332d9..e67217ae621 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/parallelCoordinatesPlot/ParallelCoordinatesPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/parallelCoordinatesPlot/ParallelCoordinatesPlotOpDesc.scala @@ -24,6 +24,7 @@ import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchema import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.{ AutofillAttributeName, AutofillAttributeNameList @@ -31,7 +32,10 @@ import org.apache.texera.amber.operator.metadata.annotations.{ import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.pybuilder.PythonTemplateBuilder -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import javax.validation.constraints.{NotNull, Size} @@ -50,7 +54,7 @@ import javax.validation.constraints.{NotNull, Size} } } """) -class ParallelCoordinatesPlotOpDesc extends PythonOperatorDescriptor { +class ParallelCoordinatesPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "dimensions", required = true) @JsonSchemaTitle("Dimensions") @@ -134,4 +138,38 @@ class ParallelCoordinatesPlotOpDesc extends PythonOperatorDescriptor { |""" finalcode.encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val dimCols = dimensions.map(pyStringLiteral).mkString("[", ", ", "]") + val colorLit = pyStringLiteral(color) + val colorFilter = + if (color != null && color.nonEmpty) s""" & (in1df[$colorLit].notnull())""" else "" + val colorArg = + if (color != null && color.nonEmpty) s""", color=$colorLit""" else "" + + s"""def render_error(error_msg): + | return '''Reason is: {}
+ | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Input table is empty.")) + |else: + | table = in1df[in1df[$dimCols].notnull().all(axis=1)$colorFilter].copy() + | if table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("No valid rows after filtering.")) + | else: + | fig = px.parallel_coordinates( + | table, + | dimensions=$dimCols$colorArg + | ) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Parallel coordinates plot saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/rangeSlider/RangeSliderOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/rangeSlider/RangeSliderOpDesc.scala index 6b1425708d5..05ea9f91d2e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/rangeSlider/RangeSliderOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/rangeSlider/RangeSliderOpDesc.scala @@ -26,9 +26,11 @@ import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBui import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import javax.validation.constraints.NotNull @@ -43,7 +45,7 @@ import javax.validation.constraints.NotNull } } """) -class RangeSliderOpDesc extends PythonOperatorDescriptor { +class RangeSliderOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "Y-axis", required = true) @JsonSchemaTitle("Y-axis") @JsonPropertyDescription("The name of the column to represent y-axis") @@ -143,4 +145,45 @@ class RangeSliderOpDesc extends PythonOperatorDescriptor { finalcode.encode } + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val functionType = duplicateType.getFunctionType + val xAxisLit = pyStringLiteral(xAxis) + val yAxisLit = pyStringLiteral(yAxis) + s"""def render_error(error_msg): + | return '''Reason is: {}
+ | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |elif $yAxisLit.strip() == "" or $xAxisLit.strip() == "": + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Y-axis or X-axis is empty")) + |else: + | table = in1df + | table = table.dropna(subset=[$xAxisLit, $yAxisLit]) + | functionType = ${pyStringLiteral(functionType)} + | if functionType.lower() == "mean": + | table = table.groupby($xAxisLit)[$yAxisLit].mean().reset_index() + | elif functionType.lower() == "sum": + | table = table.groupby($xAxisLit)[$yAxisLit].sum().reset_index() + | fig = go.Figure() + | fig.add_trace(go.Scatter(x=table[$xAxisLit], y=table[$yAxisLit], mode="markers+lines")) + | fig.update_layout( + | xaxis_title=$xAxisLit, + | yaxis_title=$yAxisLit, + | xaxis=dict( + | rangeslider=dict( + | visible=True + | ) + | ) + | ) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Range slider saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDesc.scala index 7290c057dd4..3c5a18d04a1 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDesc.scala @@ -24,11 +24,13 @@ import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import javax.validation.constraints.NotEmpty -class TablesPlotOpDesc extends PythonOperatorDescriptor { +class TablesPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonPropertyDescription("List of columns to include in the table chart") @JsonProperty(value = "add attribute", required = true) @@ -114,4 +116,40 @@ class TablesPlotOpDesc extends PythonOperatorDescriptor { .add("html-content", AttributeType.STRING) Map(operatorInfo.outputPorts.head.id -> outputSchema) } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + assert(includedColumns.nonEmpty) + // Mirror getAttributes: a Python list literal of the selected column names. + val columnsList = + includedColumns.map(c => pyStringLiteral(c.attributeName)).mkString("[", ", ", "]") + // The two guards mirror generatePythonCode's, which reports both conditions + // rather than rendering a table with no rows in it. + s"""def render_error(error_msg): + | return "Reason is: {}
".format(error_msg) + | + |attributes = $columnsList + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |else: + | table = in1df.dropna(subset=attributes) + | if table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("value column contains only non-positive numbers or nulls.")) + | else: + | filtered_table = table[attributes] + | headers = filtered_table.columns.tolist() + | cell_values = [filtered_table[col].tolist() for col in headers] + | + | fig = go.Figure(data=[go.Table( + | header=dict(values=headers), + | cells=dict(values=cell_values) + | )]) + | fig.update_layout(margin=dict(l=0, r=0, b=0, t=0)) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Tables plot saved to " + outputJson)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDesc.scala index 3add49a9e1a..4409adbcd8e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDesc.scala @@ -25,9 +25,10 @@ import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} -import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper import javax.validation.constraints.NotNull @@ -45,7 +46,7 @@ import javax.validation.constraints.NotNull } } """) -class UrlVizOpDesc extends LogicalOp { +class UrlVizOpDesc extends LogicalOp with StandaloneCodeGenerator { @JsonProperty(required = true) @JsonSchemaTitle("URL content") @@ -84,4 +85,34 @@ class UrlVizOpDesc extends LogicalOp { OperatorGroupConstants.VISUALIZATION_MEDIA_GROUP ) + // Mirrors UrlVizOpExec: wrap each urlContentAttrName value in the exact same + // iframe HTML document and emit it as the "html-content" column. + override def generateStandaloneCode(): String = { + val urlLit = pyStringLiteral(urlContentAttrName) + s"""import html + | + |def _texera_urlviz_iframe(u): + | # "null", not Python's "None": the operator interpolates the field into a + | # Scala string, and the JVM renders a null that way. + | u = "null" if pd.isna(u) else str(u) + | # The cell lands where an attribute is expected, so a quote in it would + | # close src= and leave the rest of the cell standing as attributes of the + | # iframe. escapeAttribute on the executor writes the same five characters + | # in the same order, so the two paths write the page the same way. + | u = html.escape(u, quote=True) + | return ( + | '\\n' + | '\\n' + | '\\n' + | 'Reason is: {}
+ | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |else: + | table = in1df + | x_values = list(table[${pyStringLiteral(xColumn)}]) + | y_values = list(table[${pyStringLiteral(yColumn)}]) + | total = sum(y_values) + | + | fig = go.Figure(go.Waterfall( + | name="Waterfall", orientation="v", + | measure=["relative"] * len(y_values) + ["total"], + | x=x_values + ["Total"], + | y=y_values + [0], + | textposition="outside", + | text=[f"{v:+}" for v in y_values] + [f"{total}"], + | connector={"line": {"color": "rgb(63, 63, 63)"}} + | )) + | + | fig.update_layout(showlegend=True, waterfallgap=0.3, xaxis_type="category") + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Waterfall chart saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDescSpec.scala index 2283610cd36..e6493d76473 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDescSpec.scala @@ -19,12 +19,18 @@ package org.apache.texera.amber.operator.visualization.ImageViz -import org.apache.texera.amber.core.tuple.AttributeType +import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.scalatest.BeforeAndAfter import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.io.Source +import scala.util.Try + class ImageVisualizerOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { var opDesc: ImageVisualizerOpDesc = _ before { @@ -103,4 +109,85 @@ class ImageVisualizerOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Mat code should include("encode_image_to_html") code should include("decode_python_template") } + + "ImageVisualizerOpDesc.generateStandaloneCode" should "avoid literal HTML tags that the UI code viewer can render away" in { + opDesc.binaryContent = "image_bytes" + val code = opDesc.generateStandaloneCode() + + code should include("LT = chr(60)") + code should include("GT = chr(62)") + code should include("encoded_image_str") + code should not include "Reason is: {error_msg}
"""" + // Stubs only the pytexera import seam, then hands the operator a table that arrives + // empty, one whose every row has a hole in a chosen column, and one that draws. + // Both empty branches call self.render_error, which once was not defined. + private val runtimeDriverScript: String = + """import base64 + |import sys + |import types + |from typing import Iterator, Optional + | + |import pandas as pd + | + |class UDFTableOperator: + | def decode_python_template(self, data): + | return base64.b64decode(data).decode("utf-8") + | + |stub = types.ModuleType("pytexera") + |stub.UDFTableOperator = UDFTableOperator + |stub.overrides = lambda fn: fn + |stub.Table = pd.DataFrame + |stub.TableLike = object + |stub.Iterator = Iterator + |stub.Optional = Optional + |sys.modules["pytexera"] = stub + | + |ns = {"__name__": "generated_table_chart"} + |with open(sys.argv[1]) as f: + | exec(compile(f.read(), sys.argv[1], "exec"), ns) + |op = ns["TableChartOperator"]() + | + |cases = [ + | ("empty", pd.DataFrame({"col_one": [], "col_two": []})), + | ("every_row_holed", pd.DataFrame({"col_one": [1.0, None], "col_two": [None, "b"]})), + | ("filled", pd.DataFrame({"col_one": [1, 2], "col_two": ["a", "b"]})), + |] + |for cid, df in cases: + | page = list(op.process_table(df, 0))[0]["html-content"] + | print("CASE %s %s" % (cid, page if "not available" in page else "CHART")) + |""".stripMargin + + it should "render its message when the table is empty or every row has a hole" taggedAs NeedsPythonPackages in { + val python = resolvePythonExecutable().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) + if (!canImportPandasAndPlotly(python)) cancel(s"'$python' cannot import pandas and plotly") + + withColumns() + val moduleFile = Files.createTempFile("figure_factory_table_op_", ".py") + val driverFile = Files.createTempFile("figure_factory_table_driver_", ".py") + try { + Files.write(moduleFile, opDesc.generatePythonCode().getBytes(StandardCharsets.UTF_8)) + Files.write(driverFile, runtimeDriverScript.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, driverFile.toString, moduleFile.toString) + .redirectErrorStream(true) + .start() + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly() + fail("Runtime driver timed out after 120s") + } + val output = new String(process.getInputStream.readAllBytes(), StandardCharsets.UTF_8) + val page = "Reason is: " + withClue(s"Driver output:\n$output\n") { + process.exitValue() shouldBe 0 + output should include(s"CASE empty ${page}input table is empty.
") + output should include( + s"CASE every_row_holed ${page}value column contains only non-positive numbers or nulls." + ) + output should include("CASE filled CHART") + } + } finally { + Try(Files.deleteIfExists(moduleFile)) + Try(Files.deleteIfExists(driverFile)) + () + } + } + + // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. + private def resolvePythonExecutable(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def isRunnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) } + + private def canImportPandasAndPlotly(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() + ).toOption.exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDescSpec.scala index c088cacc672..1b96687f60a 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDescSpec.scala @@ -19,15 +19,23 @@ package org.apache.texera.amber.operator.visualization.tablesChart +import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.operator.tags.IntegrationTest import org.scalatest.BeforeAndAfter +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import java.nio.charset.StandardCharsets +import java.nio.file.Files import java.util.Base64 +import java.util.concurrent.TimeUnit +import scala.util.Try class TablesPlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + var opDesc: TablesPlotOpDesc = _ before { @@ -89,14 +97,105 @@ class TablesPlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers code should not include "')','" } - it should "define the render_error the empty-table branches call" in { - // Both empty-table branches call self.render_error; without the definition they - // raised AttributeError instead of rendering the message. - opDesc.includedColumns = List(column("col_one"), column("col_two")) - val code = opDesc.generatePythonCode() - code should include("def render_error(self, error_msg) -> str:") - code should include( - """return f"Reason is: {error_msg}
"""" + // Stubs only the pytexera import seam, then hands the operator a table that arrives + // empty, one whose every row has a hole in a chosen column, and one that draws. + // Both empty branches call self.render_error, which once was not defined. + private val runtimeDriverScript: String = + """import base64 + |import sys + |import types + |from typing import Iterator, Optional + | + |import pandas as pd + | + |class UDFTableOperator: + | def decode_python_template(self, data): + | return base64.b64decode(data).decode("utf-8") + | + |stub = types.ModuleType("pytexera") + |stub.UDFTableOperator = UDFTableOperator + |stub.overrides = lambda fn: fn + |stub.Table = pd.DataFrame + |stub.TableLike = object + |stub.Iterator = Iterator + |stub.Optional = Optional + |sys.modules["pytexera"] = stub + | + |ns = {"__name__": "generated_table_chart"} + |with open(sys.argv[1]) as f: + | exec(compile(f.read(), sys.argv[1], "exec"), ns) + |op = ns["TableChartOperator"]() + | + |cases = [ + | ("empty", pd.DataFrame({"col_one": [], "col_two": []})), + | ("every_row_holed", pd.DataFrame({"col_one": [1.0, None], "col_two": [None, "b"]})), + | ("filled", pd.DataFrame({"col_one": [1, 2], "col_two": ["a", "b"]})), + |] + |for cid, df in cases: + | page = list(op.process_table(df, 0))[0]["html-content"] + | print("CASE %s %s" % (cid, page if "not available" in page else "CHART")) + |""".stripMargin + + it should "render its message when the table is empty or every row has a hole" taggedAs NeedsPythonPackages in { + val python = resolvePythonExecutable().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) + if (!canImportPandasAndPlotly(python)) cancel(s"'$python' cannot import pandas and plotly") + + opDesc.includedColumns = List(column("col_one"), column("col_two")) + val moduleFile = Files.createTempFile("tables_plot_op_", ".py") + val driverFile = Files.createTempFile("tables_plot_driver_", ".py") + try { + Files.write(moduleFile, opDesc.generatePythonCode().getBytes(StandardCharsets.UTF_8)) + Files.write(driverFile, runtimeDriverScript.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, driverFile.toString, moduleFile.toString) + .redirectErrorStream(true) + .start() + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly() + fail("Runtime driver timed out after 120s") + } + val output = new String(process.getInputStream.readAllBytes(), StandardCharsets.UTF_8) + val page = "Reason is: " + withClue(s"Driver output:\n$output\n") { + process.exitValue() shouldBe 0 + output should include(s"CASE empty ${page}input table is empty.
") + output should include( + s"CASE every_row_holed ${page}value column contains only non-positive numbers or nulls." + ) + output should include("CASE filled CHART") + } + } finally { + Try(Files.deleteIfExists(moduleFile)) + Try(Files.deleteIfExists(driverFile)) + () + } + } + + // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. + private def resolvePythonExecutable(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def isRunnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) } + + private def canImportPandasAndPlotly(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() + ).toOption.exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDescSpec.scala index 2dbbb886d1a..f5aad7db0e8 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDescSpec.scala @@ -22,17 +22,28 @@ package org.apache.texera.amber.operator.visualization.urlviz import com.fasterxml.jackson.annotation.JsonProperty import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject import org.apache.texera.amber.core.executor.OpExecWithClassName -import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} +import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.tags.IntegrationTest +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit import javax.validation.constraints.NotNull +import scala.io.Source +import scala.util.Try class UrlVizOpDescSpec extends AnyFlatSpec with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -125,4 +136,94 @@ class UrlVizOpDescSpec extends AnyFlatSpec with Matchers { val b = new UrlVizOpDesc a.operatorIdentifier should not equal b.operatorIdentifier } + + // --------------------------------------------------------------------------- + // The cell lands where an attribute is expected + // --------------------------------------------------------------------------- + + // A quote in the cell used to close `src=` and leave the rest of it standing as + // attributes of the iframe, which is the page speaking for whoever wrote the + // row. Both paths are held to the same written page for that value. + private val hostileUrl = """https://example.invalid/a" onload="alert(1)""" + + /** A configured descriptor: `urlContentAttrName` is a val, so Jackson writes it. */ + private def descFor(urlAttr: String): UrlVizOpDesc = { + val node = objectMapper.createObjectNode() + node.put("operatorType", "URLVisualizer") + node.put("urlContentAttrName", urlAttr) + objectMapper.readValue(node.toString, classOf[UrlVizOpDesc]) + } + + it should "escape a hostile cell rather than letting it add an attribute" in { + val attr = new Attribute("url", AttributeType.STRING) + val tuple = Tuple.builder(Schema().add(attr)).add(attr, hostileUrl).build() + val page = new UrlVizOpExec(objectMapper.writeValueAsString(descFor("url"))) + .processTuple(tuple, port = 0) + .next() + .getFields + .head + .asInstanceOf[String] + + page should include("""src="https://example.invalid/a" onload="alert(1)"""") + page should not include """onload="alert""" + } + + it should "write that cell the same way in the exported script" taggedAs NeedsPythonPackages in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val desc = descFor("url") + val attr = new Attribute("url", AttributeType.STRING) + val tuple = Tuple.builder(Schema().add(attr)).add(attr, hostileUrl).build() + val fromExecutor = new UrlVizOpExec(objectMapper.writeValueAsString(desc)) + .processTuple(tuple, port = 0) + .next() + .getFields + .head + .asInstanceOf[String] + + val driver = + s"""import pandas as pd + | + |in1df = pd.DataFrame({"url": [${pyStringLiteral(hostileUrl)}]}) + |${desc.generateStandaloneCode()} + | + |print(out1df["html-content"].iloc[0]) + |""".stripMargin + + val script = Files.createTempFile("urlviz-hostile-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + out.trim shouldBe fromExecutor + } + } + + private def resolvePython(): Option[String] = { + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty).toList ++ List( + "python3", + "python", + "py" + )).distinct + .find(runnable) + } + + private def canImportPandas(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() + ).toOption.exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDescSpec.scala index c6a42ef8e39..904d9aad346 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDescSpec.scala @@ -19,15 +19,23 @@ package org.apache.texera.amber.operator.visualization.waterfallChart +import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.operator.tags.IntegrationTest import org.scalatest.BeforeAndAfter +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import java.nio.charset.StandardCharsets +import java.nio.file.Files import java.util.Base64 +import java.util.concurrent.TimeUnit +import scala.util.Try class WaterfallChartOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + var opDesc: WaterfallChartOpDesc = _ before { @@ -77,4 +85,156 @@ class WaterfallChartOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matc assert(carries(code, "y_col")) code should include("class ProcessTableOperator(UDFTableOperator)") } + + private def configured(): WaterfallChartOpDesc = { + opDesc.xColumn = "x_col" + opDesc.yColumn = "y_col" + opDesc + } + + it should "give every input row a relative bar and append the total as an extra bar" in { + val figure = configured().createPlotlyFigure().plain + figure should include("""measure=["relative"] * len(y_values) + ["total"]""") + figure should not include """(len(y_values) - 1)""" + figure should include("""x=x_values + ["Total"]""") + figure should include("y=y_values + [0]") + } + + it should "pin the x axis to categories so the bars keep their input order" in { + configured().createPlotlyFigure().plain should include("""xaxis_type="category"""") + } + + // Python executable resolution, following FilledAreaPlotOpDescSpec: + // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. + private def resolvePythonExecutable(): Option[String] = { + def fromConfig: Option[String] = { + val configOpt = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + configOpt + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + } + + def isRunnable(exe: String): Boolean = { + val pTry = Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()) + pTry.toOption.exists { p => + val finished = p.waitFor(5, TimeUnit.SECONDS) + if (!finished) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) + } + + private def canImportPandasAndPlotly(python: String): Boolean = { + val pTry = Try( + new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() + ) + pTry.toOption.exists { p => + val finished = p.waitFor(60, TimeUnit.SECONDS) + if (!finished) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + } + + // Driver executed by the runtime test below. It stubs only the pytexera import seam and + // swaps plotly's html renderer for its json one, so the trace the generated module hands + // plotly can be read back bar by bar. + private val runtimeDriverScript: String = + """import base64 + |import json + |import sys + |import types + |from typing import Iterator, Optional + | + |import pandas as pd + |import plotly.io + | + |plotly.io.to_html = lambda fig, **kwargs: fig.to_json() + | + |class UDFTableOperator: + | def decode_python_template(self, data): + | return base64.b64decode(data).decode("utf-8") + | + |stub = types.ModuleType("pytexera") + |stub.UDFTableOperator = UDFTableOperator + |stub.overrides = lambda fn: fn + |stub.Table = pd.DataFrame + |stub.TableLike = object + |stub.Iterator = Iterator + |stub.Optional = Optional + |sys.modules["pytexera"] = stub + | + |ns = {"__name__": "generated_waterfall_chart"} + |with open(sys.argv[1]) as f: + | exec(compile(f.read(), sys.argv[1], "exec"), ns) + |op = ns["ProcessTableOperator"]() + | + |cases = [ + | ("labels", pd.DataFrame({"x_col": ["a", "b", "c", "d"], "y_col": [1, 2, 3, 4]})), + | ("unsorted_numeric_x", pd.DataFrame({"x_col": [30, 10, 20], "y_col": [1, 2, 3]})), + |] + | + |for cid, df in cases: + | fig = json.loads(list(op.process_table(df, 0))[0]["html-content"]) + | trace = fig["data"][0] + | print("CASE %s measure=%s x=%s y=%s text=%s xaxis=%s" % ( + | cid, + | json.dumps(trace["measure"]), + | json.dumps(list(trace["x"])), + | json.dumps(list(trace["y"])), + | json.dumps(list(trace["text"])), + | fig["layout"]["xaxis"]["type"], + | )) + |""".stripMargin + + it should "plot the last row and draw the total over the whole column at runtime" taggedAs NeedsPythonPackages in { + val python = resolvePythonExecutable().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandasAndPlotly(python)) { + cancel(s"'$python' cannot import pandas and plotly; skipping runtime verification") + } + + val moduleFile = Files.createTempFile("waterfall_chart_op_", ".py") + val driverFile = Files.createTempFile("waterfall_chart_driver_", ".py") + try { + Files.write(moduleFile, configured().generatePythonCode().getBytes(StandardCharsets.UTF_8)) + Files.write(driverFile, runtimeDriverScript.getBytes(StandardCharsets.UTF_8)) + + val process = new ProcessBuilder(python, driverFile.toString, moduleFile.toString) + .redirectErrorStream(true) + .start() + val finished = process.waitFor(120, TimeUnit.SECONDS) + if (!finished) { + process.destroyForcibly() + fail("Runtime verification driver timed out after 120s") + } + val output = new String(process.getInputStream.readAllBytes(), StandardCharsets.UTF_8) + withClue(s"Driver output:\n$output\n") { + process.exitValue() shouldBe 0 + // Four rows of 1, 2, 3, 4 give four bars plus a total of 10. The old measure list + // spent the last row on the total, drawing three bars and a fourth at 6 labelled +4. + output should include( + """CASE labels measure=["relative", "relative", "relative", "relative", "total"] """ + + """x=["a", "b", "c", "d", "Total"] y=[1, 2, 3, 4, 0] """ + + """text=["+1", "+2", "+3", "+4", "10"] xaxis=category""" + ) + // An unsorted numeric x column keeps its input order, so the bar plotly accumulates + // into is the bar the running total was computed from. + output should include( + """CASE unsorted_numeric_x measure=["relative", "relative", "relative", "total"] """ + + """x=[30, 10, 20, "Total"] y=[1, 2, 3, 0] """ + + """text=["+1", "+2", "+3", "6"] xaxis=category""" + ) + } + } finally { + Try(Files.deleteIfExists(moduleFile)) + Try(Files.deleteIfExists(driverFile)) + () + } + } }