feat(observability): logging foundations (bootstrap, bridge, sanitizer)

.github/workflows/build.yml

@@ -329,7 +329,7 @@ jobs:
           BACKPORT_TARGET_BRANCH: ${{ inputs.backport_target_branch }}
         run: |
           # Backport builds filter by the checked-out build.sbt.
-          want=(DAO Auth Config Resource Util PyBuilder WorkflowCore
+          want=(DAO Auth Config Observability Resource Util PyBuilder WorkflowCore
                 WorkflowOperator WorkflowCompiler WorkflowExecutionService)
           tasks=()
           if [ -n "${BACKPORT_TARGET_BRANCH}" ]; then

access-control-service/LICENSE-binary

@@ -242,6 +242,9 @@ Scala/Java jars:
   - com.google.guava.listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
   - com.google.j2objc.j2objc-annotations-2.8.jar
   - com.helger.profiler-1.1.1.jar
+  - com.squareup.okhttp3.okhttp-4.12.0.jar
+  - com.squareup.okio.okio-3.6.0.jar
+  - com.squareup.okio.okio-jvm-3.6.0.jar
   - com.thesamet.scalapb.lenses_2.13-0.11.20.jar
   - com.thesamet.scalapb.scalapb-json4s_2.13-0.12.0.jar
   - com.thesamet.scalapb.scalapb-runtime_2.13-0.11.20.jar
@@ -300,6 +303,18 @@ Scala/Java jars:
   - io.fabric8.kubernetes-model-scheduling-6.12.1.jar
   - io.fabric8.kubernetes-model-storageclass-6.12.1.jar
   - io.fabric8.zjsonpatch-0.3.0.jar
+  - io.opentelemetry.opentelemetry-api-1.50.0.jar
+  - io.opentelemetry.opentelemetry-context-1.50.0.jar
+  - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar
+  - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar
+  - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar
+  - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar
   - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar
   - jakarta.inject.jakarta.inject-api-2.0.1.jar
   - jakarta.validation.jakarta.validation-api-3.0.2.jar
@@ -318,6 +333,11 @@ Scala/Java jars:
   - org.hibernate.validator.hibernate-validator-7.0.5.Final.jar
   - org.javassist.javassist-3.30.2-GA.jar
   - org.jboss.logging.jboss-logging-3.5.3.Final.jar
+  - org.jetbrains.annotations-13.0.jar
+  - org.jetbrains.kotlin.kotlin-stdlib-1.9.10.jar
+  - org.jetbrains.kotlin.kotlin-stdlib-common-1.9.10.jar
+  - org.jetbrains.kotlin.kotlin-stdlib-jdk7-1.9.10.jar
+  - org.jetbrains.kotlin.kotlin-stdlib-jdk8-1.9.10.jar
   - org.jooq.jooq-3.19.36.jar
   - org.json4s.json4s-ast_2.13-4.0.1.jar
   - org.json4s.json4s-jackson-core_2.13-4.0.1.jar

access-control-service/src/main/resources/access-control-service-web-config.yaml

@@ -26,6 +26,19 @@ server:
 
 logging:
   level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO}
+  loggers:
+    # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code
+    # (org.apache.texera) without the framework firehose.
+    "org.apache.pekko": WARN
+    "org.apache.iceberg": WARN
+    "org.apache.hadoop": WARN
+    "org.apache.kafka": WARN
+    "org.eclipse.jetty": WARN
+    "org.glassfish.jersey": WARN
+    "io.grpc": WARN
+    "io.netty": WARN
+    "com.zaxxer.hikari": WARN
+    "software.amazon.awssdk": WARN
   appenders:
     - type: console
       threshold: ${TEXERA_SERVICE_LOG_LEVEL:-INFO}

access-control-service/src/main/scala/org/apache/texera/service/AccessControlService.scala

@@ -59,6 +59,8 @@ class AccessControlService extends Application[AccessControlServiceConfiguration
       configuration: AccessControlServiceConfiguration,
       environment: Environment
   ): Unit = {
+    // Bridge this service's logs to the OTel collector under its own service.name.
+    org.apache.texera.observability.OtelInit.init("access-control-service")
     // Serve backend at /api
     environment.jersey.setUrlPattern("/api/*")
 

amber/LICENSE-binary-java

@@ -363,6 +363,18 @@ Scala/Java jars:
   - org.jspecify.jspecify-1.0.0.jar
   - io.opencensus.opencensus-api-0.31.1.jar
   - io.opencensus.opencensus-contrib-http-util-0.31.1.jar
+  - io.opentelemetry.opentelemetry-api-1.50.0.jar
+  - io.opentelemetry.opentelemetry-context-1.50.0.jar
+  - io.opentelemetry.opentelemetry-exporter-common-1.50.0.jar
+  - io.opentelemetry.opentelemetry-exporter-otlp-1.50.0.jar
+  - io.opentelemetry.opentelemetry-exporter-otlp-common-1.50.0.jar
+  - io.opentelemetry.opentelemetry-exporter-sender-okhttp-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-common-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-extension-autoconfigure-spi-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-logs-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-metrics-1.50.0.jar
+  - io.opentelemetry.opentelemetry-sdk-trace-1.50.0.jar
   - io.perfmark.perfmark-api-0.27.0.jar
   - io.r2dbc.r2dbc-spi-1.0.0.RELEASE.jar
   - io.reactivex.rxjava3.rxjava-3.1.12.jar

amber/src/main/resources/computing-unit-master-config.yml

@@ -34,6 +34,18 @@ logging:
   level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO}
   loggers:
     "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO}
+    # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code
+    # (org.apache.texera) without the framework firehose.
+    "org.apache.pekko": WARN
+    "org.apache.iceberg": WARN
+    "org.apache.hadoop": WARN
+    "org.apache.kafka": WARN
+    "org.eclipse.jetty": WARN
+    "org.glassfish.jersey": WARN
+    "io.grpc": WARN
+    "io.netty": WARN
+    "com.zaxxer.hikari": WARN
+    "software.amazon.awssdk": WARN
   appenders:
     - type: console
       logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n"

amber/src/main/resources/texera-compiling-service-web-config.yml

@@ -43,6 +43,18 @@ logging:
   level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO}
   loggers:
     "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO}
+    # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code
+    # (org.apache.texera) without the framework firehose.
+    "org.apache.pekko": WARN
+    "org.apache.iceberg": WARN
+    "org.apache.hadoop": WARN
+    "org.apache.kafka": WARN
+    "org.eclipse.jetty": WARN
+    "org.glassfish.jersey": WARN
+    "io.grpc": WARN
+    "io.netty": WARN
+    "com.zaxxer.hikari": WARN
+    "software.amazon.awssdk": WARN
   appenders:
     - type: console
       logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n"

amber/src/main/resources/web-config.yml

@@ -43,6 +43,18 @@ logging:
   level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO}
   loggers:
     "io.dropwizard": ${TEXERA_SERVICE_LOG_LEVEL:-INFO}
+    # Cap noisy frameworks at WARN so TRACE/DEBUG surfaces Texera code
+    # (org.apache.texera) without the framework firehose.
+    "org.apache.pekko": WARN
+    "org.apache.iceberg": WARN
+    "org.apache.hadoop": WARN
+    "org.apache.kafka": WARN
+    "org.eclipse.jetty": WARN
+    "org.glassfish.jersey": WARN
+    "io.grpc": WARN
+    "io.netty": WARN
+    "com.zaxxer.hikari": WARN
+    "software.amazon.awssdk": WARN
   appenders:
     - type: console
       logFormat: "[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n"

amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala

@@ -138,6 +138,7 @@ class ComputingUnitMaster extends io.dropwizard.Application[Configuration] with
   }
 
   override def run(configuration: Configuration, environment: Environment): Unit = {
+    org.apache.texera.observability.OtelInit.init("computing-unit-master")
     ObjectMapperUtils.warmupObjectMapperForOperatorsSerde()
 
     SqlServer.initConnection(

amber/src/main/scala/org/apache/texera/web/TexeraWebApplication.scala

@@ -101,6 +101,7 @@ class TexeraWebApplication
   }
 
   override def run(configuration: TexeraWebConfiguration, environment: Environment): Unit = {
+    org.apache.texera.observability.OtelInit.init("texera-web-application")
     ObjectMapperUtils.warmupObjectMapperForOperatorsSerde()
 
     // serve backend at /api

bin/k8s/values-development.yaml

@@ -435,6 +435,21 @@ texeraEnvVars:
     # same value, so this is a fixed shared string rather than a per-pod random one.
     # Production environments MUST override this with a securely generated secret.
     value: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+  # OpenTelemetry (observability). Disabled by default; set OTEL_SDK_DISABLED to
+  # "false" and point the endpoint at a reachable OTLP collector (http/https only) to enable.
+  - name: OTEL_SDK_DISABLED
+    value: "true"
+  - name: OTEL_EXPORTER_OTLP_ENDPOINT
+    value: "http://127.0.0.1:4317"
+  # Comma-separated resource attributes (k1=v1,k2=v2); service.name is set per service.
+  - name: OTEL_RESOURCE_ATTRIBUTES
+    value: ""
+  # Comma-separated extra hosts allowed for the OTLP endpoint (loopback allowed by default).
+  - name: TEXERA_OTEL_ALLOWED_HOSTS
+    value: ""
+  # Metric export interval in ms.
+  - name: OTEL_METRIC_EXPORT_INTERVAL
+    value: "30000"
 
 yWebsocketServer:
   name: y-websocket-server

bin/k8s/values.yaml

@@ -545,6 +545,21 @@ texeraEnvVars:
     # If using VMs for CUs change this to the desired runtime otherwise leave as "". Recommended: "kata-qemu-nvidia-gpu-runtime-rs"
     # Note: kata containers must be installed to use kata-qemu-nvidia-gpu-runtime-r
     value: ""
+  # OpenTelemetry (observability). Disabled by default; set OTEL_SDK_DISABLED to
+  # "false" and point the endpoint at a reachable OTLP collector (http/https only) to enable.
+  - name: OTEL_SDK_DISABLED
+    value: "true"
+  - name: OTEL_EXPORTER_OTLP_ENDPOINT
+    value: "http://127.0.0.1:4317"
+  # Comma-separated resource attributes (k1=v1,k2=v2); service.name is set per service.
+  - name: OTEL_RESOURCE_ATTRIBUTES
+    value: ""
+  # Comma-separated extra hosts allowed for the OTLP endpoint (loopback allowed by default).
+  - name: TEXERA_OTEL_ALLOWED_HOSTS
+    value: ""
+  # Metric export interval in ms.
+  - name: OTEL_METRIC_EXPORT_INTERVAL
+    value: "30000"
 
 yWebsocketServer:
   name: y-websocket-server

bin/single-node/.env

@@ -101,6 +101,18 @@ TEXERA_DASHBOARD_SERVICE_ENDPOINT=http://dashboard-service:8080
 WORKFLOW_COMPILING_SERVICE_ENDPOINT=http://workflow-compiling-service:9090
 WORKFLOW_EXECUTION_SERVICE_ENDPOINT=http://workflow-runtime-coordinator-service:8085
 
+# OpenTelemetry (observability). Disabled by default; the SDK stays inert until
+# OTEL_SDK_DISABLED=false. Point the endpoint at a reachable OTLP collector
+# (http/https only) before enabling.
+OTEL_SDK_DISABLED=true
+OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4317
+# Comma-separated resource attributes (k1=v1,k2=v2); service.name is set per service.
+OTEL_RESOURCE_ATTRIBUTES=
+# Comma-separated extra hosts allowed for the OTLP endpoint (loopback allowed by default).
+TEXERA_OTEL_ALLOWED_HOSTS=
+# Metric export interval in ms.
+OTEL_METRIC_EXPORT_INTERVAL=30000
+
 # Notebook migration tool
 # Toggles the tool in the GUI; the migration itself needs an LLM key above.
 GUI_WORKFLOW_WORKSPACE_PYTHON_NOTEBOOK_MIGRATION_ENABLED=true

build.sbt

@@ -122,14 +122,19 @@
 lazy val Util = (project in file("common/util")).settings(commonModuleSettings)
 lazy val DAO = (project in file("common/dao")).settings(commonModuleSettings)
 lazy val Config = (project in file("common/config")).settings(commonModuleSettings)
+// OpenTelemetry bootstrap (OtelInit, log appender, sanitizer) shared by every
+// service entry point; pins the OTel dependency versions in one place. Depends
+// on Config to read OTEL_* settings from observability.conf.
+lazy val Observability =
+  (project in file("common/observability")).settings(commonModuleSettings).dependsOn(Config)
 lazy val Resource = (project in file("common/resource")).settings(commonModuleSettings).dependsOn(DAO)
 lazy val Auth = (project in file("common/auth"))
   .settings(commonModuleSettings)
   .configs(Test)
   .dependsOn(DAO, Config)
   .dependsOn(DAO % "test->test") // reuse MockTexeraDB embedded Postgres in tests
 lazy val ConfigService = (project in file("config-service"))
-  .dependsOn(Auth, Config, DAO, Resource)
+  .dependsOn(Auth, Config, DAO, Resource, Observability)
   .dependsOn(DAO % "test->test") // reuse MockTexeraDB embedded Postgres in tests
   .settings(commonModuleSettings)
   .settings(
@@ -139,7 +144,7 @@
     )
   )
 lazy val AccessControlService = (project in file("access-control-service"))
-  .dependsOn(Auth, Config, DAO, Resource)
+  .dependsOn(Auth, Config, DAO, Resource, Observability)
   .settings(commonModuleSettings)
   .settings(
     dependencyOverrides ++= Seq(
@@ -163,7 +168,7 @@
   .configs(Test)
   .dependsOn(DAO % "test->test") // test scope dependency
 lazy val ComputingUnitManagingService = (project in file("computing-unit-managing-service"))
-  .dependsOn(WorkflowCore, Auth, Config, Resource)
+  .dependsOn(WorkflowCore, Auth, Config, Resource, Observability)
   .configs(Test)
   .dependsOn(DAO % "test->test") // reuse MockTexeraDB embedded Postgres in tests
   .settings(commonModuleSettings)
@@ -210,7 +215,7 @@
   )
 lazy val FileService = (project in file("file-service"))
   .settings(commonModuleSettings)
-  .dependsOn(WorkflowCore, Auth, Config, Resource, Util)
+  .dependsOn(WorkflowCore, Auth, Config, Resource, Util, Observability)
   .configs(Test)
   .dependsOn(DAO % "test->test") // test scope dependency
   .dependsOn(WorkflowCore % "test->test") // reuse RustFSContainer in MockLakeFS
@@ -224,12 +229,12 @@
     // Each testcontainers-based suite starts its own LakeFS/RustFS/Postgres stack
     // and mutates JVM-wide singletons (StorageConfig endpoints, LakeFS client),
     // so every suite gets its own forked JVM; sbt runs forked groups one at a
     // time by default (Tags.ForkedTestGroup limit), keeping the stacks serial.
     Test / fork := true,
     Test / forkOptions := (Test / forkOptions).value
       .withWorkingDirectory((ThisBuild / baseDirectory).value),
     Test / testGrouping := (Test / definedTests).value.map { suite =>
       Tests.Group(suite.name, Seq(suite), Tests.SubProcess((Test / forkOptions).value))
     }
   )
 
@@ -241,7 +246,7 @@
   .configs(Test)
   .dependsOn(WorkflowOperator)
 lazy val WorkflowCompilingService = (project in file("workflow-compiling-service"))
-  .dependsOn(WorkflowCompiler, Auth, Config, Resource)
+  .dependsOn(WorkflowCompiler, Auth, Config, Resource, Observability)
   .settings(commonModuleSettings)
   .settings(
     dependencyOverrides ++= Seq(
@@ -253,7 +258,7 @@
   )
 
 lazy val WorkflowExecutionService = (project in file("amber"))
-  .dependsOn(WorkflowCompiler, Auth, Config)
+  .dependsOn(WorkflowCompiler, Auth, Config, Observability)
   .settings(commonModuleSettings)
   .settings(
     dependencyOverrides ++= Seq(
@@ -272,7 +277,7 @@
   .configs(Test)
   .dependsOn(DAO % "test->test", Auth % "test->test") // test scope dependency
 lazy val NotebookMigrationService = (project in file("notebook-migration-service"))
-  .dependsOn(Auth, Config, DAO, Resource)
+  .dependsOn(Auth, Config, DAO, Resource, Observability)
   .settings(commonModuleSettings)
   .settings(
     dependencyOverrides ++= Seq(
@@ -288,6 +293,7 @@
     // common libraries
     Auth,
     Config,
+    Observability,
     Resource,
     Util,
     DAO,

common/config/src/main/resources/observability.conf

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# OpenTelemetry SDK bootstrap settings, consumed by OtelInit. Each value has a
+# safe default and an optional environment override, so an operator can find and
+# tune every knob here instead of in code.
+observability {
+    # Master switch. Disabled by default: no exporters start and no telemetry is
+    # emitted until this is set to false (i.e. OTEL_SDK_DISABLED=false).
+    sdk-disabled = "true"
+    sdk-disabled = ${?OTEL_SDK_DISABLED}
+
+    # OTLP collector endpoint. http/https only; loopback-only host allowlist by
+    # default (extend via allowed-hosts below).
+    endpoint = "http://127.0.0.1:4317"
+    endpoint = ${?OTEL_EXPORTER_OTLP_ENDPOINT}
+
+    # Comma-separated resource attributes (k1=v1,k2=v2). service.name is set by
+    # the service and cannot be overridden here.
+    resource-attributes = ""
+    resource-attributes = ${?OTEL_RESOURCE_ATTRIBUTES}
+
+    # Comma-separated extra hosts added to the endpoint allowlist.
+    allowed-hosts = ""
+    allowed-hosts = ${?TEXERA_OTEL_ALLOWED_HOSTS}
+
+    # Metric export interval in milliseconds; out-of-range values fall back to
+    # the default.
+    metric-export-interval-ms = "30000"
+    metric-export-interval-ms = ${?OTEL_METRIC_EXPORT_INTERVAL}
+}

common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala

@@ -48,6 +48,13 @@ object EnvironmentalVariable {
   val ENV_USER_JWT_TOKEN = "USER_JWT_TOKEN"
   val ENV_AUTH_JWT_SECRET = "AUTH_JWT_SECRET"
 
+  // OpenTelemetry observability (see observability.conf)
+  val ENV_OTEL_SDK_DISABLED = "OTEL_SDK_DISABLED"
+  val ENV_OTEL_EXPORTER_OTLP_ENDPOINT = "OTEL_EXPORTER_OTLP_ENDPOINT"
+  val ENV_OTEL_RESOURCE_ATTRIBUTES = "OTEL_RESOURCE_ATTRIBUTES"
+  val ENV_OTEL_METRIC_EXPORT_INTERVAL = "OTEL_METRIC_EXPORT_INTERVAL"
+  val ENV_TEXERA_OTEL_ALLOWED_HOSTS = "TEXERA_OTEL_ALLOWED_HOSTS"
+
   /**
     * Dataset-mount vars injected into the CU pod. The mount is performed by the per-node
     * mounter and reaches the pod through mount propagation, so the pod only needs to know