TEZ-4677: Upgrades Tez to compile and run with JDK 25

README.md

@@ -46,9 +46,9 @@ For instructions on how to contribute to Tez, refer to:
 Requirements
 ------------
 
-* JDK 21+
+* JDK 25+
 * Maven 3.9.14 or later
-* spotbugs 4.9.3 or later (if running spotbugs)
+* spotbugs 4.10.2 or later (if running spotbugs)
 * ProtocolBuffer 3.25.5
 * Hadoop 3.x
 

build-tools/docker/Dockerfile

@@ -35,7 +35,7 @@ ARG HADOLINT_VERSION="2.14.0"
 ARG HADOLINT_SHA="5ffd7ed8f27894941a82f06229ed0dc75814eeb985d224d4fc3c7cf516f31cc7e6cc2d57348d4026084ac622c765e63c8274fdc6c36c0de03c4a5dda8f4ebf6f"
 ARG BUF_VERSION="1.68.2"
 ARG BUF_SHA="557ea42d00458466e3421bd1cf5781d882a95b0c1c0e54efffc326fdf9993d02"
-ARG SPOTBUGS_VERSION="4.9.3"
+ARG SPOTBUGS_VERSION="4.10.2"
 ARG MAVEN_VERSION="3.9.15"
 ARG ASTROID_VERSION="4.0.3"
 ARG PYLINT_VERSION="4.0.4"
@@ -106,7 +106,7 @@ RUN apk add --no-cache curl tar \
 ####################
 
 # Switching to eclipse-temurin to resolves cross-platform amd64/arm64 architecture issues
-FROM eclipse-temurin:21-jdk-noble AS tezbase
+FROM eclipse-temurin:25-jdk-noble AS tezbase
 
 WORKDIR /root
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]

pom.xml

@@ -23,7 +23,7 @@
   <parent>
     <groupId>org.apache</groupId>
     <artifactId>apache</artifactId>
-    <version>37</version>
+    <version>38</version>
   </parent>
   <groupId>org.apache.tez</groupId>
   <artifactId>tez</artifactId>
@@ -47,8 +47,8 @@
     <!-- Build Properties -->
     <!-- Default empty argLine to prevent Surefire crashes when the 'jacoco' profile is inactive -->
     <argLine/>
-    <javaVersion>21</javaVersion>
-    <maven.compiler.release>21</maven.compiler.release>
+    <javaVersion>25</javaVersion>
+    <maven.compiler.release>25</maven.compiler.release>
     <maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
     <failIfNoTests>false</failIfNoTests>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -76,8 +76,8 @@
     <commons-lang3.version>3.19.0</commons-lang3.version>
     <curator.version>5.9.0</curator.version>
     <dependency-check-maven.version>3.2.0</dependency-check-maven.version>
-    <spotbugs.version>4.9.3</spotbugs.version>
-    <spotbugs-maven-plugin.version>4.9.3.0</spotbugs-maven-plugin.version>
+    <spotbugs.version>4.10.2</spotbugs.version>
+    <spotbugs-maven-plugin.version>4.10.2.0</spotbugs-maven-plugin.version>
     <plexus-velocity.version>2.3.0</plexus-velocity.version>
     <frontend-maven-plugin.version>1.15.0</frontend-maven-plugin.version>
     <guava.version>33.4.8-jre</guava.version>

tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java

@@ -69,7 +69,7 @@ public void maxTasksAllocationsCannotBeExceeded() {
     // Allocate up to max tasks
     for (int i = 0; i < MAX_TASKS; i++) {
       Priority priority = Priority.newInstance(20);
-      requestHandler.addAllocateTaskRequest(new Long(i), null, priority, null);
+      requestHandler.addAllocateTaskRequest((long) i, null, priority, null);
       requestHandler.dispatchRequest();
       requestHandler.allocateTask();
     }
@@ -80,7 +80,7 @@ public void maxTasksAllocationsCannotBeExceeded() {
 
     // Deallocate down to zero
     for (int i = 0; i < MAX_TASKS; i++) {
-      requestHandler.addDeallocateTaskRequest(new Long(i));
+      requestHandler.addDeallocateTaskRequest((long) i);
       requestHandler.dispatchRequest();
     }
 

tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java

@@ -153,10 +153,10 @@ public void preemptDescendantsOnly() {
 
     ApplicationId appId = ApplicationId.newInstance(2000, 1);
     ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
-    Long parentTask1 = new Long(1);
-    Long parentTask2 = new Long(2);
-    Long childTask1 = new Long(3);
-    Long grandchildTask1 = new Long(4);
+    Long parentTask1 = 1L;
+    Long parentTask2 = 2L;
+    Long childTask1 = 3L;
+    Long grandchildTask1 = 4L;
 
     TaskSchedulerContext
         mockContext = TestTaskSchedulerHelpers.setupMockTaskSchedulerContext("", 0, "", true,

tez-dist/src/docker/Dockerfile

@@ -34,7 +34,7 @@ RUN mkdir -p /opt/tez \
         -C /opt/tez \
     && rm -rf /opt/tez-$TEZ_VERSION.tar.gz
 
-FROM eclipse-temurin:21-jdk-ubi9-minimal AS run
+FROM eclipse-temurin:25-jdk-ubi9-minimal AS run
 
 ARG UID=1000
 ARG TEZ_VERSION

tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java

@@ -30,15 +30,15 @@ private void testCRUD(ObjectRegistry objectRegistry) {
 
     Assert.assertNull(objectRegistry.get("foo"));
     Assert.assertFalse(objectRegistry.delete("foo"));
-    Integer one = new Integer(1);
-    Integer two_1 = new Integer(2);
-    Integer two_2 = new Integer(3);
+    Integer one = 1;
+    Integer twoFirst = 2;
+    Integer twoSecond = 3;
     Assert.assertNull(objectRegistry.cacheForDAG("one", one));
     Assert.assertEquals(one, objectRegistry.get("one"));
-    Assert.assertNull(objectRegistry.cacheForDAG("two", two_1));
-    Assert.assertNotNull(objectRegistry.cacheForSession("two", two_2));
-    Assert.assertNotEquals(two_1, objectRegistry.get("two"));
-    Assert.assertEquals(two_2, objectRegistry.get("two"));
+    Assert.assertNull(objectRegistry.cacheForDAG("two", twoFirst));
+    Assert.assertNotNull(objectRegistry.cacheForSession("two", twoSecond));
+    Assert.assertNotEquals(twoFirst, objectRegistry.get("two"));
+    Assert.assertEquals(twoSecond, objectRegistry.get("two"));
     Assert.assertTrue(objectRegistry.delete("one"));
     Assert.assertFalse(objectRegistry.delete("one"));
 

tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/FastByteComparisons.java

@@ -36,16 +36,13 @@
  * limitations under the License.
  */
 
-import java.lang.reflect.Field;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.VarHandle;
 import java.nio.ByteOrder;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
 
 import com.google.common.primitives.Longs;
 import com.google.common.primitives.UnsignedBytes;
 
-import sun.misc.Unsafe;
-
 /**
  * Same as {@link org.apache.hadoop.io.FastByteComparisons}
  *
@@ -79,10 +76,10 @@ private static Comparer<byte[]> lexicographicalComparerJavaImpl() {
 
   /**
    * Provides a lexicographical comparer implementation; either a Java
-   * implementation or a faster implementation based on {@link sun.misc.Unsafe}.
+   * implementation or a faster implementation based on {@link java.lang.invoke.VarHandle}.
    *
    * <p>Uses reflection to gracefully fall back to the Java implementation if
-   * {@code Unsafe} isn't available.
+   * {@code VarHandle} isn't available.
    */
   private static class LexicographicalComparerHolder {
     static final String UNSAFE_COMPARER_NAME =
@@ -137,35 +134,12 @@ public int compareTo(byte[] buffer1, int offset1, int length1,
     private enum UnsafeComparer implements Comparer<byte[]> {
       INSTANCE;
 
-      static final Unsafe theUnsafe;
-
-      /** The offset to the first element in a byte array. */
-      static final int BYTE_ARRAY_BASE_OFFSET;
-
-      static {
-        theUnsafe = (Unsafe) AccessController.doPrivileged(
-            new PrivilegedAction<Object>() {
-              @Override
-              public Object run() {
-                try {
-                  Field f = Unsafe.class.getDeclaredField("theUnsafe");
-                  f.setAccessible(true);
-                  return f.get(null);
-                } catch (NoSuchFieldException | IllegalAccessException e) {
-                  // It doesn't matter what we throw;
-                  // it's swallowed in getBestComparer().
-                  throw new Error();
-                }
-              }
-            });
-
-        BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class);
-
-        // sanity check - this should never fail
-        if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
-          throw new AssertionError();
-        }
-      }
+      /**
+       * A VarHandle for reading longs from a byte array using native byte order.
+       * This replaces the deprecated sun.misc.Unsafe getLong(Object, long) API.
+       */
+      static final VarHandle LONG_VIEW =
+          MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.nativeOrder());
 
       static final boolean littleEndian =
           ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
@@ -200,17 +174,15 @@ public int compareTo(byte[] buffer1, int offset1, int length1,
         }
         int minLength = Math.min(length1, length2);
         int minWords = minLength / Longs.BYTES;
-        int offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
-        int offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
 
         /*
          * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a
          * time is no slower than comparing 4 bytes at a time even on 32-bit.
          * On the other hand, it is substantially faster on 64-bit.
          */
         for (int i = 0; i < minWords * Longs.BYTES; i += Longs.BYTES) {
-          long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
-          long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
+          long lw = (long) LONG_VIEW.get(buffer1, offset1 + i);
+          long rw = (long) LONG_VIEW.get(buffer2, offset2 + i);
           long diff = lw ^ rw;
 
           if (diff != 0) {

tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java

@@ -148,8 +148,8 @@ public void testLargeDataSize() throws IOException {
     Assert.assertEquals(2, newEdgeManagers.size());
     Assert.assertEquals(0, manager.pendingTasks.size()); // all tasks scheduled
     Assert.assertEquals(2, scheduledTasks.size());
-    Assert.assertTrue(scheduledTasks.contains(new Integer(0)));
-    Assert.assertTrue(scheduledTasks.contains(new Integer(1)));
+    Assert.assertTrue(scheduledTasks.contains(0));
+    Assert.assertTrue(scheduledTasks.contains(1));
     Assert.assertEquals(4, manager.numBipartiteSourceTasksCompleted);
     Assert.assertEquals(4, manager.numVertexManagerEventsReceived);
     Assert.assertEquals(Long.MAX_VALUE >> 1 << 1, manager.completedSourceTasksOutputSize);

tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerBase.java

@@ -423,8 +423,8 @@ public void testAutoParallelism() throws Exception {
     // TODO improve tests for parallelism
     Assert.assertEquals(0, manager.pendingTasks.size()); // all tasks scheduled
     Assert.assertEquals(2, scheduledTasks.size());
-    Assert.assertTrue(scheduledTasks.contains(new Integer(0)));
-    Assert.assertTrue(scheduledTasks.contains(new Integer(1)));
+    Assert.assertTrue(scheduledTasks.contains(0));
+    Assert.assertTrue(scheduledTasks.contains(1));
     Assert.assertEquals(2, manager.numBipartiteSourceTasksCompleted);
     Assert.assertEquals(2, manager.numVertexManagerEventsReceived);
     Assert.assertEquals(100 * MB, manager.completedSourceTasksOutputSize);

tez-tests/src/test/java/org/apache/tez/test/TestInput.java

@@ -75,7 +75,15 @@ public class TestInput extends AbstractLogicalInput {
   Set<Integer> failingTaskIndices = Sets.newHashSet();
   Set<Integer> failingTaskAttempts = Sets.newHashSet();
   Set<Integer> failingInputIndices = Sets.newHashSet();
-  Integer failAll = new Integer(-1);
+  private Integer failAll = -1;
+
+  public Integer getFailAll() {
+    return failAll;
+  }
+
+  public void setFailAll(Integer failAll) {
+    this.failAll = failAll;
+  }
   int[] inputValues;
 
   /**
@@ -227,7 +235,7 @@ public int doRead() {
               previousAttemptReadFailed = true;
             } else {
               for (int i=0 ; i<getContext().getTaskAttemptNumber(); ++i) {
-                if (failingTaskAttempts.contains(new Integer(i))) {
+                if (failingTaskAttempts.contains(i)) {
                   previousAttemptReadFailed = true;
                   break;
                 }

tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java

@@ -59,7 +59,15 @@ public class TestProcessor extends AbstractLogicalIOProcessor {
   long sleepMs;
   Set<Integer> failingTaskIndices = Sets.newHashSet();
   int failingTaskAttemptUpto = 0;
-  Integer failAll = new Integer(-1);
+  private Integer failAll = -1;
+
+  public Integer getFailAll() {
+    return failAll;
+  }
+
+  public void setFailAll(Integer failAll) {
+    this.failAll = failAll;
+  }
 
   int verifyValue = -1;
   Set<Integer> verifyTaskIndices = Sets.newHashSet();
@@ -293,7 +301,7 @@ public void run(Map<String, LogicalInput> inputs,
         + " attempt: " + getContext().getTaskAttemptNumber()
         + " is: " + sum);
     if (verifyTaskIndices
-        .contains(new Integer(getContext().getTaskIndex()))) {
+        .contains(getContext().getTaskIndex())) {
       if (verifyValue != -1 && verifyValue != sum) {
         // expected output value set and not equal to observed value
         String msg = "Expected output mismatch of current FailingProcessor: "