diff --git a/README.md b/README.md index 422c560783..2fc75d717d 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,9 @@ For instructions on how to contribute to Tez, refer to: Requirements ------------ -* JDK 21+ +* JDK 25+ * Maven 3.9.14 or later -* spotbugs 4.9.3 or later (if running spotbugs) +* spotbugs 4.10.2 or later (if running spotbugs) * ProtocolBuffer 3.25.5 * Hadoop 3.x diff --git a/build-tools/docker/Dockerfile b/build-tools/docker/Dockerfile index 12fa58d890..ab0201357b 100644 --- a/build-tools/docker/Dockerfile +++ b/build-tools/docker/Dockerfile @@ -35,7 +35,7 @@ ARG HADOLINT_VERSION="2.14.0" ARG HADOLINT_SHA="5ffd7ed8f27894941a82f06229ed0dc75814eeb985d224d4fc3c7cf516f31cc7e6cc2d57348d4026084ac622c765e63c8274fdc6c36c0de03c4a5dda8f4ebf6f" ARG BUF_VERSION="1.68.2" ARG BUF_SHA="557ea42d00458466e3421bd1cf5781d882a95b0c1c0e54efffc326fdf9993d02" -ARG SPOTBUGS_VERSION="4.9.3" +ARG SPOTBUGS_VERSION="4.10.2" ARG MAVEN_VERSION="3.9.15" ARG ASTROID_VERSION="4.0.3" ARG PYLINT_VERSION="4.0.4" @@ -106,7 +106,7 @@ RUN apk add --no-cache curl tar \ #################### # Switching to eclipse-temurin to resolves cross-platform amd64/arm64 architecture issues -FROM eclipse-temurin:21-jdk-noble AS tezbase +FROM eclipse-temurin:25-jdk-noble AS tezbase WORKDIR /root SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/pom.xml b/pom.xml index fec11d4d7c..31e7dd8151 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ org.apache apache - 37 + 38 org.apache.tez tez @@ -47,8 +47,8 @@ - 21 - 21 + 25 + 25 true false UTF-8 @@ -76,8 +76,8 @@ 3.19.0 5.9.0 3.2.0 - 4.9.3 - 4.9.3.0 + 4.10.2 + 4.10.2.0 2.3.0 1.15.0 33.4.8-jre diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java index 9fdbe5ef2f..234b917208 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java @@ -69,7 +69,7 @@ public void maxTasksAllocationsCannotBeExceeded() { // Allocate up to max tasks for (int i = 0; i < MAX_TASKS; i++) { Priority priority = Priority.newInstance(20); - requestHandler.addAllocateTaskRequest(new Long(i), null, priority, null); + requestHandler.addAllocateTaskRequest((long) i, null, priority, null); requestHandler.dispatchRequest(); requestHandler.allocateTask(); } @@ -80,7 +80,7 @@ public void maxTasksAllocationsCannotBeExceeded() { // Deallocate down to zero for (int i = 0; i < MAX_TASKS; i++) { - requestHandler.addDeallocateTaskRequest(new Long(i)); + requestHandler.addDeallocateTaskRequest((long) i); requestHandler.dispatchRequest(); } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java index fafeb5e436..5f89317920 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java @@ -162,10 +162,10 @@ public void preemptDescendantsOnly() { ApplicationId appId = ApplicationId.newInstance(2000, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); - Long parentTask1 = new Long(1); - Long parentTask2 = new Long(2); - Long childTask1 = new Long(3); - Long grandchildTask1 = new Long(4); + Long parentTask1 = 1L; + Long parentTask2 = 2L; + Long childTask1 = 3L; + Long grandchildTask1 = 4L; TaskSchedulerContext mockContext = TestTaskSchedulerHelpers.setupMockTaskSchedulerContext("", 0, "", true, appAttemptId, 1000l, null, tezConf); diff --git a/tez-dist/src/docker/Dockerfile b/tez-dist/src/docker/Dockerfile index 3d13f5e2b9..7b18b0f9da 100644 --- a/tez-dist/src/docker/Dockerfile +++ b/tez-dist/src/docker/Dockerfile @@ -34,7 +34,7 @@ RUN mkdir -p /opt/tez \ -C /opt/tez \ && rm -rf /opt/tez-$TEZ_VERSION.tar.gz -FROM eclipse-temurin:21-jdk-ubi9-minimal AS run +FROM eclipse-temurin:25-jdk-ubi9-minimal AS run ARG UID=1000 ARG TEZ_VERSION diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java index 72f6860259..b0275a8e53 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/objectregistry/TestObjectRegistry.java @@ -39,15 +39,15 @@ private void testCRUD(ObjectRegistry objectRegistry) { assertNull(objectRegistry.get("foo")); assertFalse(objectRegistry.delete("foo")); - Integer one = new Integer(1); - Integer two_1 = new Integer(2); - Integer two_2 = new Integer(3); + Integer one = 1; + Integer twoFirst = 2; + Integer twoSecond = 3; assertNull(objectRegistry.cacheForDAG("one", one)); assertEquals(one, objectRegistry.get("one")); - assertNull(objectRegistry.cacheForDAG("two", two_1)); - assertNotNull(objectRegistry.cacheForSession("two", two_2)); - assertNotEquals(two_1, objectRegistry.get("two")); - assertEquals(two_2, objectRegistry.get("two")); + assertNull(objectRegistry.cacheForDAG("two", twoFirst)); + assertNotNull(objectRegistry.cacheForSession("two", twoSecond)); + assertNotEquals(twoFirst, objectRegistry.get("two")); + assertEquals(twoSecond, objectRegistry.get("two")); assertTrue(objectRegistry.delete("one")); assertFalse(objectRegistry.delete("one")); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/FastByteComparisons.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/FastByteComparisons.java index a392416a04..4f66ec8869 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/FastByteComparisons.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/FastByteComparisons.java @@ -36,16 +36,13 @@ * limitations under the License. */ -import java.lang.reflect.Field; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; import java.nio.ByteOrder; -import java.security.AccessController; -import java.security.PrivilegedAction; import com.google.common.primitives.Longs; import com.google.common.primitives.UnsignedBytes; -import sun.misc.Unsafe; - /** * Same as {@link org.apache.hadoop.io.FastByteComparisons} * @@ -79,10 +76,10 @@ private static Comparer lexicographicalComparerJavaImpl() { /** * Provides a lexicographical comparer implementation; either a Java - * implementation or a faster implementation based on {@link sun.misc.Unsafe}. + * implementation or a faster implementation based on {@link java.lang.invoke.VarHandle}. * *

Uses reflection to gracefully fall back to the Java implementation if - * {@code Unsafe} isn't available. + * {@code VarHandle} isn't available. */ private static class LexicographicalComparerHolder { static final String UNSAFE_COMPARER_NAME = @@ -137,35 +134,12 @@ public int compareTo(byte[] buffer1, int offset1, int length1, private enum UnsafeComparer implements Comparer { INSTANCE; - static final Unsafe theUnsafe; - - /** The offset to the first element in a byte array. */ - static final int BYTE_ARRAY_BASE_OFFSET; - - static { - theUnsafe = (Unsafe) AccessController.doPrivileged( - new PrivilegedAction() { - @Override - public Object run() { - try { - Field f = Unsafe.class.getDeclaredField("theUnsafe"); - f.setAccessible(true); - return f.get(null); - } catch (NoSuchFieldException | IllegalAccessException e) { - // It doesn't matter what we throw; - // it's swallowed in getBestComparer(). - throw new Error(); - } - } - }); - - BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class); - - // sanity check - this should never fail - if (theUnsafe.arrayIndexScale(byte[].class) != 1) { - throw new AssertionError(); - } - } + /** + * A VarHandle for reading longs from a byte array using native byte order. + * This replaces the deprecated sun.misc.Unsafe getLong(Object, long) API. + */ + static final VarHandle LONG_VIEW = + MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.nativeOrder()); static final boolean littleEndian = ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN); @@ -200,8 +174,6 @@ public int compareTo(byte[] buffer1, int offset1, int length1, } int minLength = Math.min(length1, length2); int minWords = minLength / Longs.BYTES; - int offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET; - int offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET; /* * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a @@ -209,8 +181,8 @@ public int compareTo(byte[] buffer1, int offset1, int length1, * On the other hand, it is substantially faster on 64-bit. */ for (int i = 0; i < minWords * Longs.BYTES; i += Longs.BYTES) { - long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i); - long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i); + long lw = (long) LONG_VIEW.get(buffer1, offset1 + i); + long rw = (long) LONG_VIEW.get(buffer2, offset2 + i); long diff = lw ^ rw; if (diff != 0) { diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java index 3f6a9ddfb1..8f8dcce238 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java @@ -153,8 +153,8 @@ public void testLargeDataSize() throws IOException { assertEquals(2, newEdgeManagers.size()); assertEquals(0, manager.pendingTasks.size()); // all tasks scheduled assertEquals(2, scheduledTasks.size()); - assertTrue(scheduledTasks.contains(new Integer(0))); - assertTrue(scheduledTasks.contains(new Integer(1))); + assertTrue(scheduledTasks.contains(0)); + assertTrue(scheduledTasks.contains(1)); assertEquals(4, manager.numBipartiteSourceTasksCompleted); assertEquals(4, manager.numVertexManagerEventsReceived); assertEquals(Long.MAX_VALUE >> 1 << 1, manager.completedSourceTasksOutputSize); diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerBase.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerBase.java index beb715b20e..fdc069d2de 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerBase.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerBase.java @@ -432,8 +432,8 @@ public void testAutoParallelism(Class shuffl // TODO improve tests for parallelism assertEquals(0, manager.pendingTasks.size()); // all tasks scheduled assertEquals(2, scheduledTasks.size()); - assertTrue(scheduledTasks.contains(new Integer(0))); - assertTrue(scheduledTasks.contains(new Integer(1))); + assertTrue(scheduledTasks.contains(0)); + assertTrue(scheduledTasks.contains(1)); assertEquals(2, manager.numBipartiteSourceTasksCompleted); assertEquals(2, manager.numVertexManagerEventsReceived); assertEquals(100 * MB, manager.completedSourceTasksOutputSize); diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestInput.java b/tez-tests/src/test/java/org/apache/tez/test/TestInput.java index 9a4532eaae..52b287ca50 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestInput.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestInput.java @@ -75,7 +75,15 @@ public class TestInput extends AbstractLogicalInput { Set failingTaskIndices = Sets.newHashSet(); Set failingTaskAttempts = Sets.newHashSet(); Set failingInputIndices = Sets.newHashSet(); - Integer failAll = new Integer(-1); + private Integer failAll = -1; + + public Integer getFailAll() { + return failAll; + } + + public void setFailAll(Integer failAll) { + this.failAll = failAll; + } int[] inputValues; /** @@ -227,7 +235,7 @@ public int doRead() { previousAttemptReadFailed = true; } else { for (int i=0 ; i failingTaskIndices = Sets.newHashSet(); int failingTaskAttemptUpto = 0; - Integer failAll = new Integer(-1); + private Integer failAll = -1; + + public Integer getFailAll() { + return failAll; + } + + public void setFailAll(Integer failAll) { + this.failAll = failAll; + } int verifyValue = -1; Set verifyTaskIndices = Sets.newHashSet(); @@ -293,7 +301,7 @@ public void run(Map inputs, + " attempt: " + getContext().getTaskAttemptNumber() + " is: " + sum); if (verifyTaskIndices - .contains(new Integer(getContext().getTaskIndex()))) { + .contains(getContext().getTaskIndex())) { if (verifyValue != -1 && verifyValue != sum) { // expected output value set and not equal to observed value String msg = "Expected output mismatch of current FailingProcessor: "