[SPARK-53138][CORE][BUILD] Split common-utils Java code into a new module

.github/workflows/build_and_test.yml

@@ -112,7 +112,7 @@ jobs:
             ui=false
             docs=false
           fi
-          build=`./dev/is-changed.py -m "core,unsafe,kvstore,avro,utils,network-common,network-shuffle,repl,launcher,examples,sketch,variant,api,catalyst,hive-thriftserver,mllib-local,mllib,graphx,streaming,sql-kafka-0-10,streaming-kafka-0-10,streaming-kinesis-asl,kubernetes,hadoop-cloud,spark-ganglia-lgpl,profiler,protobuf,yarn,connect,sql,hive,pipelines"`
+          build=`./dev/is-changed.py -m "core,unsafe,kvstore,avro,utils,utils-java,network-common,network-shuffle,repl,launcher,examples,sketch,variant,api,catalyst,hive-thriftserver,mllib-local,mllib,graphx,streaming,sql-kafka-0-10,streaming-kafka-0-10,streaming-kinesis-asl,kubernetes,hadoop-cloud,spark-ganglia-lgpl,profiler,protobuf,yarn,connect,sql,hive,pipelines"`
           precondition="
             {
               \"build\": \"$build\",
@@ -242,7 +242,7 @@ jobs:
         # Note that the modules below are from sparktestsupport/modules.py.
         modules:
           - >-
-            core, unsafe, kvstore, avro, utils,
+            core, unsafe, kvstore, avro, utils, utils-java,
             network-common, network-shuffle, repl, launcher,
             examples, sketch, variant
           - >-

.github/workflows/maven_test.yml

@@ -67,7 +67,7 @@ jobs:
           - hive2.3
         modules:
           - >-
-            core,launcher,common#unsafe,common#kvstore,common#network-common,common#network-shuffle,common#sketch,common#utils,common#variant
+            core,launcher,common#unsafe,common#kvstore,common#network-common,common#network-shuffle,common#sketch,common#utils,common#utils-java,common#variant
           - >-
             graphx,streaming,hadoop-cloud
           - >-
@@ -214,6 +214,10 @@ jobs:
             # SPARK-52441: Remove sql/pipelines from TEST_MODULES for branch-4.0, this branch can be deleted after the EOL of branch-4.0.
             TEST_MODULES=${TEST_MODULES/,sql\/pipelines/}
             ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Phadoop-cloud -Pjvm-profiler -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} test -fae
+          elif [[ "$MODULES_TO_TEST" == *"common#utils-java"* && "$INPUT_BRANCH" == "branch-4.0" ]]; then
+            # SPARK-53138: Remove common/utils-java from TEST_MODULES for branch-4.0, this branch can be deleted after the EOL of branch-4.0.
+            TEST_MODULES=${TEST_MODULES/,common\/utils-java/}
+            ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Phadoop-cloud -Pjvm-profiler -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} test -fae
           else
             ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Phadoop-cloud -Pjvm-profiler -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} test -fae
           fi

common/network-common/pom.xml

@@ -170,7 +170,7 @@
 
     <dependency>
       <groupId>org.apache.spark</groupId>
-      <artifactId>spark-common-utils_${scala.binary.version}</artifactId>
+      <artifactId>spark-common-utils-java_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
 

common/network-common/src/test/java/org/apache/spark/network/RpcIntegrationSuite.java

@@ -38,7 +38,6 @@
 import org.apache.spark.network.util.MapConfigProvider;
 import org.apache.spark.network.util.TransportConf;
 import org.apache.spark.util.Pair;
-import org.apache.spark.util.SparkFileUtils$;
 
 public class RpcIntegrationSuite {
   static TransportConf conf;
@@ -429,7 +428,7 @@ private static class VerifyingStreamCallback implements StreamCallbackWithID {
 
     void verify() throws IOException {
       if (streamId.equals("file")) {
-        assertTrue(SparkFileUtils$.MODULE$.contentEquals(testData.testFile, outFile),
+        assertTrue(JavaUtils.contentEquals(testData.testFile, outFile),
           "File stream did not match.");
       } else {
         byte[] result = ((ByteArrayOutputStream)out).toByteArray();

common/network-common/src/test/java/org/apache/spark/network/StreamSuite.java

@@ -42,9 +42,9 @@
 import org.apache.spark.network.server.RpcHandler;
 import org.apache.spark.network.server.StreamManager;
 import org.apache.spark.network.server.TransportServer;
+import org.apache.spark.network.util.JavaUtils;
 import org.apache.spark.network.util.MapConfigProvider;
 import org.apache.spark.network.util.TransportConf;
-import org.apache.spark.util.SparkFileUtils$;
 
 public class StreamSuite {
   private static final String[] STREAMS = StreamTestHelper.STREAMS;
@@ -212,7 +212,7 @@ public void run() {
         callback.waitForCompletion(timeoutMs);
 
         if (srcBuffer == null) {
-          assertTrue(SparkFileUtils$.MODULE$.contentEquals(testData.testFile, outFile),
+          assertTrue(JavaUtils.contentEquals(testData.testFile, outFile),
             "File stream did not match.");
         } else {
           ByteBuffer base;

common/network-common/src/test/java/org/apache/spark/network/util/DBProviderSuite.java

@@ -26,8 +26,6 @@
 import java.io.File;
 import java.io.IOException;
 
-import org.apache.spark.util.SparkSystemUtils$;
-
 import static org.junit.jupiter.api.Assumptions.assumeFalse;
 
 public class DBProviderSuite {
@@ -39,7 +37,7 @@ public void testRockDBCheckVersionFailed() throws IOException, InterruptedExcept
 
   @Test
   public void testLevelDBCheckVersionFailed() throws IOException, InterruptedException {
-    assumeFalse(SparkSystemUtils$.MODULE$.isMacOnAppleSilicon());
+    assumeFalse(JavaUtils.isMacOnAppleSilicon);
     testCheckVersionFailed(DBBackend.LEVELDB, "leveldb");
   }
 

common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java

@@ -23,10 +23,10 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.spark.network.shuffle.protocol.ExecutorShuffleInfo;
+import org.apache.spark.network.util.JavaUtils;
 import org.apache.spark.network.util.MapConfigProvider;
 import org.apache.spark.network.util.TransportConf;
 import org.apache.spark.network.shuffle.ExternalShuffleBlockResolver.AppExecId;
-import org.apache.spark.util.SparkStreamUtils$;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
@@ -82,17 +82,17 @@ public void testSortShuffleBlocks() throws IOException {
 
     try (InputStream block0Stream = resolver.getBlockData(
         "app0", "exec0", 0, 0, 0).createInputStream()) {
-      assertEquals(sortBlock0, SparkStreamUtils$.MODULE$.toString(block0Stream));
+      assertEquals(sortBlock0, JavaUtils.toString(block0Stream));
     }
 
     try (InputStream block1Stream = resolver.getBlockData(
         "app0", "exec0", 0, 0, 1).createInputStream()) {
-      assertEquals(sortBlock1, SparkStreamUtils$.MODULE$.toString(block1Stream));
+      assertEquals(sortBlock1, JavaUtils.toString(block1Stream));
     }
 
     try (InputStream blocksStream = resolver.getContinuousBlocksData(
         "app0", "exec0", 0, 0, 0, 2).createInputStream()) {
-      assertEquals(sortBlock0 + sortBlock1, SparkStreamUtils$.MODULE$.toString(blocksStream));
+      assertEquals(sortBlock0 + sortBlock1, JavaUtils.toString(blocksStream));
     }
   }
 

common/network-yarn/pom.xml

@@ -48,7 +48,7 @@
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-tags_${scala.binary.version}</artifactId>
-      <scope>test</scope>
+      <scope>provided</scope>
     </dependency>
 
     <!--

common/utils-java/pom.xml

@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.spark</groupId>
+    <artifactId>spark-parent_2.13</artifactId>
+    <version>4.1.0-SNAPSHOT</version>
+    <relativePath>../../pom.xml</relativePath>
+  </parent>
+
+  <artifactId>spark-common-utils-java_2.13</artifactId>
+  <packaging>jar</packaging>
+  <name>Spark Project Common Java Utils</name>
+  <url>https://spark.apache.org/</url>
+  <properties>
+    <sbt.project.name>common-utils-java</sbt.project.name>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-tags_${scala.binary.version}</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>jul-to-slf4j</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>jcl-over-slf4j</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j2-impl</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-1.2-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-layout-template-json</artifactId>
+    </dependency>
+  </dependencies>
+  <build>
+    <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
+    <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
+  </build>
+</project>

common/utils/src/main/java/org/apache/spark/network/util/JavaUtils.java → common/utils-java/src/main/java/org/apache/spark/network/util/JavaUtils.java

@@ -43,7 +43,6 @@
 import org.apache.spark.internal.SparkLoggerFactory;
 import org.apache.spark.internal.LogKeys;
 import org.apache.spark.internal.MDC;
-import org.apache.spark.util.SparkSystemUtils$;
 
 /**
  * General utilities available in the network package. Many of these are sourced from Spark's
@@ -265,9 +264,7 @@ public static void deleteRecursively(File file, FilenameFilter filter)
     // On Unix systems, use operating system command to run faster
     // If that does not work out, fallback to the Java IO way
     // We exclude Apple Silicon test environment due to the limited resource issues.
-    if (SparkSystemUtils$.MODULE$.isUnix() && filter == null &&
-        !(SparkSystemUtils$.MODULE$.isMac() && (System.getenv("SPARK_TESTING") != null ||
-        System.getProperty("spark.testing") != null))) {
+    if (isUnix && filter == null && !(isMac && isTesting())) {
       try {
         deleteRecursivelyUsingUnixNative(file);
         return;
@@ -663,4 +660,81 @@ public static int checkedCast(long value) {
     }
     return (int) value;
   }
+
+  /** Return true if the content of the files are equal or they both don't exist */
+  public static boolean contentEquals(File file1, File file2) throws IOException {
+    if (file1 == null && file2 != null || file1 != null && file2 == null) {
+      return false;
+    } else if (file1 == null && file2 == null || !file1.exists() && !file2.exists()) {
+      return true;
+    } else if (!file1.exists() || !file2.exists()) {
+      return false;
+    } else if (file1.isDirectory() || file2.isDirectory()) {
+      throw new IllegalArgumentException("Input is not a file: %s or %s".formatted(file1, file2));
+    } else if (file1.length() != file2.length()) {
+      return false;
+    } else {
+      Path path1 = file1.toPath();
+      Path path2 = file2.toPath();
+      return Files.isSameFile(path1, path2) || Files.mismatch(path1, path2) == -1L;
+    }
+  }
+
+  public static String toString(InputStream in) throws IOException {
+    return new String(in.readAllBytes(), StandardCharsets.UTF_8);
+  }
+
+  /**
+   * Indicates whether Spark is currently running unit tests.
+   */
+  public static boolean isTesting() {
+    return System.getenv("SPARK_TESTING") != null || System.getProperty("spark.testing") != null;
+  }
+
+  /**
+   * The `os.name` system property.
+   */
+  public static String osName = System.getProperty("os.name");
+
+  /**
+   * The `os.version` system property.
+   */
+  public static String osVersion = System.getProperty("os.version");
+
+  /**
+   * The `java.version` system property.
+   */
+  public static String javaVersion = Runtime.version().toString();
+
+  /**
+   * The `os.arch` system property.
+   */
+  public static String osArch = System.getProperty("os.arch");
+
+  /**
+   * Whether the underlying operating system is Windows.
+   */
+  public static boolean isWindows = osName.regionMatches(true, 0, "Windows", 0, 7);
+
+  /**
+   * Whether the underlying operating system is Mac OS X.
+   */
+  public static boolean isMac = osName.regionMatches(true, 0, "Mac OS X", 0, 8);
+
+  /**
+   * Whether the underlying operating system is Mac OS X and processor is Apple Silicon.
+   */
+  public static boolean isMacOnAppleSilicon = isMac && osArch.equals("aarch64");
+
+  /**
+   * Whether the underlying operating system is Linux.
+   */
+  public static boolean isLinux = osName.regionMatches(true, 0, "Linux", 0, 5);
+
+  /**
+   * Whether the underlying operating system is UNIX.
+   */
+  public static boolean isUnix = Stream.of("AIX", "HP-UX", "Irix", "Linux", "Mac OS X", "Solaris",
+    "SunOS", "FreeBSD", "OpenBSD", "NetBSD")
+    .anyMatch(prefix -> osName.regionMatches(true, 0, prefix, 0, prefix.length()));
 }

common/utils/src/test/java/org/apache/spark/util/StructuredSparkLoggerSuite.java → common/utils-java/src/test/java/org/apache/spark/util/StructuredSparkLoggerSuite.java

@@ -24,7 +24,6 @@
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 
-import org.apache.spark.internal.Logging$;
 import org.apache.spark.internal.SparkLogger;
 import org.apache.spark.internal.SparkLoggerFactory;
 
@@ -33,13 +32,13 @@ public class StructuredSparkLoggerSuite extends SparkLoggerSuiteBase {
   // Enable Structured Logging before running the tests
   @BeforeAll
   public static void setup() {
-    Logging$.MODULE$.enableStructuredLogging();
+    SparkLoggerFactory.enableStructuredLogging();
   }
 
   // Disable Structured Logging after running the tests
   @AfterAll
   public static void teardown() {
-    Logging$.MODULE$.disableStructuredLogging();
+    SparkLoggerFactory.disableStructuredLogging();
   }
 
   private static final SparkLogger LOGGER =