[AURON #2257] Avoid URI reparsing in JNI Hadoop paths

auron-core/pom.xml

@@ -77,6 +77,12 @@
       <scope>provided</scope>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client-runtime</artifactId>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>org.junit.jupiter</groupId>
       <artifactId>junit-jupiter-api</artifactId>

auron-core/src/main/java/org/apache/auron/jni/JniBridge.java

@@ -19,7 +19,6 @@
 import java.io.IOException;
 import java.lang.management.BufferPoolMXBean;
 import java.lang.management.ManagementFactory;
-import java.net.URI;
 import java.nio.ByteBuffer;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
@@ -71,12 +70,11 @@ public static void putResource(String key, Object value) {
     }
 
     public static FSDataInputWrapper openFileAsDataInputWrapper(FileSystem fs, String path) throws Exception {
-        // the path is a URI string, so we need to convert it to a URI object
-        return FSDataInputWrapper.wrap(fs.open(new Path(new URI(path))));
+        return FSDataInputWrapper.wrap(fs.open(new Path(path)));
     }
 
     public static FSDataOutputWrapper createFileAsDataOutputWrapper(FileSystem fs, String path) throws Exception {
-        return FSDataOutputWrapper.wrap(fs.create(new Path(new URI(path))));
+        return FSDataOutputWrapper.wrap(fs.create(new Path(path)));
     }
 
     public static long getDirectMemoryUsed() {

auron-core/src/test/java/org/apache/auron/jni/JniBridgeTest.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.auron.jni;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.jupiter.api.Test;
+
+public class JniBridgeTest {
+
+    @Test
+    public void testFileWrappersPreserveLiteralHashInHdfsPath() throws Exception {
+        String path = "hdfs://mycluster/tmp/channel=wx_repro#mini/spark-submit-123/part-00000.json";
+        CapturingFileSystem cfs = new CapturingFileSystem();
+
+        JniBridge.openFileAsDataInputWrapper(cfs, path).close();
+        JniBridge.createFileAsDataOutputWrapper(cfs, path).close();
+
+        assertPathPreservesHash(cfs.openedPath);
+        assertPathPreservesHash(cfs.createdPath);
+    }
+
+    @Test
+    public void testFileWrappersPreserveNormalizedPercentPathStrings() throws Exception {
+        String path = "file:/tmp/t1/part=test%20test/part-00000.parquet";
+        CapturingFileSystem cfs = new CapturingFileSystem();
+
+        JniBridge.openFileAsDataInputWrapper(cfs, path).close();
+        JniBridge.createFileAsDataOutputWrapper(cfs, path).close();
+
+        assertEquals(
+                "/tmp/t1/part=test%20test/part-00000.parquet",
+                cfs.openedPath.toUri().getPath());
+        assertEquals(
+                "/tmp/t1/part=test%20test/part-00000.parquet",
+                cfs.createdPath.toUri().getPath());
+    }
+
+    private static void assertPathPreservesHash(Path path) {
+        assertEquals(
+                "/tmp/channel=wx_repro#mini/spark-submit-123/part-00000.json",
+                path.toUri().getPath());
+        assertNull(path.toUri().getFragment());
+    }
+
+    private static class CapturingFileSystem extends RawLocalFileSystem {
+        private final Statistics statistics = new Statistics("hdfs");
+        private Path openedPath;
+        private Path createdPath;
+
+        @Override
+        public FSDataInputStream open(Path path) throws IOException {
+            openedPath = path;
+            return new FSDataInputStream(new EmptyFSInputStream());
+        }
+
+        @Override
+        public FSDataOutputStream create(Path path) throws IOException {
+            createdPath = path;
+            return new FSDataOutputStream(new ByteArrayOutputStream(), statistics);
+        }
+    }
+
+    private static class EmptyFSInputStream extends FSInputStream {
+        @Override
+        public void seek(long pos) {}
+
+        @Override
+        public long getPos() {
+            return 0;
+        }
+
+        @Override
+        public boolean seekToNewSource(long targetPos) {
+            return false;
+        }
+
+        @Override
+        public int read() {
+            return -1;
+        }
+    }
+}

spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/ShimsImpl.scala

@@ -971,6 +971,10 @@ class ShimsImpl extends Shims with Logging {
       size: Long): PartitionedFile =
     PartitionedFile(partitionValues, filePath, offset, size)
 
+  @sparkver("3.0 / 3.1 / 3.2 / 3.3")
+  override def getPartitionedFilePathString(file: PartitionedFile): String =
+    file.filePath
+
   @sparkver("3.4 / 3.5 / 4.0 / 4.1")
   override def getPartitionedFile(
       partitionValues: InternalRow,
@@ -982,6 +986,10 @@ class ShimsImpl extends Shims with Logging {
     PartitionedFile(partitionValues, SparkPath.fromPath(new Path(filePath)), offset, size)
   }
 
+  @sparkver("3.4 / 3.5 / 4.0 / 4.1")
+  override def getPartitionedFilePathString(file: PartitionedFile): String =
+    file.toPath.toString
+
   @sparkver("3.1 / 3.2 / 3.3 / 3.4 / 3.5 / 4.0 / 4.1")
   override def getMinPartitionNum(sparkSession: SparkSession): Int =
     sparkSession.sessionState.conf.filesMinPartitionNum

spark-extension/src/main/scala/org/apache/spark/sql/auron/Shims.scala

@@ -272,6 +272,8 @@ abstract class Shims {
       offset: Long,
       size: Long): PartitionedFile
 
+  def getPartitionedFilePathString(file: PartitionedFile): String
+
   def getMinPartitionNum(sparkSession: SparkSession): Int
 
   @nowarn("cat=unused") // Some params temporarily unused

spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeFileSourceScanBase.scala

@@ -114,7 +114,7 @@ abstract class NativeFileSourceScanBase(basedFileScan: FileSourceScanExec)
       }
       pb.PartitionedFile
         .newBuilder()
-        .setPath(s"${file.filePath}")
+        .setPath(Shims.get.getPartitionedFilePathString(file))
         .setSize(fileSizes(file.filePath))
         .addAllPartitionValues(nativePartitionValues.asJava)
         .setLastModifiedNs(0)