Parallelize file uploads in fs cp command.

NEXT_CHANGELOG.md

@@ -6,6 +6,9 @@
 
 ### CLI
 
+* Improve performance of `databricks fs cp` command by parallelizing file uploads when
+  copying directories with the `--recursive` flag.
+
 ### Bundles
 
 ### Dependency updates

acceptance/cmd/fs/cp/dir-to-dir/localdir/file1.txt

@@ -0,0 +1 @@
+file1 content

acceptance/cmd/fs/cp/dir-to-dir/localdir/file2.txt

@@ -0,0 +1 @@
+file2 content

acceptance/cmd/fs/cp/dir-to-dir/output.txt

@@ -0,0 +1,10 @@
+
+>>> [CLI] fs cp -r localdir dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/uploaded-dir
+localdir/file1.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/uploaded-dir/file1.txt
+localdir/file2.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/uploaded-dir/file2.txt
+
+>>> [CLI] fs cat dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/uploaded-dir/file1.txt
+file1 content
+
+>>> [CLI] fs cat dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/uploaded-dir/file2.txt
+file2 content

acceptance/cmd/fs/cp/dir-to-dir/script

@@ -0,0 +1,21 @@
+CATALOG_NAME="main"
+SCHEMA_NAME="default"
+VOLUME_NAME="fs-cp-test-${UNIQUE_NAME}"
+
+cleanup() {
+  $CLI volumes delete "${CATALOG_NAME}.${SCHEMA_NAME}.${VOLUME_NAME}" 2>/dev/null || true
+}
+trap cleanup EXIT
+
+# Create volume for testing.
+$CLI volumes create "${CATALOG_NAME}" "${SCHEMA_NAME}" "${VOLUME_NAME}" MANAGED >/dev/null
+
+# Create parent directory.
+$CLI fs mkdir dbfs:/Volumes/${CATALOG_NAME}/${SCHEMA_NAME}/${VOLUME_NAME}
+
+# Recursive directory copy (output sorted for deterministic ordering).
+trace $CLI fs cp -r localdir dbfs:/Volumes/${CATALOG_NAME}/${SCHEMA_NAME}/${VOLUME_NAME}/uploaded-dir 2>&1 | sort
+
+# Verify files were uploaded correctly.
+trace $CLI fs cat dbfs:/Volumes/${CATALOG_NAME}/${SCHEMA_NAME}/${VOLUME_NAME}/uploaded-dir/file1.txt
+trace $CLI fs cat dbfs:/Volumes/${CATALOG_NAME}/${SCHEMA_NAME}/${VOLUME_NAME}/uploaded-dir/file2.txt

acceptance/cmd/fs/cp/dir-to-dir/test.toml

@@ -0,0 +1,3 @@
+Local = true
+Cloud = true
+RequiresUnityCatalog = true

acceptance/cmd/fs/cp/file-to-dir/local.txt

@@ -0,0 +1 @@
+hello world!

acceptance/cmd/fs/cp/file-to-dir/output.txt

@@ -0,0 +1,6 @@
+
+>>> [CLI] fs cp local.txt dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/mydir/
+local.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/mydir/local.txt
+
+>>> [CLI] fs cat dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/mydir/local.txt
+hello world!

acceptance/cmd/fs/cp/file-to-dir/script

@@ -0,0 +1,20 @@
+CATALOG_NAME="main"
+SCHEMA_NAME="default"
+VOLUME_NAME="fs-cp-test-${UNIQUE_NAME}"
+
+cleanup() {
+  $CLI volumes delete "${CATALOG_NAME}.${SCHEMA_NAME}.${VOLUME_NAME}" 2>/dev/null || true
+}
+trap cleanup EXIT
+
+# Create volume for testing.
+$CLI volumes create "${CATALOG_NAME}" "${SCHEMA_NAME}" "${VOLUME_NAME}" MANAGED >/dev/null
+
+# Create target directory.
+$CLI fs mkdir dbfs:/Volumes/${CATALOG_NAME}/${SCHEMA_NAME}/${VOLUME_NAME}/mydir
+
+# Copy file into a directory (trailing slash indicates directory target).
+trace $CLI fs cp local.txt dbfs:/Volumes/${CATALOG_NAME}/${SCHEMA_NAME}/${VOLUME_NAME}/mydir/
+
+# Verify file was uploaded correctly.
+trace $CLI fs cat dbfs:/Volumes/${CATALOG_NAME}/${SCHEMA_NAME}/${VOLUME_NAME}/mydir/local.txt

acceptance/cmd/fs/cp/file-to-dir/test.toml

@@ -0,0 +1,3 @@
+Local = true
+Cloud = true
+RequiresUnityCatalog = true

acceptance/cmd/fs/cp/file-to-file/local.txt

@@ -0,0 +1 @@
+hello world!

acceptance/cmd/fs/cp/file-to-file/output.txt

@@ -0,0 +1,12 @@
+
+>>> [CLI] fs cp local.txt dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/uploaded.txt
+local.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/uploaded.txt
+
+>>> [CLI] fs cat dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/uploaded.txt
+hello world!
+
+>>> [CLI] fs cp dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/uploaded.txt downloaded.txt
+dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/uploaded.txt -> downloaded.txt
+
+>>> cat downloaded.txt
+hello world!

acceptance/cmd/fs/cp/file-to-file/script

@@ -0,0 +1,27 @@
+CATALOG_NAME="main"
+SCHEMA_NAME="default"
+VOLUME_NAME="fs-cp-test-${UNIQUE_NAME}"
+
+cleanup() {
+  $CLI volumes delete "${CATALOG_NAME}.${SCHEMA_NAME}.${VOLUME_NAME}" 2>/dev/null || true
+}
+trap cleanup EXIT
+
+# Create volume for testing.
+$CLI volumes create "${CATALOG_NAME}" "${SCHEMA_NAME}" "${VOLUME_NAME}" MANAGED >/dev/null
+
+# Create parent directory.
+$CLI fs mkdir dbfs:/Volumes/${CATALOG_NAME}/${SCHEMA_NAME}/${VOLUME_NAME}
+
+# Upload local file to volume.
+trace $CLI fs cp local.txt dbfs:/Volumes/${CATALOG_NAME}/${SCHEMA_NAME}/${VOLUME_NAME}/uploaded.txt
+
+# Verify file was uploaded correctly.
+trace $CLI fs cat dbfs:/Volumes/${CATALOG_NAME}/${SCHEMA_NAME}/${VOLUME_NAME}/uploaded.txt
+
+# Download the same file back to verify round-trip.
+trace $CLI fs cp dbfs:/Volumes/${CATALOG_NAME}/${SCHEMA_NAME}/${VOLUME_NAME}/uploaded.txt downloaded.txt
+
+# Verify downloaded content matches original.
+trace cat downloaded.txt
+rm downloaded.txt

acceptance/cmd/fs/cp/file-to-file/test.toml

@@ -0,0 +1,3 @@
+Local = true
+Cloud = true
+RequiresUnityCatalog = true

acceptance/cmd/fs/cp/input-validation/output.txt

@@ -0,0 +1,10 @@
+
+>>> errcode [CLI] fs cp src dst --concurrency -1
+Error: --concurrency must be at least 1
+
+Exit code: 1
+
+>>> errcode [CLI] fs cp src dst --concurrency 0
+Error: --concurrency must be at least 1
+
+Exit code: 1

acceptance/cmd/fs/cp/input-validation/script

@@ -0,0 +1,3 @@
+# Invalid concurrency values should fail.
+trace errcode $CLI fs cp src dst --concurrency -1
+trace errcode $CLI fs cp src dst --concurrency 0

acceptance/cmd/fs/cp/input-validation/test.toml

@@ -0,0 +1,2 @@
+Local = true
+Cloud = false