feat: replace custom shuffle block format with Arrow IPC streams

common/src/main/scala/org/apache/comet/CometConf.scala

@@ -434,11 +434,11 @@ object CometConf extends ShimCometConf {
     conf(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.compression.codec")
       .category(CATEGORY_SHUFFLE)
       .doc(
-        "The codec of Comet native shuffle used to compress shuffle data. lz4, zstd, and " +
-          "snappy are supported. Compression can be disabled by setting " +
+        "The codec of Comet native shuffle used to compress shuffle data. " +
+          "Supported codecs: lz4, zstd. Compression can be disabled by setting " +
           "spark.shuffle.compress=false.")
       .stringConf
-      .checkValues(Set("zstd", "lz4", "snappy"))
+      .checkValues(Set("zstd", "lz4"))
       .createWithDefault("lz4")
 
   val COMET_EXEC_SHUFFLE_COMPRESSION_ZSTD_LEVEL: ConfigEntry[Int] =
@@ -528,11 +528,10 @@ object CometConf extends ShimCometConf {
       .category(CATEGORY_SHUFFLE)
       .doc("Size of the write buffer in bytes used by the native shuffle writer when writing " +
         "shuffle data to disk. Larger values may improve write performance by reducing " +
-        "the number of system calls, but will use more memory. " +
-        "The default is 1MB which provides a good balance between performance and memory usage.")
-      .bytesConf(ByteUnit.MiB)
+        "the number of system calls, but will use more memory.")
+      .bytesConf(ByteUnit.KiB)
       .checkValue(v => v > 0, "Write buffer size must be positive")
-      .createWithDefault(1)
+      .createWithDefault(8)
 
   val COMET_SHUFFLE_PREFER_DICTIONARY_RATIO: ConfigEntry[Double] = conf(
     "spark.comet.shuffle.preferDictionary.ratio")