Fix LeanDataWriter Compression method so it supports unicode filenames

Compression/Compression.cs

@@ -24,7 +24,6 @@
 using ICSharpCode.SharpZipLib.Core;
 using ICSharpCode.SharpZipLib.GZip;
 using ICSharpCode.SharpZipLib.Tar;
-using Ionic.Zip;
 using QuantConnect.Logging;
 using ZipEntry = ICSharpCode.SharpZipLib.Zip.ZipEntry;
 using ZipFile = Ionic.Zip.ZipFile;
@@ -177,26 +176,7 @@ public static bool ZipData(string zipPath, string zipEntry, IEnumerable<string>
         /// <returns>True on success</returns>
         public static bool ZipCreateAppendData(string path, string entry, string data, bool overrideEntry = false)
         {
-            try
-            {
-                using (var zip = File.Exists(path) ? ZipFile.Read(path) : new ZipFile(path))
-                {
-                    if (zip.ContainsEntry(entry) && overrideEntry)
-                    {
-                        zip.RemoveEntry(entry);
-                    }
-
-                    zip.AddEntry(entry, data);
-                    zip.UseZip64WhenSaving = Zip64Option.Always;
-                    zip.Save();
-                }
-            }
-            catch (Exception err)
-            {
-                Log.Error(err);
-                return false;
-            }
-            return true;
+            return ZipCreateAppendData(path, entry, Encoding.UTF8.GetBytes(data), overrideEntry);
         }
 
         /// <summary>
@@ -208,20 +188,39 @@ public static bool ZipCreateAppendData(string path, string entry, string data, b
         /// <param name="overrideEntry">True if should override entry if it already exists</param>
         /// <returns>True on success</returns>
         public static bool ZipCreateAppendData(string path, string entry, byte[] data, bool overrideEntry = false)
+        {
+            return ZipCreateAppendData(path, entry, s => s.Write(data, 0, data.Length), overrideEntry);
+        }
+
+        /// <summary>
+        /// Append the zip data to the file-entry specified.
+        /// </summary>
+        /// <param name="path">The zip file path</param>
+        /// <param name="entry">The entry name</param>
+        /// <param name="write">Write data callback</param>
+        /// <param name="overrideEntry">True if should override entry if it already exists</param>
+        /// <returns>True on success</returns>
+        private static bool ZipCreateAppendData(string path, string entry, Action<Stream> write, bool overrideEntry = false)
         {
             try
             {
-                using (var zip = File.Exists(path) ? ZipFile.Read(path) : new ZipFile(path))
+                using var fs = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
+                using var archive = new ZipArchive(fs, ZipArchiveMode.Update, leaveOpen: false);
+
+                var existing = archive.GetEntry(entry);
+                if (existing != null)
                 {
-                    if (overrideEntry && zip.ContainsEntry(entry))
+                    if (!overrideEntry)
                     {
-                        zip.RemoveEntry(entry);
+                        return false;
                     }
-
-                    zip.AddEntry(entry, data);
-                    zip.UseZip64WhenSaving = Zip64Option.Always;
-                    zip.Save();
+                    existing.Delete();
                 }
+
+                var zipEntry = archive.CreateEntry(entry, CompressionLevel.Optimal);
+
+                using var entryStream = zipEntry.Open();
+                write(entryStream);
             }
             catch (Exception err)
             {

Tests/ToolBox/LeanDataWriterTests.cs

@@ -233,6 +233,25 @@ public void LeanDataWriter_CanWriteEquity()
             Assert.AreEqual(data.First().Value.Count, 3);
         }
 
+        [Test]
+        public void LeanDataWriter_CanSupportUtf8Chars()
+        {
+            var symbol = Symbol.Create("币安人生usdt", SecurityType.CryptoFuture, Market.Binance);
+            var filePath = LeanData.GenerateZipFilePath(_dataDirectory, symbol, _date, Resolution.Tick, TickType.Trade);
+
+            var leanDataWriter = new LeanDataWriter(Resolution.Tick, symbol, _dataDirectory);
+            leanDataWriter.Write(GetTicks(symbol));
+
+            Assert.IsTrue(File.Exists(filePath));
+            Assert.IsFalse(File.Exists(filePath + ".tmp"));
+
+            var data = QuantConnect.Compression.Unzip(filePath);
+
+            var entry = data.First();
+            Assert.AreEqual(entry.Key, "20170316_币安人生usdt_tick_trade_perp.csv");
+            Assert.AreEqual(entry.Value.Count, 3);
+        }
+
         [TestCase("CON")]
         [TestCase("PRN")]
         [TestCase("AUX")]