fix: Retry after partial file reads.

Author: kinyoklion

pkgs/sdk/server/src/Internal/DataSources/FileDataSource.cs

@@ -28,6 +28,10 @@ internal sealed class FileDataSource : IDataSource
         private volatile int _lastVersion;
         private object _updateLock = new object();
 
+        private const int MaxRetries = 5;
+        private readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(0.6);
+        private readonly Dictionary<string, int> _retryCounts = new Dictionary<string, int>();
+
         public FileDataSource(IDataSourceUpdates dataSourceUpdates, FileDataTypes.IFileReader fileReader,
             List<string> paths, bool autoUpdate, Func<string, object> alternateParser, bool skipMissingPaths,
             FileDataTypes.DuplicateKeysHandling duplicateKeysHandling,
@@ -94,6 +98,7 @@ private void LoadAll()
                 var version = Interlocked.Increment(ref _lastVersion);
                 var flags = new Dictionary<string, ItemDescriptor>();
                 var segments = new Dictionary<string, ItemDescriptor>();
+                var shouldRetry = false;
                 foreach (var path in _paths)
                 {
                     try
@@ -102,17 +107,56 @@ private void LoadAll()
                         _logger.Debug("file data: {0}", content);
                         var data = _parser.Parse(content, version);
                         _dataMerger.AddToData(data, flags, segments);
+                        // Remove any retry count associated with this path.
+                        _retryCounts.Remove(path);
                     }
                     catch (FileNotFoundException) when (_skipMissingPaths)
                     {
                         _logger.Debug("{0}: {1}", path, "File not found");
                     }
+                    catch (System.Text.Json.JsonException)
+                    {
+                        // We may have received the notification of a file change while the file was being written.
+                        // So we may read an empty or partially written file. So, when we encounter a JSON parsing issue
+                        // we will retry after a short delay.
+                        // We will retry up to MaxRetries times before giving up.
+                        if (!_retryCounts.ContainsKey(path))
+                        {
+                            _retryCounts[path] = 0;
+                        }
+                        _retryCounts[path]++;
+
+                        if (_retryCounts[path] < MaxRetries)
+                        {
+                            shouldRetry = true;
+                            _logger.Warn("{0}: {1}", path, "Failed to parse file, retrying in " + RetryDelay.TotalMilliseconds + " milliseconds");
+                        }
+                        else
+                        {
+                            _logger.Error("{0}: {1}", path, "Failed to parse file after " + MaxRetries + " retries");
+                        }
+
+                        break;
+                    }
                     catch (Exception e)
                     {
                         LogHelpers.LogException(_logger, "Failed to load " + path, e);
                         return;
                     }
                 }
+                
+                if (shouldRetry)
+                {
+                    Task.Run(async () =>
+                    {
+                        await Task.Delay(RetryDelay);
+                        LoadAll();
+                    });
+                    return;
+                }
+                // If any files failed to load, from anything other than not existing, then that
+                // update would fail. This behavior is retained with the addition of the retry. But it should be
+                // examined.
 
                 var allData = new FullDataSet<ItemDescriptor>(
                     ImmutableDictionary.Create<DataKind, KeyedItems<ItemDescriptor>>()