Refresh HTTP cache on miss for exact-version restore (#3116)

src/NuGet.Core/NuGet.DependencyResolver.Core/ResolverUtility.cs

@@ -17,6 +17,23 @@ namespace NuGet.DependencyResolver
 {
     public static class ResolverUtility
     {
+        // Opt-out for refresh-on-miss. See https://github.com/NuGet/Home/issues/3116.
+        // When set to "false" (case-insensitive), NuGet will not refresh the HTTP cache when
+        // the cached versions list does not satisfy the requested version range.
+        private const string RefreshHttpCacheOnMissEnvVar = "NUGET_HTTP_CACHE_REFRESH_ON_MISS";
+
+        private static readonly Lazy<bool> s_refreshHttpCacheOnMissEnabled = new(static () =>
+            IsRefreshOnMissEnabled(EnvironmentVariableWrapper.Instance));
+
+        internal static bool IsRefreshHttpCacheOnMissEnabled => s_refreshHttpCacheOnMissEnabled.Value;
+
+        internal static bool IsRefreshOnMissEnabled(IEnvironmentVariableReader environmentVariableReader)
+        {
+            string? value = environmentVariableReader.GetEnvironmentVariable(RefreshHttpCacheOnMissEnvVar);
+            return !string.Equals(value, "false", StringComparison.OrdinalIgnoreCase)
+                && !string.Equals(value, "0", StringComparison.Ordinal);
+        }
+
         public static Task<GraphItem<RemoteResolveResult>> FindLibraryCachedAsync(
             LibraryRange libraryRange,
             NuGetFramework framework,
@@ -55,6 +72,8 @@ public static async Task<GraphItem<RemoteResolveResult>> FindLibraryEntryAsync(
                 LogIfPackageSourceMappingIsEnabled(libraryRange.Name, context, remoteDependencyProviders);
             }
 
+            bool httpCacheRefreshedOnMiss = false;
+
             // Try up to two times to get the package. The second
             // retry will refresh the cache if a package is listed
             // but fails to download. This can happen if the feed prunes
@@ -74,40 +93,99 @@ public static async Task<GraphItem<RemoteResolveResult>> FindLibraryEntryAsync(
                     context.Logger,
                     cancellationToken);
 
+                bool isStaleCacheMiss = match == null || IsCacheStaleForExactVersion(match, libraryRange);
+
+                if (isStaleCacheMiss
+                    && !httpCacheRefreshedOnMiss
+                    && IsRefreshHttpCacheOnMissEnabled
+                    && libraryRange.TypeConstraintAllows(LibraryDependencyTarget.Package)
+                    && libraryRange.VersionRange != null
+                    && remoteDependencyProviders.Any(p => p.IsHttp))
+                {
+                    // The cached versions list for at least one HTTP-backed source did not contain
+                    // the requested version. Refresh the HTTP cache once and retry before declaring
+                    // the package unresolved. See https://github.com/NuGet/Home/issues/3116.
+                    httpCacheRefreshedOnMiss = true;
+                    currentCacheContext = currentCacheContext.WithRefreshCacheTrue();
+
+                    context.Logger.LogMinimal(string.Format(
+                        CultureInfo.CurrentCulture,
+                        Strings.Log_RefreshingHttpCacheOnMiss,
+                        libraryRange.Name,
+                        libraryRange.VersionRange.ToString()));
+
+                    // Restart the loop so we re-query providers with a fresh cache.
+                    i = -1;
+                    continue;
+                }
+
                 if (match == null)
                 {
                     return CreateUnresolvedResult(libraryRange);
                 }
-                else
+
+                if (match.Library?.Type == LibraryType.Unresolved)
                 {
+                    // Already exhausted the refresh-on-miss path (or it wasn't applicable) and we still
+                    // have an unresolved match. Return it as unresolved rather than attempting download.
+                    return CreateUnresolvedResult(libraryRange);
+                }
 
-                    try
-                    {
-                        graphItem = await CreateGraphItemAsync(match, framework, currentCacheContext, context.Logger, cancellationToken);
-                    }
-                    catch (InvalidCacheProtocolException) when (i == 0)
-                    {
-                        // 1st failure, invalidate the cache and try again.
-                        // Clear the on disk and memory caches during the next request.
-                        currentCacheContext = currentCacheContext.WithRefreshCacheTrue();
-                    }
-                    catch (PackageNotFoundProtocolException ex) when (match.Provider.IsHttp && match.Provider.Source != null)
-                    {
-                        // 2nd failure, the feed is likely corrupt or removing packages too fast to keep up with.
-                        var message = string.Format(CultureInfo.CurrentCulture,
-                                                    Strings.Error_PackageNotFoundWhenExpected,
-                                                        match.Provider.Source,
-                                                    ex.PackageIdentity.ToString());
-                        context.Logger.LogError(message);
-
-                        throw new FatalProtocolException(message, ex);
-                    }
+                try
+                {
+                    graphItem = await CreateGraphItemAsync(match, framework, currentCacheContext, context.Logger, cancellationToken);
+                }
+                catch (InvalidCacheProtocolException) when (i == 0)
+                {
+                    // 1st failure, invalidate the cache and try again.
+                    // Clear the on disk and memory caches during the next request.
+                    currentCacheContext = currentCacheContext.WithRefreshCacheTrue();
+                }
+                catch (PackageNotFoundProtocolException ex) when (match.Provider.IsHttp && match.Provider.Source != null)
+                {
+                    // 2nd failure, the feed is likely corrupt or removing packages too fast to keep up with.
+                    var message = string.Format(CultureInfo.CurrentCulture,
+                                                Strings.Error_PackageNotFoundWhenExpected,
+                                                    match.Provider.Source,
+                                                ex.PackageIdentity.ToString());
+                    context.Logger.LogError(message);
+
+                    throw new FatalProtocolException(message, ex);
                 }
             }
 
             return graphItem!;
         }
 
+        /// <summary>
+        /// Returns <see langword="true" /> when the resolved match for an exact-version request
+        /// does not satisfy the requested minimum version, which strongly suggests the cached
+        /// versions list on at least one HTTP source is stale.
+        /// </summary>
+        /// <remarks>
+        /// Only exact (non-floating, min-inclusive) version requests qualify: those are the ones
+        /// where "the cache says the version doesn't exist" is unambiguous. Floating ranges may
+        /// be legitimately satisfied by an older cached version.
+        /// </remarks>
+        private static bool IsCacheStaleForExactVersion(RemoteMatch? match, LibraryRange libraryRange)
+        {
+            if (match?.Library?.Type != LibraryType.Unresolved)
+            {
+                return false;
+            }
+
+            var versionRange = libraryRange.VersionRange;
+            if (versionRange == null
+                || versionRange.IsFloating
+                || !versionRange.IsMinInclusive
+                || versionRange.MinVersion == null)
+            {
+                return false;
+            }
+
+            return true;
+        }
+
         private static async Task<GraphItem<RemoteResolveResult>> CreateGraphItemAsync(
             RemoteMatch match,
             NuGetFramework framework,

src/NuGet.Core/NuGet.DependencyResolver.Core/Strings.resx

@@ -120,6 +120,10 @@
   <data name="Error_PackageNotFoundWhenExpected" xml:space="preserve">
     <value>The feed '{0}' lists package '{1}' but multiple attempts to download the nupkg have failed. The feed is either invalid or required packages were removed while the current operation was in progress. Verify the package exists on the feed and try again.</value>
   </data>
+  <data name="Log_RefreshingHttpCacheOnMiss" xml:space="preserve">
+    <value>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</value>
+    <comment>{0} - Package id, {1} - version range string</comment>
+  </data>
   <data name="Log_MatchingSourceFoundForPackage" xml:space="preserve">
     <value>Package source mapping matches found for package ID '{0}' are: '{1}'.</value>
     <comment>{0} - Package id {1} - list of sources</comment>

src/NuGet.Core/NuGet.DependencyResolver.Core/xlf/Strings.cs.xlf

@@ -17,6 +17,11 @@
         <target state="translated">Pro ID balíčku {0} se nenašla shoda mapování zdroje balíčku.</target>
         <note>{0} - Package id</note>
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
     </body>
   </file>
 </xliff>

src/NuGet.Core/NuGet.DependencyResolver.Core/xlf/Strings.de.xlf

@@ -17,6 +17,11 @@
         <target state="translated">Die Paketquellzuordnungsübereinstimmung für die Paket-ID "{0}" wurde nicht gefunden.</target>
         <note>{0} - Package id</note>
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
     </body>
   </file>
 </xliff>

src/NuGet.Core/NuGet.DependencyResolver.Core/xlf/Strings.es.xlf

@@ -17,6 +17,11 @@
         <target state="translated">Las coincidencias de mapa de origen del paquete no se encuentran para el id. de paquete "{0}".</target>
         <note>{0} - Package id</note>
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
     </body>
   </file>
 </xliff>

src/NuGet.Core/NuGet.DependencyResolver.Core/xlf/Strings.fr.xlf

@@ -17,6 +17,11 @@
         <target state="translated">Une correspondance de mappage de source de paquet n'a pas été trouvée pour l'ID de paquet '{0}'.</target>
         <note>{0} - Package id</note>
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
     </body>
   </file>
 </xliff>

src/NuGet.Core/NuGet.DependencyResolver.Core/xlf/Strings.it.xlf

@@ -17,6 +17,11 @@
         <target state="translated">Corrispondenza del mapping di origine del pacchetto non trovata per l'ID pacchetto '{0}'.</target>
         <note>{0} - Package id</note>
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
     </body>
   </file>
 </xliff>

src/NuGet.Core/NuGet.DependencyResolver.Core/xlf/Strings.ja.xlf

@@ -17,6 +17,11 @@
         <target state="translated">パッケージ ID '{0}' には一致するパッケージ ソース マッピングが見つかりませんでした。</target>
         <note>{0} - Package id</note>
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
     </body>
   </file>
 </xliff>

src/NuGet.Core/NuGet.DependencyResolver.Core/xlf/Strings.ko.xlf

@@ -17,6 +17,11 @@
         <target state="translated">‘{0}’ 패키지 ID에 대한 패키지 소스 맵 일치 검색 결과가 없습니다.</target>
         <note>{0} - Package id</note>
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
     </body>
   </file>
 </xliff>

src/NuGet.Core/NuGet.DependencyResolver.Core/xlf/Strings.pl.xlf

@@ -17,6 +17,11 @@
         <target state="translated">Nie znaleziono dopasowania mapowania źródła pakietu dla identyfikatora pakietu "{0}".</target>
         <note>{0} - Package id</note>
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
     </body>
   </file>
 </xliff>

src/NuGet.Core/NuGet.DependencyResolver.Core/xlf/Strings.pt-BR.xlf

@@ -17,6 +17,11 @@
         <target state="translated">As correspondências de mapeamento de origem do pacote não foram encontradas para a ID '{0}'.</target>
         <note>{0} - Package id</note>
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
     </body>
   </file>
 </xliff>

src/NuGet.Core/NuGet.DependencyResolver.Core/xlf/Strings.ru.xlf

@@ -17,6 +17,11 @@
         <target state="translated">Соответствия для сопоставления источника пакета с идентификатором "{0}" не обнаружены.</target>
         <note>{0} - Package id</note>
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
     </body>
   </file>
 </xliff>

src/NuGet.Core/NuGet.DependencyResolver.Core/xlf/Strings.tr.xlf

@@ -17,6 +17,11 @@
         <target state="translated">Paket kimliği '{0}' için paket kaynağı eşlemesi eşleşmeleri bulunamadı.</target>
         <note>{0} - Package id</note>
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
     </body>
   </file>
 </xliff>

src/NuGet.Core/NuGet.DependencyResolver.Core/xlf/Strings.zh-Hans.xlf

@@ -17,6 +17,11 @@
         <target state="translated">找不到包 ID "{0}" 的包源映射匹配项。</target>
         <note>{0} - Package id</note>
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
     </body>
   </file>
 </xliff>

src/NuGet.Core/NuGet.DependencyResolver.Core/xlf/Strings.zh-Hant.xlf

@@ -17,6 +17,11 @@
         <target state="translated">找不到與套件識別碼 '{0}' 相符的套件來源對應。</target>
         <note>{0} - Package id</note>
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
     </body>
   </file>
 </xliff>