Add refresh flag to force refresh of the Registry state

README.md

@@ -1142,6 +1142,16 @@ The transfer procedure is automated by Spago commands, and goes as follows:
 The `-v` flag will print out all the `purs` commands that `spago` invokes during its operations,
 plus a lot of diagnostic info, so you might want to use it to troubleshoot weird behaviours and/or crashes.
 
+### Force a registry refresh
+
+Spago caches the Registry Index for 15 minutes. If you need a package that was just published, you can force a refresh:
+
+```console
+$ spago install --refresh some-new-package
+```
+
+The `--refresh` flag works with any command that accesses the Registry (`install`, `build`, `fetch`, etc.).
+
 ### Install autocompletions for `bash`
 
 You can just add this to your `.bashrc`:

bin/src/Flags.purs

@@ -135,15 +135,30 @@ noColor =
         <> O.help "Force logging without ANSI color escape sequences"
     )
 
-offline :: Parser OnlineStatus
-offline =
+offlineFlag :: Parser OnlineStatus
+offlineFlag =
   O.flag
     Online
     Offline
     ( O.long "offline"
         <> O.help "Do not attempt to use the network. Warning: this will fail if you don't have the necessary dependencies already cached"
     )
 
+refreshFlag :: Parser Boolean
+refreshFlag =
+  O.switch
+    ( O.long "refresh"
+        <> O.help "Force refresh of the Registry, bypassing the cache"
+    )
+
+offline :: Parser OnlineStatus
+offline = combineOfflineRefresh <$> offlineFlag <*> refreshFlag
+  where
+  combineOfflineRefresh offlineStatus refresh = case offlineStatus, refresh of
+    Offline, _ -> Offline
+    _, true -> OnlineRefreshRegistry
+    _, false -> offlineStatus
+
 json :: Parser Boolean
 json =
   O.switch

src/Spago/Prelude.purs

@@ -46,7 +46,18 @@ import Spago.Path as Path
 import Spago.Paths as Paths
 import Unsafe.Coerce (unsafeCoerce)
 
-data OnlineStatus = Offline | Online | OnlineBypassCache
+-- | Controls network/caching behavior for registry operations.
+-- | - `Offline`: Never use network, fail if data not cached locally
+-- | - `Online`: Normal mode with 15-minute cache for registry data
+-- | - `OnlineRefreshRegistry`: Force refresh of registry repos (bypasses 15-min staleness
+-- |   check), but AVar cache still works normally. Used for `--refresh` flag.
+-- | - `OnlineBypassCache`: Bypass ALL caches (process AVar + DB). See usage
+-- |   in `Spago.Command.Registry.transfer` for details on when this is appropriate.
+data OnlineStatus
+  = Offline
+  | Online
+  | OnlineRefreshRegistry
+  | OnlineBypassCache
 
 derive instance Eq OnlineStatus
 

src/Spago/Registry.purs

@@ -172,9 +172,9 @@ getRegistryFns registryBox registryLock = do
     -- we keep track of how old the latest pull was - if the last pull was recent enough
     -- we just move on, otherwise run the fibers
     { db, offline } <- ask
-    fetchingFreshRegistry <- shouldFetchRegistryRepos db
+    fetchingFreshRegistry <- shouldFetchRegistryRepos offline db
     -- we also check if we need to bypass this cache (for when we need the freshest data)
-    when (fetchingFreshRegistry || offline == OnlineBypassCache) do
+    when (fetchingFreshRegistry || offline == OnlineBypassCache || offline == OnlineRefreshRegistry) do
       -- clone the registry and index repo, or update them
       logInfo "Refreshing the Registry Index..."
       parallelise
@@ -264,7 +264,7 @@ getMetadataImpl db onlineStatus name =
 -- Parallelised version of `getMetadataImpl`
 getMetadataForPackagesImpl :: Db -> OnlineStatus -> Array PackageName -> Spago (LogEnv ()) (Either String (Map PackageName Metadata))
 getMetadataForPackagesImpl db onlineStatus names = do
-  (map Map.fromFoldable <<< sequence) <$> case onlineStatus == OnlineBypassCache of
+  (map Map.fromFoldable <<< sequence) <$> case onlineStatus == OnlineBypassCache || onlineStatus == OnlineRefreshRegistry of
     true -> do
       logDebug "Bypassing cache, reading metadata from file"
       parTraverseSpago metadataFromFile names
@@ -367,31 +367,37 @@ isVersionCompatible installedVersion minVersion =
       [ a, b, _c ], [ x, y, _z ] | a /= 0 && a == x && b >= y -> true
       _, _ -> false
 
--- | Check if we have fetched the registry recently enough, so we don't hit the net all the time
-shouldFetchRegistryRepos :: ∀ a. Db -> Spago (LogEnv a) Boolean
-shouldFetchRegistryRepos db = do
+-- | Check if we have fetched the registry recently enough, so we don't hit the net all the time.
+-- | When `OnlineRefreshRegistry` is set, always fetch regardless of staleness.
+shouldFetchRegistryRepos :: ∀ a. OnlineStatus -> Db -> Spago (LogEnv a) Boolean
+shouldFetchRegistryRepos offline db = do
   now <- liftEffect $ Now.nowDateTime
   let registryKey = "registry"
-  maybeLastRegistryFetch <- liftEffect $ Db.getLastPull db registryKey
-  case maybeLastRegistryFetch of
-    -- No record, so we have to fetch
-    Nothing -> do
-      logDebug "No record of last registry pull, will fetch"
-      liftEffect $ Db.updateLastPull db registryKey now
-      pure true
-    -- We have a record, so we check if it's old enough
-    Just lastRegistryFetch -> do
-      let staleAfter = Minutes 15.0
-      let (timeDiff :: Minutes) = DateTime.diff now lastRegistryFetch
-      let isOldEnough = timeDiff > staleAfter
-      -- We check if it's old, but also if we have it at all
-      registryExists <- FS.exists Paths.registryPath
-      if isOldEnough || not registryExists then do
-        logDebug "Registry is old, refreshing"
+  if offline == OnlineRefreshRegistry then do
+    logDebug "Refresh flag set, will fetch registry"
+    liftEffect $ Db.updateLastPull db registryKey now
+    pure true
+  else do
+    maybeLastRegistryFetch <- liftEffect $ Db.getLastPull db registryKey
+    case maybeLastRegistryFetch of
+      -- No record, so we have to fetch
+      Nothing -> do
+        logDebug "No record of last registry pull, will fetch"
         liftEffect $ Db.updateLastPull db registryKey now
         pure true
-      else do
-        pure false
+      -- We have a record, so we check if it's old enough
+      Just lastRegistryFetch -> do
+        let staleAfter = Minutes 15.0
+        let (timeDiff :: Minutes) = DateTime.diff now lastRegistryFetch
+        let isOldEnough = timeDiff > staleAfter
+        -- We check if it's old, but also if we have it at all
+        registryExists <- FS.exists Paths.registryPath
+        if isOldEnough || not registryExists then do
+          logDebug "Registry is old, refreshing"
+          liftEffect $ Db.updateLastPull db registryKey now
+          pure true
+        else do
+          pure false
 
 --------------------------------------------------------------------------------
 -- | Registry operations

test-fixtures/1146-cli-help/build.txt

@@ -1,6 +1,6 @@
 Invalid option `--bogus'
 
-Usage: index.dev.js build [--migrate] [--monochrome|--no-color] [--offline] [-q|--quiet] [-v|--verbose] [--backend-args ARGS] [--ensure-ranges] [--json-errors] [--output DIR] [--pedantic-packages] [--pure] [--purs-args ARGS] [-p|--package PACKAGE] ([--verbose-stats] | [--censor-stats]) [--strict]
+Usage: index.dev.js build [--migrate] [--monochrome|--no-color] [--offline] [--refresh] [-q|--quiet] [-v|--verbose] [--backend-args ARGS] [--ensure-ranges] [--json-errors] [--output DIR] [--pedantic-packages] [--pure] [--purs-args ARGS] [-p|--package PACKAGE] ([--verbose-stats] | [--censor-stats]) [--strict]
   Compile the project
 
 Available options:
@@ -9,6 +9,7 @@ Available options:
   --offline                Do not attempt to use the network. Warning: this will
                            fail if you don't have the necessary dependencies
                            already cached
+  --refresh                Force refresh of the Registry, bypassing the cache
   -q,--quiet               Suppress all spago logging
   -v,--verbose             Enable additional debug logging, e.g. printing `purs`
                            commands

test-fixtures/1146-cli-help/registry-search.txt

@@ -1,6 +1,6 @@
 Invalid option `--bogus'
 
-Usage: index.dev.js registry search [--migrate] [--monochrome|--no-color] [--offline] [-q|--quiet] [-v|--verbose] [--json] PACKAGE
+Usage: index.dev.js registry search [--migrate] [--monochrome|--no-color] [--offline] [--refresh] [-q|--quiet] [-v|--verbose] [--json] PACKAGE
   Search for package names in the Registry
 
 Available options:
@@ -9,6 +9,7 @@ Available options:
   --offline                Do not attempt to use the network. Warning: this will
                            fail if you don't have the necessary dependencies
                            already cached
+  --refresh                Force refresh of the Registry, bypassing the cache
   -q,--quiet               Suppress all spago logging
   -v,--verbose             Enable additional debug logging, e.g. printing `purs`
                            commands

test-fixtures/init/subpackage/conflicting-flags.txt

@@ -1,6 +1,6 @@
 Invalid option `--subpackage'
 
-Usage: index.dev.js init [--migrate] [--monochrome|--no-color] [--offline] [-q|--quiet] [-v|--verbose] ([--subpackage ARG] | [--name ARG]) [--package-set ARG] [--use-solver]
+Usage: index.dev.js init [--migrate] [--monochrome|--no-color] [--offline] [--refresh] [-q|--quiet] [-v|--verbose] ([--subpackage ARG] | [--name ARG]) [--package-set ARG] [--use-solver]
   Initialise a new project
 
 Available options:
@@ -9,6 +9,7 @@ Available options:
   --offline                Do not attempt to use the network. Warning: this will
                            fail if you don't have the necessary dependencies
                            already cached
+  --refresh                Force refresh of the Registry, bypassing the cache
   -q,--quiet               Suppress all spago logging
   -v,--verbose             Enable additional debug logging, e.g. printing `purs`
                            commands

test/Spago/Install.purs

@@ -138,6 +138,19 @@ spec = Spec.around withTempDir do
       writeConfigWithEither testCwd
       spago [ "install", "--offline", "either" ] >>= shouldBeFailureErr (fixture "offline.txt")
 
+    Spec.it "refresh flag forces registry refresh and bypasses DB metadata cache" \{ spago, testCwd } -> do
+      spago [ "init" ] >>= shouldBeSuccess
+      -- The --refresh flag should force a registry refresh regardless of cache age
+      result1 <- spago [ "install", "--refresh" ]
+      shouldBeSuccess result1
+      either _.stderr _.stderr result1 `shouldContain` "Refreshing the Registry Index..."
+      -- Remove lockfile to force dependency resolution on next install
+      FS.unlink (testCwd </> "spago.lock")
+      -- With --refresh, should also bypass DB metadata cache and read from files
+      result2 <- spago [ "install", "-v", "--refresh", "effect" ]
+      shouldBeSuccess result2
+      either _.stderr _.stderr result2 `shouldContain` "Bypassing cache, reading metadata from file"
+
     Spec.it "skips cloning during resolution when git package has declared deps" \{ spago, testCwd, fixture } -> do
       FS.copyFile { src: fixture "git-declared-deps/with-declared-deps.yaml", dst: testCwd </> "spago.yaml" }
       FS.mkdirp (testCwd </> "src")