SOLR-18083: Fix replication and other general operations in readOnly mode

changelog/unreleased/solr-18083-fix-read-only-behavior.yml

@@ -0,0 +1,9 @@
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+title: Fix normal operation errors for readOnly collections
+type: other # added, changed, fixed, deprecated, removed, dependency_update, security, other
+authors:
+  - name: Houston Putman
+    nick: HoustonPutman
+links:
+  - name: SOLR-18083
+    url: https://issues.apache.org/jira/browse/SOLR-18083

solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java

@@ -303,6 +303,8 @@ private void commitOnLeader(String leaderBaseUrl, String coreName)
       // ureq.getParams().set(UpdateParams.OPEN_SEARCHER, onlyLeaderIndexes);
       // Why do we need to open searcher if "onlyLeaderIndexes"?
       ureq.getParams().set(UpdateParams.OPEN_SEARCHER, false);
+      // If the leader is readOnly, do not fail since the core is already committed.
+      ureq.getParams().set(UpdateParams.FAIL_ON_READ_ONLY, false);
       ureq.setAction(AbstractUpdateRequest.ACTION.COMMIT, false, true).process(client);
     }
   }
@@ -657,15 +659,17 @@ public final void doSyncOrReplicateRecovery(SolrCore core) throws Exception {
           break;
         }
 
-        // we wait a bit so that any updates on the leader
-        // that started before they saw recovering state
-        // are sure to have finished (see SOLR-7141 for
-        // discussion around current value)
-        // TODO since SOLR-11216, we probably won't need this
-        try {
-          Thread.sleep(waitForUpdatesWithStaleStatePauseMilliSeconds);
-        } catch (InterruptedException e) {
-          Thread.currentThread().interrupt();
+        if (!core.readOnly) {
+          // we wait a bit so that any updates on the leader
+          // that started before they saw recovering state
+          // are sure to have finished (see SOLR-7141 for
+          // discussion around current value)
+          // TODO since SOLR-11216, we probably won't need this
+          try {
+            Thread.sleep(waitForUpdatesWithStaleStatePauseMilliSeconds);
+          } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+          }
         }
 
         // first thing we just try to sync

solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java

@@ -195,7 +195,7 @@ void runLeaderProcess(boolean weAreReplacement) throws KeeperException, Interrup
         // first cancel any current recovery
         core.getUpdateHandler().getSolrCoreState().cancelRecovery();
 
-        if (weAreReplacement) {
+        if (weAreReplacement && !core.readOnly) {
           // wait a moment for any floating updates to finish
           try {
             Thread.sleep(2500);

solr/core/src/java/org/apache/solr/core/SolrCore.java

@@ -2476,7 +2476,7 @@ public RefCounted<SolrIndexSearcher> openNewSearcher(
                   true,
                   directoryFactory);
         } else {
-          RefCounted<IndexWriter> writer = getSolrCoreState().getIndexWriter(this);
+          RefCounted<IndexWriter> writer = getSolrCoreState().getIndexWriter(this, true);
           DirectoryReader newReader = null;
           try {
             newReader = indexReaderFactory.newReader(writer.get(), this);

solr/core/src/java/org/apache/solr/handler/IndexFetcher.java

@@ -530,14 +530,14 @@ IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean forceCoreRel
           // we just clear ours and commit
           log.info("New index in Leader. Deleting mine...");
           RefCounted<IndexWriter> iw =
-              solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(solrCore);
+              solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(solrCore, true);
           try {
             iw.get().deleteAll();
           } finally {
             iw.decref();
           }
           assert TestInjection.injectDelayBeforeFollowerCommitRefresh();
-          if (skipCommitOnLeaderVersionZero) {
+          if (skipCommitOnLeaderVersionZero || solrCore.readOnly) {
             openNewSearcherAndUpdateCommitPoint();
           } else {
             SolrQueryRequest req = new LocalSolrQueryRequest(solrCore, new ModifiableSolrParams());
@@ -624,7 +624,7 @@ IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean forceCoreRel
           // are successfully deleted
           solrCore.getUpdateHandler().newIndexWriter(true);
           RefCounted<IndexWriter> writer =
-              solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null);
+              solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null, true);
           try {
             IndexWriter indexWriter = writer.get();
             int c = 0;

solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java

@@ -1378,7 +1378,7 @@ public void inform(SolrCore core) {
 
           // ensure the writer is initialized so that we have a list of commit points
           RefCounted<IndexWriter> iw =
-              core.getUpdateHandler().getSolrCoreState().getIndexWriter(core);
+              core.getUpdateHandler().getSolrCoreState().getIndexWriter(core, true);
           iw.decref();
 
         } catch (IOException e) {

solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java

@@ -99,6 +99,7 @@ public static void updateCommit(CommitUpdateCommand cmd, SolrParams params) {
     cmd.maxOptimizeSegments =
         params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);
     cmd.prepareCommit = params.getBool(UpdateParams.PREPARE_COMMIT, cmd.prepareCommit);
+    cmd.failOnReadOnly = params.getBool(UpdateParams.FAIL_ON_READ_ONLY, cmd.failOnReadOnly);
   }
 
   /**

solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java

@@ -39,6 +39,7 @@ public class CommitUpdateCommand extends UpdateCommand {
   public boolean expungeDeletes = false;
   public boolean softCommit = false;
   public boolean prepareCommit = false;
+  public boolean failOnReadOnly = true; // fail the commit if the core or collection is readOnly
 
   /**
    * User provided commit data. Can be let to null if there is none. It is possible to commit this
@@ -97,7 +98,9 @@ public String toString() {
             .append(",softCommit=")
             .append(softCommit)
             .append(",prepareCommit=")
-            .append(prepareCommit);
+            .append(prepareCommit)
+            .append(",failOnReadOnly=")
+            .append(failOnReadOnly);
     if (commitData != null) {
       sb.append(",commitData=").append(commitData);
     }

solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java

@@ -106,8 +106,9 @@ private void closeIndexWriter(IndexWriterCloser closer) {
   }
 
   @Override
-  public RefCounted<IndexWriter> getIndexWriter(SolrCore core) throws IOException {
-    if (core != null && (!core.indexEnabled || core.readOnly)) {
+  public RefCounted<IndexWriter> getIndexWriter(SolrCore core, boolean readOnlyCompatible)
+      throws IOException {
+    if (core != null && (!core.indexEnabled || (!readOnlyCompatible && core.readOnly))) {
       throw new SolrException(
           SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Indexing is temporarily disabled");
     }

solr/core/src/java/org/apache/solr/update/SolrCoreState.java

@@ -178,7 +178,19 @@ public void deregisterInFlightUpdate() {
    *
    * @throws IOException If there is a low-level I/O error.
    */
-  public abstract RefCounted<IndexWriter> getIndexWriter(SolrCore core) throws IOException;
+  public RefCounted<IndexWriter> getIndexWriter(SolrCore core) throws IOException {
+    return getIndexWriter(core, false);
+  }
+
+  /**
+   * Get the current IndexWriter. If a new IndexWriter must be created, use the settings from the
+   * given {@link SolrCore}. Be very careful using the {@code readOnlyCompatible} flag, by default
+   * it should be false if the returned indexWriter will be used for writing.
+   *
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public abstract RefCounted<IndexWriter> getIndexWriter(SolrCore core, boolean readOnlyCompatible)
+      throws IOException;
 
   /**
    * Rollback the current IndexWriter. When creating the new IndexWriter use the settings from the

solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java

@@ -161,7 +161,13 @@ public void processCommit(CommitUpdateCommand cmd) throws IOException {
     assert TestInjection.injectFailUpdateRequests();
 
     if (isReadOnly()) {
-      throw new SolrException(ErrorCode.FORBIDDEN, "Collection " + collection + " is read-only.");
+      if (cmd.failOnReadOnly) {
+        throw new SolrException(ErrorCode.FORBIDDEN, "Collection " + collection + " is read-only.");
+      } else {
+        // Committing on a readOnly core/collection is a no-op, since the core was committed when
+        // becoming read-only and hasn't had any updates since.
+        return;
+      }
     }
 
     List<SolrCmdDistributor.Node> nodes = null;

solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java

@@ -46,6 +46,9 @@ public interface UpdateParams {
   /** expert: calls IndexWriter.prepareCommit */
   public static String PREPARE_COMMIT = "prepareCommit";
 
+  /** Fail a commit when the core or collection is in read-only mode */
+  public static String FAIL_ON_READ_ONLY = "failOnReadOnly";
+
   /** Rollback update commands */
   public static String ROLLBACK = "rollback";