Support running embedded-zk in "ensemble" mode

changelog/unreleased/spike-zk-quorum.yml

@@ -0,0 +1,6 @@
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+title: capability for Solr to run embedded ZooKeeper in a quorum/ensemble mode, allowing multiple Solr nodes to form a distributed ZooKeeper ensemble within their own processes
+type: other # added, changed, fixed, deprecated, removed, dependency_update, security, other
+authors:
+  - name: Eric Pugh
+  - name: Jason Gerlowski

solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java

@@ -42,20 +42,18 @@ public class SolrZkServer {
 
   public static final String ZK_WHITELIST_PROPERTY = "zookeeper.4lw.commands.whitelist";
 
-  boolean zkRun = false;
   String zkHost;
 
   int solrPort;
   Properties props;
   SolrZkServerProps zkProps;
 
-  private Thread zkThread; // the thread running a zookeeper server, only if zkRun is true
+  private Thread zkThread; // the thread running a zookeeper server, only if zkServerEnabled is true
 
   private Path dataHome; // o.a.zookeeper.**.QuorumPeerConfig needs a File not a Path
   private String confHome;
 
-  public SolrZkServer(boolean zkRun, String zkHost, Path dataHome, String confHome, int solrPort) {
-    this.zkRun = zkRun;
+  public SolrZkServer(String zkHost, Path dataHome, String confHome, int solrPort) {
     this.zkHost = zkHost;
     this.dataHome = dataHome;
     this.confHome = confHome;
@@ -71,11 +69,6 @@ public String getClientString() {
       return null;
     }
 
-    // if the string wasn't passed as zkHost, then use the standalone server we started
-    if (!zkRun) {
-      return null;
-    }
-
     InetSocketAddress addr = zkProps.getClientPortAddress();
     String hostName;
     // We cannot advertise 0.0.0.0, so choose the best host to advertise
@@ -94,7 +87,6 @@ public void parseConfig() {
       // set default data dir
       // TODO: use something based on IP+port???  support ensemble all from same solr home?
       zkProps.setDataDir(dataHome);
-      zkProps.zkRun = zkRun;
       zkProps.solrPort = Integer.toString(solrPort);
     }
 
@@ -113,7 +105,7 @@ public void parseConfig() {
 
     try {
       props = SolrZkServerProps.getProperties(zooCfgPath);
-      SolrZkServerProps.injectServers(props, zkRun, zkHost);
+      SolrZkServerProps.injectServers(props, zkHost);
       // This is the address that the embedded Zookeeper will bind to. Like Solr, it defaults to
       // "127.0.0.1".
       props.setProperty(
@@ -123,9 +115,8 @@ public void parseConfig() {
       }
       zkProps.parseProperties(props);
     } catch (QuorumPeerConfig.ConfigException | IOException e) {
-      if (zkRun) {
-        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
-      }
+
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
     }
   }
 
@@ -134,9 +125,6 @@ public Map<Long, QuorumPeer.QuorumServer> getServers() {
   }
 
   public void start() {
-    if (!zkRun) {
-      return;
-    }
 
     if (System.getProperty(ZK_WHITELIST_PROPERTY) == null) {
       System.setProperty(ZK_WHITELIST_PROPERTY, "ruok, mntr, conf");
@@ -163,20 +151,11 @@ public void start() {
             },
             "embeddedZkServer");
 
-    if (zkProps.getServers().size() > 1) {
-      if (log.isInfoEnabled()) {
-        log.info(
-            "STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port {}, listening on host {}",
-            zkProps.getClientPortAddress().getPort(),
-            zkProps.getClientPortAddress().getAddress().getHostAddress());
-      }
-    } else {
-      if (log.isInfoEnabled()) {
-        log.info(
-            "STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port {}, listening on host {}",
-            zkProps.getClientPortAddress().getPort(),
-            zkProps.getClientPortAddress().getAddress().getHostAddress());
-      }
+    if (log.isInfoEnabled()) {
+      log.info(
+          "STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port {}, listening on host {}",
+          zkProps.getClientPortAddress().getPort(),
+          zkProps.getClientPortAddress().getAddress().getHostAddress());
     }
 
     zkThread.setDaemon(true);
@@ -203,9 +182,7 @@ public void start() {
   }
 
   public void stop() {
-    if (!zkRun) {
-      return;
-    }
+
     zkThread.interrupt();
   }
 }
@@ -216,7 +193,6 @@ class SolrZkServerProps extends QuorumPeerConfig {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
   String solrPort; // port that Solr is listening on
-  boolean zkRun;
 
   /**
    * Parse a ZooKeeper configuration file
@@ -245,10 +221,10 @@ public static Properties getProperties(Path configPath) throws ConfigException {
   // Given zkHost=localhost:1111,localhost:2222 this will inject
   // server.0=localhost:1112:1113
   // server.1=localhost:2223:2224
-  public static void injectServers(Properties props, boolean zkRun, String zkHost) {
+  public static void injectServers(Properties props, String zkHost) {
 
     // if clientPort not already set, use zkRun
-    if (zkRun && props.getProperty("clientPort") == null) {
+    if (props.getProperty("clientPort") == null) {
       // int portIdx = zkRun.lastIndexOf(':');
       int portIdx = "".lastIndexOf(':');
       if (portIdx > 0) {

solr/core/src/java/org/apache/solr/core/CoreContainer.java

@@ -290,7 +290,7 @@ public JerseyAppHandlerCache getJerseyAppHandlerCache() {
 
   private final ObjectCache objectCache = new ObjectCache();
 
-  public final NodeRoles nodeRoles = new NodeRoles(System.getProperty(NodeRoles.NODE_ROLES_PROP));
+  public final NodeRoles nodeRoles = new NodeRoles(EnvUtils.getProperty(NodeRoles.NODE_ROLES_PROP));
 
   private final ExecutorService indexSearcherExecutor;
 

solr/core/src/java/org/apache/solr/core/NodeConfig.java

@@ -236,11 +236,16 @@ public static NodeConfig loadNodeConfig(Path solrHome, Properties nodeProperties
     initModules(loader, null);
     nodeProperties = SolrXmlConfig.wrapAndSetZkHostFromSysPropIfNeeded(nodeProperties);
 
-    // TODO: Only job of this block is to
-    //  delay starting a solr core to satisfy
-    //  ZkFailoverTest test case...
     String zkHost = nodeProperties.getProperty(SolrXmlConfig.ZK_HOST);
-    if (StrUtils.isNotNullOrEmpty(zkHost)) {
+    NodeRoles nodeRoles = new NodeRoles(EnvUtils.getProperty(NodeRoles.NODE_ROLES_PROP));
+    boolean zookeeperQuorumNode =
+        NodeRoles.MODE_ON.equals(nodeRoles.getRoleMode(NodeRoles.Role.ZOOKEEPER_QUORUM));
+
+    // This block demonstrates how we pause and wait for a ZooKeeper to be available before
+    // continuing.
+    // See the ZkFailoverTest to see how changing solr.cloud.wait.for.zk.seconds impacts this
+    // capability.
+    if (StrUtils.isNotNullOrEmpty(zkHost) && !zookeeperQuorumNode) {
       int startUpZkTimeOut =
           1000
               * Integer.getInteger(

solr/core/src/java/org/apache/solr/core/NodeRoles.java

@@ -113,6 +113,18 @@ public String modeWhenRoleIsAbsent() {
       public Set<String> supportedModes() {
         return Set.of(MODE_ON, MODE_OFF);
       }
+    },
+
+    ZOOKEEPER_QUORUM("zookeeper_quorum") {
+      @Override
+      public Set<String> supportedModes() {
+        return Set.of(MODE_ON, MODE_OFF);
+      }
+
+      @Override
+      public String modeWhenRoleIsAbsent() {
+        return MODE_OFF;
+      }
     };
 
     public final String roleName;