Merge remote-tracking branch 'origin/main' into control-center-integration

Author: noctarius

docs/k8s_mgmt.md

@@ -0,0 +1,98 @@
+# SimplyBlock Management Node on Kubernetes
+
+This guide explains how to deploy the SimplyBlock Management Node on a Kubernetes cluster using the provided bootstrap scripts.
+
+---
+
+## Prerequisites
+
+- A Linux host or VM with access to the required bootstrap scripts:
+  - `./bootstrap-k3s.sh`
+  - `./bootstrap-cluster.sh`
+- Access to the kubernetes Cluster
+
+---
+
+## Step-by-Step Guide
+
+### 1. Bootstrap the Kubernetes Cluster with Worker Node Support
+
+Run the following command to deploy a K3s-based Kubernetes cluster with support for storage worker nodes:
+
+```bash
+./bootstrap-k3s.sh --k8s-snode
+```
+
+### 2. Prepare an Administrative Host
+
+Once the cluster is bootstrapped, copy the generated kubeconfig file (~/.kube/config or the one output by K3s) to a Linux host where you will perform SimplyBlock cluster administrative tasks. Also, update the IP in the kubeconfig file from 127.0.0.1 to ``<mgmt-worker-node-ip>``:
+
+```bash
+scp /etc/rancher/k3s/k3s.yaml <admin-host>:/home/<user>/.kube/config
+```
+
+> **Important:** The administrative host must be a **Linux machine** that:
+>
+> - Has access to the Kubernetes `kubeconfig` file  
+> - Can reach the Kubernetes **worker nodes over the network**, including the following ports:
+>   - `6443` for Kubernetes API server  
+>   - `4500` for FoundationDB  
+>   - `80` and `443` for HTTP and HTTPS access (if applicable)  
+> - Is used for managing and operating the SimplyBlock cluster (not necessarily running the Management Node itself)
+
+Install SimplyBlock CLI and FoundationDB Client
+On the administrative host, install the following tools:
+
+SimplyBlock CLI (sbctl):
+
+```bash
+pip install sbctl
+```
+
+FoundationDB Client (for RPM-based systems like CentOS/RHEL):
+
+```bash
+sudo yum install -y https://github.com/apple/foundationdb/releases/download/7.3.3/foundationdb-clients-7.3.3-1.el7.x86_64.rpm
+```
+
+
+### 3. Deploy the Cluster in Kubernetes Mode
+Now run the bootstrap cluster script
+```bash
+./bootstrap-cluster.sh --mode kubernetes
+```
+
+### 4. Add FDB Configuration file on Administrative Host
+
+Create foundationdb config directory
+
+```bash
+mkdir /etc/foundationdb
+```
+
+Retrieve the cluster config and write it to fdb.cluster
+
+```bash
+kubectl -n simplyblock get cm simplyblock-config \
+  -o jsonpath="{.data.FDB_CLUSTER_FILE_CONTENTS}" \
+  | sudo tee /etc/foundationdb/fdb.cluster > /dev/null
+```
+
+Optional: Verify the contents
+
+```bash
+cat /etc/foundationdb/fdb.cluster
+```
+
+### 5. Verification
+You can verify that the Management Node is running by checking the pods in the namespace (e.g., simplyblock):
+
+```bash
+kubectl get pods -n simplyblock
+```
+
+List the Bootstrapped cluster.
+
+```bash
+sbctl cluster list
+```

simplyblock_cli/cli-reference.yaml

@@ -701,6 +701,10 @@ commands:
             help: "Management interface name, e.g. eth0"
             dest: ifname
             type: str
+          - name: "--mgmt-ip"
+            help: "Management IP address to use for the node (e.g., 192.168.1.10)"
+            dest: mgmt_ip
+            type: str
           - name: "--log-del-interval"
             help: "Logging retention policy, default: 3d"
             dest: log_del_interval
@@ -1526,10 +1530,14 @@ commands:
             help: "Cluster secret"
             dest: cluster_secret
             type: str
-          - name: "ifname"
+          - name: "--ifname"
             help: "Management interface name"
             dest: ifname
             type: str
+          - name: "--mgmt-ip"
+            help: "Management IP address to use for the node (e.g., 192.168.1.10)"
+            dest: mgmt_ip
+            type: str
           - name: "--mode"
             help: "Environment to deploy management services, default: docker "
             dest: mode

simplyblock_cli/cli.py

@@ -341,6 +341,7 @@ def init_cluster__create(self, subparser):
         argument = subcommand.add_argument('--prov-cap-warn', help='Capacity warning level in percent, default: 250', type=int, default=250, dest='prov_cap_warn')
         argument = subcommand.add_argument('--prov-cap-crit', help='Capacity critical level in percent, default: 500', type=int, default=500, dest='prov_cap_crit')
         argument = subcommand.add_argument('--ifname', help='Management interface name, e.g. eth0', type=str, dest='ifname')
+        argument = subcommand.add_argument('--mgmt-ip', help='Management IP address to use for the node (e.g., 192.168.1.10)', type=str, dest='mgmt_ip')
         argument = subcommand.add_argument('--log-del-interval', help='Logging retention policy, default: 3d', type=str, default='3d', dest='log_del_interval')
         argument = subcommand.add_argument('--metrics-retention-period', help='Retention period for I/O statistics (Prometheus), default: 7d', type=str, default='7d', dest='metrics_retention_period')
         argument = subcommand.add_argument('--contact-point', help='Email or slack webhook url to be used for alerting', type=str, default='', dest='contact_point')
@@ -633,7 +634,8 @@ def init_control_plane__add(self, subparser):
         subcommand.add_argument('cluster_ip', help='Cluster IP address', type=str)
         subcommand.add_argument('cluster_id', help='Cluster id', type=str)
         subcommand.add_argument('cluster_secret', help='Cluster secret', type=str)
-        subcommand.add_argument('ifname', help='Management interface name', type=str)
+        argument = subcommand.add_argument('--ifname', help='Management interface name', type=str, dest='ifname')
+        argument = subcommand.add_argument('--mgmt-ip', help='Management IP address to use for the node (e.g., 192.168.1.10)', type=str, dest='mgmt_ip')
         argument = subcommand.add_argument('--mode', help='Environment to deploy management services, default: docker ', type=str, default='docker', dest='mode', choices=['docker','kubernetes',])
 
     def init_control_plane__list(self, subparser):

simplyblock_cli/clibase.py

@@ -551,8 +551,9 @@ def control_plane__add(self, sub_command, args):
         cluster_ip = args.cluster_ip
         cluster_secret = args.cluster_secret
         ifname = args.ifname
+        mgmt_ip = args.mgmt_ip
         mode = args.mode
-        return mgmt_ops.deploy_mgmt_node(cluster_ip, cluster_id, ifname, cluster_secret, mode)
+        return mgmt_ops.deploy_mgmt_node(cluster_ip, cluster_id, ifname, mgmt_ip, cluster_secret, mode)
 
     def control_plane__list(self, sub_command, args):
         return mgmt_ops.list_mgmt_nodes(args.json)
@@ -664,6 +665,7 @@ def cluster_create(self, args):
         prov_cap_warn = args.prov_cap_warn
         prov_cap_crit = args.prov_cap_crit
         ifname = args.ifname
+        mgmt_ip = args.mgmt_ip
         distr_ndcs = args.distr_ndcs
         distr_npcs = args.distr_npcs
         distr_bs = args.distr_bs
@@ -687,7 +689,7 @@ def cluster_create(self, args):
         return cluster_ops.create_cluster(
             blk_size, page_size_in_blocks,
             CLI_PASS, cap_warn, cap_crit, prov_cap_warn, prov_cap_crit,
-            ifname, log_del_interval, metrics_retention_period, contact_point, grafana_endpoint,
+            ifname, mgmt_ip, log_del_interval, metrics_retention_period, contact_point, grafana_endpoint,
             distr_ndcs, distr_npcs, distr_bs, distr_chunk_bs, ha_type, mode, enable_node_affinity,
             qpair_count, max_queue_size, inflight_io_threshold, enable_qos, disable_monitoring,
             strict_node_anti_affinity, name, refresh_token_secret)

simplyblock_core/cluster_ops.py

@@ -79,12 +79,19 @@ def _create_update_user(cluster_id, grafana_url, grafana_secret, user_secret, up
 
 
 def _add_graylog_input(cluster_ip, password):
-    url = f"http://{cluster_ip}/graylog/api/system/inputs"
+    base_url = f"http://{cluster_ip}/graylog/api"
+    input_url = f"{base_url}/system/inputs"
 
     retries = 30
-    reachable=False
-    while retries > 0:
+    reachable = False
+    session = requests.session()
+    session.auth = ("admin", password)
+    headers = {
+        'X-Requested-By': 'setup-script',
+        'Content-Type': 'application/json',
+    }
 
+    while retries > 0:
         payload = json.dumps({
             "title": "spdk log input",
             "type": "org.graylog2.inputs.gelf.tcp.GELFTCPInput",
@@ -99,30 +106,61 @@ def _add_graylog_input(cluster_ip, password):
             },
             "global": True
         })
-        headers = {
-            'X-Requested-By': '',
-            'Content-Type': 'application/json',
-        }
-        session = requests.session()
-        session.auth = ("admin", password)
-        response = session.request("POST", url, headers=headers, data=payload)
+
+        response = session.post(input_url, headers=headers, data=payload)
         if response.status_code == 201:
             logger.info("Graylog input created...")
-            reachable=True
+            reachable = True
             break
 
         logger.debug(response.text)
         retries -= 1
         time.sleep(5)
+
     if not reachable:
         logger.error(f"Failed to create graylog input: {response.text}")
         return False
 
-    return response.status_code == 201
+    inputs_response = session.get(input_url, headers=headers)
+    if inputs_response.status_code != 200:
+        logger.error(f"Failed to retrieve inputs: {inputs_response.text}")
+        return False
+
+    input_id = None
+    for item in inputs_response.json()["inputs"]:
+        if item["title"] == "spdk log input":
+            input_id = item["id"]
+            break
+
+    if not input_id:
+        logger.error("Could not find created input to add extractor.")
+        return False
+
+    extractor_url = f"{input_url}/{input_id}/extractors"
+    extractor_payload = {
+        "title": "Extract Kubernetes JSON",
+        "type": "json",
+        "converters": [],
+        "order": 0,
+        "cursor_strategy": "copy",
+        "source_field": "message",
+        "target_field": "",
+        "extractor_config": {},
+        "condition_type": "none",
+        "condition_value": ""
+    }
+
+    extractor_response = session.post(extractor_url, headers=headers, data=json.dumps(extractor_payload))
+    if extractor_response.status_code != 201:
+        logger.error(f"Failed to add JSON extractor: {extractor_response.text}")
+        return False
+
+    logger.info("JSON extractor added successfully.")
+    return True
 
 def _set_max_result_window(cluster_ip, max_window=100000):
 
-    url_existing_indices = f"http://{cluster_ip}:9200/_all/_settings"
+    url_existing_indices = f"http://{cluster_ip}/opensearch/_all/_settings"
 
     retries = 30
     reachable=False
@@ -149,7 +187,7 @@ def _set_max_result_window(cluster_ip, max_window=100000):
         logger.error(f"Failed to update settings for existing indices: {response.text}")
         return False
 
-    url_template = f"http://{cluster_ip}:9200/_template/all_indices_template"
+    url_template = f"http://{cluster_ip}/opensearch/_template/all_indices_template"
     payload_template = json.dumps({
         "index_patterns": ["*"],
         "settings": {
@@ -166,7 +204,7 @@ def _set_max_result_window(cluster_ip, max_window=100000):
 
 
 def create_cluster(blk_size, page_size_in_blocks, cli_pass,
-                   cap_warn, cap_crit, prov_cap_warn, prov_cap_crit, ifname, log_del_interval, metrics_retention_period,
+                   cap_warn, cap_crit, prov_cap_warn, prov_cap_crit, ifname, mgmt_ip, log_del_interval, metrics_retention_period,
                    contact_point, grafana_endpoint, distr_ndcs, distr_npcs, distr_bs, distr_chunk_bs, ha_type, mode,
                    enable_node_affinity, qpair_count, max_queue_size, inflight_io_threshold, enable_qos, disable_monitoring,
                    strict_node_anti_affinity, name, refresh_token_secret) -> str:
@@ -178,17 +216,16 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass,
     scripts.install_deps(mode)
     logger.info("Installing dependencies > Done")
 
-    if not ifname:
-        ifname = "eth0"
-
-    dev_ip = utils.get_iface_ip(ifname)
-    if not dev_ip:
-        raise ValueError(f"Error getting interface ip: {ifname}")
+    if mode == "docker": 
+        if not ifname:
+            ifname = "eth0"
 
-    db_connection = f"{utils.generate_string(8)}:{utils.generate_string(32)}@{dev_ip}:4500"
-    scripts.set_db_config(db_connection)
+        dev_ip = utils.get_iface_ip(ifname)
+        if not dev_ip:
+            raise ValueError(f"Error getting interface ip: {ifname}")
 
-    if mode == "docker": 
+        db_connection = f"{utils.generate_string(8)}:{utils.generate_string(32)}@{dev_ip}:4500"
+        scripts.set_db_config(db_connection)
         logger.info(f"Node IP: {dev_ip}")
         scripts.configure_docker(dev_ip)
         logger.info("Configuring docker swarm...")
@@ -218,6 +255,14 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass,
             logger.info(f"Labeled node '{hostname}' with app=graylog")
         else:
             logger.warning("Could not find current node for labeling")
+    elif mode == "kubernetes":
+        dev_ip = mgmt_ip
+        if not dev_ip:
+            raise ValueError("Error getting ip: For Kubernetes-based deployments, please supply --mgmt-ip.")
+
+        current_node = utils.get_node_name_by_ip(dev_ip)
+        utils.label_node_as_mgmt_plane(current_node)
+        db_connection = f"{utils.generate_string(8)}:{utils.generate_string(32)}@{dev_ip}:4500"
 
     if not cli_pass:
         cli_pass = utils.generate_string(10)
@@ -286,7 +331,8 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass,
         log_level = "DEBUG" if constants.LOG_WEB_DEBUG else "INFO"
         scripts.deploy_k8s_stack(cli_pass, dev_ip, constants.SIMPLY_BLOCK_DOCKER_IMAGE, cluster.secret, cluster.uuid,
                                 log_del_interval, metrics_retention_period, log_level, cluster.grafana_endpoint,
-                                 contact_point, constants.K8S_NAMESPACE, str(disable_monitoring), refresh_token_secret)
+                                 contact_point, db_connection, constants.K8S_NAMESPACE, str(disable_monitoring),
+                                 refresh_token_secret)
         logger.info("Deploying helm stack > Done")
 
     logger.info("Configuring DB...")
@@ -307,7 +353,7 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass,
 
     cluster_events.cluster_create(cluster)
 
-    mgmt_node_ops.add_mgmt_node(dev_ip, cluster.uuid)
+    mgmt_node_ops.add_mgmt_node(dev_ip, mode, cluster.uuid)
 
     logger.info("New Cluster has been created")
     logger.info(cluster.uuid)