DOC-1386 improve/tighten Markdown generation

src/hdx_cli/cli_interface/common/click_extensions.py

@@ -38,7 +38,7 @@ def _get_depth(ctx: click.Context) -> int:
 }
 
 
-def _generate_options_table(command: click.Command, ctx: click.Context) -> str:
+def _generate_options_table(command: click.Command, ctx: click.Context, text: str) -> str:
     """Generates a Markdown table for a command's options."""
     opts = [
         p.get_help_record(ctx)
@@ -47,11 +47,13 @@ def _generate_options_table(command: click.Command, ctx: click.Context) -> str:
     ]
 
     if not opts:
-        return ""
+        return None
 
+    depth = _get_depth(ctx)
+    subheading = "#" * (depth + 1)
     format_args = _create_format_args(ctx)
     table_parts = [
-        "**Options**",
+        f"{subheading} Options for {text}",
         "",  # Adds a blank line before the table
         "| Option | Description |",
         "|:-------|:------------|",
@@ -160,6 +162,7 @@ def to_markdown(self, ctx: click.Context) -> str:
         depth = _get_depth(ctx)
         heading = "#" * depth
         command_title = self.name.replace("-", " ").replace("_", " ").title()
+        command_title += f" ({resource})"
         md_parts = [f"{heading} {command_title}\n"]
 
         format_args = _create_format_args(ctx)
@@ -175,14 +178,17 @@ def to_markdown(self, ctx: click.Context) -> str:
         usage_line = self.get_usage(ctx).replace("Usage: ", "")
         # if usage_line is too long, click adds '\n' at some point. It cleans it up.
         usage_line = re.sub(r"\s*\n\s*", " ", usage_line).strip()
-        md_parts.append(f"**Usage**\n\n```bash\n{usage_line}\n```\n")
+        subheading = "#" * (depth + 1)
+        md_parts.append(f"{subheading} Usage for {command_title}\n\n```bash\n{usage_line}\n```\n")
 
         # Options
-        md_parts.append(_generate_options_table(self, ctx))
+        options = _generate_options_table(self, ctx, command_title)
+        if options is not None:
+            md_parts.append(options)
 
         # Examples
         if examples_str:
-            md_parts.append("**Examples**\n")
+            md_parts.append(f"{subheading} Examples for {command_title}\n")
             md_parts.append(f"```bash\n{inspect.cleandoc(examples_str)}\n```\n")
 
         # Markdown-only content
@@ -233,10 +239,13 @@ def to_markdown(self, ctx: click.Context) -> str:
         usage_line = self.get_usage(ctx).replace("Usage: ", "")
         # if usage_line is too long, click adds '\n' at some point. It cleans it up.
         usage_line = re.sub(r"\s*\n\s*", " ", usage_line).strip()
-        md_parts.append(f"**Usage**\n\n```bash\n{usage_line}\n```\n")
+        subheading = "#" * (depth + 1)
+        md_parts.append(f"{subheading} Usage for {group_title}\n\n```bash\n{usage_line}\n```\n")
 
         # Options
-        md_parts.append(_generate_options_table(self, ctx))
+        options = _generate_options_table(self, ctx, group_title)
+        if options is not None:
+            md_parts.append(options)
 
         # Markdown-only content
         if markdown_only_content:
@@ -253,13 +262,10 @@ def to_markdown(self, ctx: click.Context) -> str:
             (subgroups_md if isinstance(cmd, click.Group) else direct_commands_md).append(cmd_md)
 
         if direct_commands_md:
-            md_parts.append("\n\n".join(direct_commands_md))
+            md_parts.append("\n".join(direct_commands_md))
 
         if subgroups_md:
-            if direct_commands_md:
-                md_parts.append("\n\n")
-
-            md_parts.append("\n\n".join(subgroups_md))
+            md_parts.append("\n".join(subgroups_md))
 
         return "\n".join(md_parts)
 

src/hdx_cli/cli_interface/common/misc_operations.py

@@ -41,6 +41,7 @@ def settings(ctx: click.Context, key: str | None, value: str | None, force: bool
 
     \b
     This command operates in three modes:
+
     - *LIST*: Invoked with no arguments, it lists all settings.
     - *GET*: Invoked with only a KEY, it retrieves the value of that setting.
     - *SET*: Invoked with a KEY and a VALUE, it sets the value for that setting.

src/hdx_cli/cli_interface/docs/commands.py

@@ -75,7 +75,7 @@ def _generate_full_docs(root_ctx: click.Context, output_path: Path):
         frontmatter_parts.append("---")
 
         frontmatter = "\n".join(frontmatter_parts)
-        version_line = f"> _hdxcli v{VERSION}_"
+        version_line = f"Documentation covers _hdxcli v{VERSION}_"
 
         # The markdown file starts with the frontmatter and the version line
         final_md_content = f"{frontmatter}\n\n{version_line}\n\n"
@@ -88,7 +88,7 @@ def _generate_full_docs(root_ctx: click.Context, output_path: Path):
                 md_content = command.to_markdown(cmd_ctx)
                 body_parts.append(md_content)
 
-        final_md_content += "\n\n---\n\n".join(body_parts)
+        final_md_content += "\n---\n\n".join(body_parts)
 
         file_path = output_path / group_filename
         file_path.write_text(final_md_content, encoding="utf-8")

src/hdx_cli/cli_interface/docs/config.py

@@ -7,9 +7,7 @@
         "commands": ["init", "profile", "set", "unset"],
         "frontmatter": {
             "title": "Configuration",
-            "slug": "hdxcli-configuration",
-            "excerpt": "Commands to configure the HDXCLI. Manage connection profiles to your clusters and set default contexts (project/table) to streamline your operations.",
-            "category": "65774de90f4b3f001039021c",
+            "subtitle": "Commands to configure the HDXCLI. Manage connection profiles to your clusters and set default contexts (project/table) to streamline your operations.",
             "hidden": False,
         },
     },
@@ -18,9 +16,7 @@
         "commands": ["project", "table", "dictionary", "function", "transform", "view", "column"],
         "frontmatter": {
             "title": "Resource Management",
-            "slug": "hdxcli-resource-management",
-            "excerpt": "Commands for the lifecycle (create, list, delete, modify) of core Hydrolix resources. Use these to manage the structure of your data.",
-            "category": "65774de90f4b3f001039021c",
+            "subtitle": "Commands for the lifecycle (create, list, delete, modify) of core Hydrolix resources. Use these to manage the structure of your data.",
             "hidden": False,
         },
     },
@@ -29,9 +25,7 @@
         "commands": ["stream", "job", "source", "storage"],
         "frontmatter": {
             "title": "Data & Jobs",
-            "slug": "hdxcli-data-and-jobs",
-            "excerpt": "Commands for data ingestion and process management. This covers real-time streaming, batch jobs, and storage configuration.",
-            "category": "65774de90f4b3f001039021c",
+            "subtitle": "Commands for data ingestion and process management. This covers real-time streaming, batch jobs, and storage configuration.",
             "hidden": False,
         },
     },
@@ -40,9 +34,7 @@
         "commands": ["user", "service-account", "role", "row-policy", "credential"],
         "frontmatter": {
             "title": "Security & Access Control",
-            "slug": "hdxcli-security-and-access",
-            "excerpt": "Commands for security management. Administer user and service accounts, assign roles and permissions, define row-level access policies and configure credentials for external systems.",
-            "category": "65774de90f4b3f001039021c",
+            "subtitle": "Commands for security management. Administer user and service accounts, assign roles and permissions, define row-level access policies and configure credentials for external systems.",
             "hidden": False,
         },
     },
@@ -60,9 +52,7 @@
         ],
         "frontmatter": {
             "title": "Utilities & High-Level Operations",
-            "slug": "hdxcli-utilities-and-operations",
-            "excerpt": "Commands for advanced and utility operations. Find tools for cross-cluster migrations, integrity checks, resource pool management, and resource summaries here.",
-            "category": "65774de90f4b3f001039021c",
+            "subtitle": "Commands for advanced and utility operations. Find tools for cross-cluster migrations, integrity checks, resource pool management, and resource summaries here.",
             "hidden": False,
         },
     },

src/hdx_cli/cli_interface/migrate/commands.py

@@ -186,36 +186,39 @@ def migrate(
 
     \b
     This command orchestrates a table migration, which can involve two main stages:
+
     1. *Resource Creation*: Replicates the source project, table, and transforms
        on the target cluster. Optionally, it can also migrate associated
        functions and dictionaries.
+    \b
     2. *Data Migration*: Copies the table's data from the source storage
        to the target and updates the catalog to make the data queryable.
 
     \b
-    **Arguments**:
-      - `SOURCE_TABLE`: The source table to migrate, in 'project.table' format.
-      - `TARGET_TABLE`: The destination for the migration, in 'project.table' format.
+    Arguments:
+
+    - `SOURCE_TABLE`: The source table to migrate, in 'project.table' format.
+    - `TARGET_TABLE`: The destination for the migration, in 'project.table' format.
 
     \b
     **Key Options**:
     - Target Cluster: Specify the destination with `--target-profile` or with individual
       connection details (`--target-hostname`, `--target-username`, etc.).
     \b
     - Migration Scope (`--only`):
-      - *resources*: Migrates only the project, table, and other definitions.
-      - *data*: Migrates only the data, assuming resources already exist.
-      - If omitted, a full migration (resources and data) is performed.
+        - *resources*: Migrates only the project, table, and other definitions.
+        - *data*: Migrates only the data, assuming resources already exist.
+        - If omitted, a full migration (resources and data) is performed.
     \b
     - Data Handling:
-      - `--reuse-partitions`: For clusters sharing storage. Migrates the table
-        definition but reuses the existing data, avoiding a data copy.
-      - `--from-date`/`--to-date`: Filter the data to be migrated by a date range.
+        - `--reuse-partitions`: For clusters sharing storage. Migrates the table
+          definition but reuses the existing data, avoiding a data copy.
+        - `--from-date`/`--to-date`: Filter the data to be migrated by a date range.
     \b
     - Rclone Remote:
-      - `--rc-host`, `--rc-user`, `--rc-pass`: Connection details for the Rclone
-        server that will perform the data transfer. Required for any migration
-        that copies data.
+        - `--rc-host`, `--rc-user`, `--rc-pass`: Connection details for the Rclone
+          server that will perform the data transfer. Required for any migration
+          that copies data.
 
     \b
     Examples:

src/hdx_cli/cli_interface/project/commands.py

@@ -110,6 +110,8 @@ def migrate(
 
     \b
     Options allow for customizing the migration:
+
+    \b
     - `--dictionaries`: Include associated dictionaries.
     - `--functions`: Include associated functions.
     - `--only`: Migrate only the {resource}, skipping all dependencies.

src/hdx_cli/cli_interface/shadow/commands.py

@@ -89,7 +89,7 @@ def shadow(ctx: click.Context, project_name: str):
     "shadow_table_name",
     required=False,
     default=None,
-    help="Name of the shadow table. Default: shadow_ + 'source-table-name'.",
+    help="Name of the shadow table. Default: `shadow_` + 'source-table-name'.",
 )
 @click.option(
     "--table-settings",

src/hdx_cli/cli_interface/storage/commands.py

@@ -121,8 +121,9 @@ def create(
 
     \b
     A {resource} can be created in two ways:
-      - Using a JSON settings file via `--settings-filename`.
-      - Providing individual settings as options (`--bucket-path`, `--bucket-name`, etc).
+
+    - Using a JSON settings file via `--settings-filename`.
+    - Providing individual settings as options (`--bucket-path`, `--bucket-name`, etc).
 
     \b
     Examples:

src/hdx_cli/main.py

@@ -175,10 +175,11 @@ def init():
     will exit with an error to prevent overwriting settings.
 
     \f
-    **Example Output**
+    Example Output
 
     The following is a sample of the interactive `init` session:
-    ```
+
+    ```shell
     $ hdxcli init
     No configuration found for your Hydrolix cluster.
     Let's create the 'default' profile to get you started.