Arm backend: Add docgen script to pre-push hook (pytorch#18284)

This will make sure the generated docs don't drift from the actual
documentation.

Signed-off-by: Sebastian Larsson <sebastian.larsson@arm.com>

Author: Sebastian-Larsson

backends/arm/scripts/docgen/docgen.py

@@ -123,8 +123,8 @@ def generate_document(job: DocumentationJob):
 
     content = content.replace(job.placeholder, job.replacement_text)
 
-    # Remove multiple new lines at end of document if it exists
-    if content.endswith("\n\n"):
+    # Normalize generated docs to a single trailing newline.
+    while content.endswith("\n\n"):
         content = content.removesuffix("\n")
 
     with open(job.output_path, "w") as f:

backends/arm/scripts/docgen/vgf/backends-arm-vgf-overview.md.in

@@ -81,6 +81,8 @@ described in the rest of this guide but with a concrete end-to-end sample.
 
 **→{doc}`/backends/arm-vgf/tutorials/arm-vgf-tutorials` — Tutorials.**
 
+**→{doc}`/backends/arm-vgf/VGF_op_support` — VGF supported operators.**
+
 
 ```{toctree}
 :maxdepth: 2
@@ -91,4 +93,5 @@ arm-vgf-partitioner
 arm-vgf-quantization
 arm-vgf-troubleshooting
 tutorials/arm-vgf-tutorials
+VGF_op_support
 ```

backends/arm/scripts/pre-push

@@ -21,6 +21,55 @@ WARNING="${YELLOW}[WARNING]${RESET}"
 ERROR="${RED}[ERROR]${RESET}"
 SUCCESS="${GREEN}[SUCCESS]${RESET}"
 
+DOCGEN_OUTPUTS=(
+    "docs/source/backends/arm-ethos-u/arm-ethos-u-overview.md"
+    "docs/source/backends/arm-ethos-u/arm-ethos-u-partitioner.md"
+    "docs/source/backends/arm-ethos-u/arm-ethos-u-quantization.md"
+    "docs/source/backends/arm-ethos-u/tutorials/ethos-u-getting-started.md"
+    "docs/source/backends/arm-vgf/arm-vgf-overview.md"
+    "docs/source/backends/arm-vgf/arm-vgf-partitioner.md"
+    "docs/source/backends/arm-vgf/arm-vgf-quantization.md"
+    "docs/source/backends/arm-vgf/tutorials/vgf-getting-started.md"
+)
+
+RUN_DOCGEN=0
+
+is_docgen_trigger_file() {
+    local file_path="$1"
+    case "${file_path}" in
+        backends/arm/common/arm_compile_spec.py|\
+        backends/arm/ethosu/*.py|\
+        backends/arm/quantizer/*.py|\
+        backends/arm/scripts/docgen/*|\
+        backends/arm/tosa/partitioner.py|\
+        backends/arm/vgf/*.py|\
+        examples/arm/ethos_u_minimal_example.ipynb|\
+        examples/arm/vgf_minimal_example.ipynb)
+            return 0
+            ;;
+    esac
+    return 1
+}
+
+run_docgen_check() {
+    echo -e "${INFO} Running Arm docgen"
+    env/bin/python backends/arm/scripts/docgen/docgen.py
+    if [[ $? -ne 0 ]]; then
+        echo -e "${ERROR} Failed to run Arm docgen"
+        FAILED=1
+        return
+    fi
+
+    if ! git diff --quiet HEAD -- "${DOCGEN_OUTPUTS[@]}"; then
+        echo -e "${ERROR} Arm docgen updated generated documentation." >&2
+        echo -e "${INFO} Review and commit the regenerated docs before pushing."
+        git diff HEAD -- "${DOCGEN_OUTPUTS[@]}"
+        FAILED=1
+    else
+        echo -e "${SUCCESS} Arm docgen OK"
+    fi
+}
+
 # This list of imperative verbs was compiled from the entire list of Executorch
 # commits. It should be fairly exhaustive, but add more verbs if you find one
 # that's missing.
@@ -92,6 +141,14 @@ for COMMIT in ${COMMITS}; do
 
     commit_files=$(git diff-tree --no-commit-id --name-only \
         --diff-filter=ACMR ${COMMIT} -r)
+    if [[ $RUN_DOCGEN -eq 0 ]]; then
+        for committed_file in $commit_files; do
+            if is_docgen_trigger_file "${committed_file}"; then
+                RUN_DOCGEN=1
+                break
+            fi
+        done
+    fi
 
     # Check license headers
     # We do a simple check of if all committed headers contain
@@ -258,6 +315,12 @@ for COMMIT in ${COMMITS}; do
     echo "" # Newline to visually separate commit processing
 done
 
+if [[ $RUN_DOCGEN -eq 1 ]]; then
+    run_docgen_check
+else
+    echo -e "${INFO} Skipping Arm docgen (no public API inputs changed)"
+fi
+
 if [[ $FAILED ]]; then
     echo -e "${INFO} Fix your commit message errors with"\
         "'git commit --amend' or 'git commit --fixup=<SHA>'"

docs/source/backends/arm-ethos-u/arm-ethos-u-quantization.md

@@ -25,7 +25,7 @@ Args:
         Ethos-U targets.
 
 ```python
-def EthosUQuantizer.quantize_with_submodules(self, model: 'GraphModule', calibration_samples: 'list[tuple]', is_qat: 'bool' = False):
+def EthosUQuantizer.quantize_with_submodules(self, model: 'GraphModule', calibration_samples: 'list[tuple]', is_qat: 'bool' = False, fold_quantize: 'bool' = True):
 ```
 Quantizes a GraphModule in a way such that conditional submodules are
 handled properly.
@@ -41,6 +41,8 @@ Args:
         model with submodules, at least one sample per code path is
         needed.
 - **is_qat (bool)**: Whether to do quantization aware training or not.
+- **fold_quantize (bool)**: Enables or disables constant folding when quantization
+        is completed.
 
 Returns:
 - **GraphModule**: The quantized model.

docs/source/backends/arm-vgf/arm-vgf-quantization.md

@@ -44,7 +44,7 @@ Args:
         targets.
 
 ```python
-def VgfQuantizer.quantize_with_submodules(self, model: 'GraphModule', calibration_samples: 'list[tuple]', is_qat: 'bool' = False):
+def VgfQuantizer.quantize_with_submodules(self, model: 'GraphModule', calibration_samples: 'list[tuple]', is_qat: 'bool' = False, fold_quantize: 'bool' = True):
 ```
 Quantizes a GraphModule in a way such that conditional submodules are
 handled properly.
@@ -60,6 +60,8 @@ Args:
         model with submodules, at least one sample per code path is
         needed.
 - **is_qat (bool)**: Whether to do quantization aware training or not.
+- **fold_quantize (bool)**: Enables or disables constant folding when quantization
+        is completed.
 
 Returns:
 - **GraphModule**: The quantized model.