[ML] Automate patch version bump in CI pipeline

.buildkite/job-version-bump-phase2.json.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+#
+# Phase 2 of the ml-cpp version-bump pipeline (uploaded by
+# dev-tools/version_bump_upload_phase2.sh after validate). Step-level `if` cannot
+# use Buildkite meta-data; gating is done in that shell script instead.
+
+import contextlib
+import json
+import os
+
+
+WOLFI_IMAGE = "docker.elastic.co/release-eng/wolfi-build-essential-release-eng:latest"
+
+
+def main():
+    pipeline_steps = [
+        {
+            "label": "Bump version to ${NEW_VERSION}",
+            "key": "bump-version",
+            "depends_on": "schedule-version-bump-follow-up",
+            "agents": {
+                "image": WOLFI_IMAGE,
+                "cpu": "250m",
+                "memory": "512Mi",
+            },
+            "env": {
+                "VERSION_BUMP_MERGE_AUTO": os.environ.get("VERSION_BUMP_MERGE_AUTO", "true"),
+            },
+            "command": [
+                "dev-tools/bump_version.sh",
+            ],
+        },
+        {
+            "label": "Notify :slack: — version bump PR needs approval",
+            "key": "queue-slack-notify",
+            "depends_on": "bump-version",
+            "command": [
+                ".buildkite/pipelines/send_slack_version_bump_notification.sh",
+            ],
+            "agents": {
+                # Same image as bump-version: the minimal python image does not ship
+                # buildkite-agent, so meta-data get / pipeline upload silently skipped Slack.
+                "image": WOLFI_IMAGE,
+                "cpu": "250m",
+                "memory": "512Mi",
+            },
+        },
+        {
+            "label": "Fetch DRA Artifacts",
+            "key": "fetch-dra-artifacts",
+            "depends_on": "queue-slack-notify",
+            "agents": {
+                "image": WOLFI_IMAGE,
+                "cpu": "250m",
+                "memory": "512Mi",
+                "ephemeralStorage": "1Gi",
+            },
+            "command": [
+                "python3",
+                "dev-tools/wait_version_bump_dra.py",
+            ],
+            "timeout_in_minutes": 240,
+            "retry": {
+                "automatic": [{"exit_status": "*", "limit": 2}],
+                "manual": {"permit_on_passed": True},
+            },
+        },
+    ]
+
+    print(json.dumps({"steps": pipeline_steps}, indent=2))
+
+
+if __name__ == "__main__":
+    with contextlib.suppress(KeyboardInterrupt):
+        main()

.buildkite/job-version-bump.json.py

@@ -8,87 +8,50 @@
 # compliance with the Elastic License 2.0 and the foregoing additional
 # limitation.
 #
-# This script generates JSON for the ml-cpp version bump pipeline.
-# It is intended to be triggered by the centralized release-eng pipeline.
-# It can be integrated into existing or new workflows and includes a plugin
-# that polls artifact URLs until the expected version is available.
-
+# Phase 1 of the ml-cpp version bump pipeline (dynamic upload from release-eng).
+#
+# Buildkite step `if` expressions cannot use build meta-data (see
+# https://buildkite.com/docs/pipelines/conditionals ). validate_version_bump_params.sh
+# sets ml_cpp_version_bump_noop when origin already matches NEW_VERSION; phase 2
+# (Slack, bump, DRA wait) is uploaded only when needed by
+# dev-tools/version_bump_upload_phase2.sh.
 
 import contextlib
 import json
 
 
+WOLFI_IMAGE = "docker.elastic.co/release-eng/wolfi-build-essential-release-eng:latest"
+
+
 def main():
-    pipeline = {}
-    # TODO: replace the block step with version bump logic
     pipeline_steps = [
         {
-            "label": "Queue a :slack: notification for the pipeline",
+            "label": "Validate version bump parameters",
+            "key": "validate-version-bump",
             "depends_on": None,
-            "command": ".buildkite/pipelines/send_version_bump_notification.sh | buildkite-agent pipeline upload",
             "agents": {
-                "image": "python",
-            },
-        },
-        {
-            "block": "Ready to fetch for DRA artifacts?",
-            "prompt": (
-                "Unblock when your team is ready to proceed.\n\n"
-                "Trigger parameters:\n"
-                "- NEW_VERSION: ${NEW_VERSION}\n"
-                "- BRANCH: ${BRANCH}\n"
-                "- WORKFLOW: ${WORKFLOW}\n"
-            ),
-            "key": "block-get-dra-artifacts",
-            "blocked_state": "running",
-        },
-        {
-            "label": "Fetch DRA Artifacts",
-            "key": "fetch-dra-artifacts",
-            "depends_on": "block-get-dra-artifacts",
-            "agents": {
-                "image": "docker.elastic.co/release-eng/wolfi-build-essential-release-eng:latest",
+                "image": WOLFI_IMAGE,
                 "cpu": "250m",
                 "memory": "512Mi",
-                "ephemeralStorage": "1Gi",
             },
             "command": [
-                'echo "Starting DRA artifacts retrieval..."',
+                "dev-tools/validate_version_bump_params.sh",
             ],
-            "timeout_in_minutes": 240,
-            "retry": {
-                "automatic": [
-                    {
-                        "exit_status": "*",
-                        "limit": 2,
-                    }
-                ],
-                "manual": {"permit_on_passed": True},
+        },
+        {
+            "label": "Schedule version bump follow-up steps",
+            "key": "schedule-version-bump-follow-up",
+            "depends_on": "validate-version-bump",
+            "agents": {
+                "image": "python",
             },
-            "plugins": [
-                {
-                    "elastic/json-watcher#v1.0.0": {
-                        "url": "https://artifacts-staging.elastic.co/ml-cpp/latest/${BRANCH}.json",
-                        "field": ".version",
-                        "expected_value": "${NEW_VERSION}",
-                        "polling_interval": "30",
-                    }
-                },
-                {
-                    "elastic/json-watcher#v1.0.0": {
-                        "url": "https://storage.googleapis.com/elastic-artifacts-snapshot/ml-cpp/latest/${BRANCH}.json",
-                        "field": ".version",
-                        "expected_value": "${NEW_VERSION}-SNAPSHOT",
-                        "polling_interval": "30",
-                    }
-                },
+            "command": [
+                "dev-tools/version_bump_upload_phase2.sh",
             ],
         },
     ]
 
-    pipeline["steps"] = pipeline_steps
-
-    print(json.dumps(pipeline, indent=2))
+    print(json.dumps({"steps": pipeline_steps}, indent=2))
 
 
 if __name__ == "__main__":

.buildkite/pipelines/format_and_validation.yml.sh

@@ -18,4 +18,13 @@ steps:
     notify:
       - github_commit_status:
           context: "Validate formatting with clang-format"
+
+  - label: "dev-tools pytest"
+    key: "dev_tools_pytest"
+    command: ".buildkite/scripts/steps/dev_tools_pytest.sh"
+    agents:
+      image: "python:3"
+    notify:
+      - github_commit_status:
+          context: "dev-tools pytest"
 EOL

.buildkite/pipelines/send_slack_version_bump_notification.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+#
+# Single Slack notification for the ml-cpp-version-bump pipeline: runs after the
+# bump step opens the PR. Reads ml_cpp_version_bump_pr_url from Buildkite meta-data
+# (set by dev-tools/bump_version.sh) and posts the PR link so reviewers can approve.
+#
+# Slack notify must live on the step (see Buildkite docs): build-level notify fires only
+# on build.finished — after every downstream step including long DRA waits — so the
+# message would appear hours late or never if someone checks earlier.
+#
+# Optional env:
+#   ML_CPP_VERSION_BUMP_SLACK_CHANNEL — override channel (default #machine-learn-build)
+
+set -euo pipefail
+
+CHANNEL="${ML_CPP_VERSION_BUMP_SLACK_CHANNEL:-#machine-learn-build}"
+
+if [[ "${BUILDKITE:-}" != "true" ]]; then
+    echo "BUILDKITE is not true — skipping Slack notification (local run)."
+    exit 0
+fi
+
+if ! command -v buildkite-agent >/dev/null 2>&1; then
+    echo "ERROR: buildkite-agent not in PATH; cannot read meta-data or upload Slack notify pipeline." >&2
+    echo "Use the same agent image as bump-version (Wolfi), not a minimal python image." >&2
+    exit 1
+fi
+
+pr_url=""
+changed="false"
+pr_url=$(buildkite-agent meta-data get "ml_cpp_version_bump_pr_url" 2>/dev/null || true)
+changed=$(buildkite-agent meta-data get "ml_cpp_version_bump_changed" 2>/dev/null || echo "false")
+# Meta-data values must not contain stray whitespace (Breaks truthiness.)
+pr_url=$(echo -n "${pr_url}" | tr -d '\r')
+changed=$(echo -n "${changed}" | tr -d '\r')
+
+if [[ -z "${pr_url}" && "${changed}" != "true" ]]; then
+    echo "No version-bump PR opened (pr_url empty, ml_cpp_version_bump_changed=${changed}); skipping Slack notification."
+    exit 0
+fi
+
+if [[ -z "${pr_url}" && "${changed}" == "true" ]]; then
+    body_line="DRY RUN — no pull request URL (simulated bump)."
+else
+    body_line="Pull request (approval required): ${pr_url}"
+fi
+
+(
+    cat <<EOF
+steps:
+  - label: "Schedule :slack: notification (version bump)"
+    command: "echo schedule :slack: notification"
+    notify:
+      - slack:
+          channels:
+            - "${CHANNEL}"
+          message: |
+            **Version bump PR — approval required**
+            ${body_line}
+            Branch: \${BUILDKITE_BRANCH}
+            NEW_VERSION: \${NEW_VERSION:-"(unset)"}
+            BRANCH (param): \${BRANCH:-"(unset)"}
+            VERSION_BUMP_MERGE_AUTO: \${VERSION_BUMP_MERGE_AUTO:-"(unset)"}
+            DRY_RUN: \${DRY_RUN:-"(unset)"}
+            Pipeline: \${BUILDKITE_BUILD_URL}
+            Build: \${BUILDKITE_BUILD_NUMBER}
+            Please review and approve this pull request so it can merge (subject to branch protection).
+EOF
+) | buildkite-agent pipeline upload

.buildkite/pipelines/send_version_bump_notification.sh

@@ -23,7 +23,6 @@ notify:
         :large_green_circle: Version bump pipeline waiting for approval
         Branch: \${BUILDKITE_BRANCH}
         New version: \${NEW_VERSION}
-        Workflow: \${WORKFLOW}
         Pipeline: \${BUILDKITE_BUILD_URL}
     if: build.state == "blocked"
   - slack:
@@ -33,6 +32,5 @@ notify:
         Version bump pipeline finished (${BUILDKITE_BUILD_URL})
         Branch: \${BUILDKITE_BRANCH}
         New version: \${NEW_VERSION}
-        Workflow: \${WORKFLOW}
     if: build.state != "blocked"
 EOL

.buildkite/scripts/steps/dev_tools_pytest.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+#
+# Installs dev-tools pytest deps once per job (PyPI), then runs run_dev_tools_tests.sh.
+# Keeping pip here isolates network/bootstrap from the test runner; baking deps into
+# the agent image would avoid live PyPI on every PR if desired.
+
+set -euo pipefail
+
+REPO_ROOT="${REPO_ROOT:-$(git rev-parse --show-toplevel)}"
+cd "${REPO_ROOT}"
+
+if ! python3 -m pip install -q -r "${REPO_ROOT}/dev-tools/test-requirements.txt"; then
+    echo "ERROR: pip install failed for dev-tools/test-requirements.txt (network or PyPI?)." >&2
+    echo "Install manually on this agent, then re-run:" >&2
+    echo "  python3 -m pip install -r ${REPO_ROOT}/dev-tools/test-requirements.txt" >&2
+    exit 1
+fi
+
+exec "${REPO_ROOT}/dev-tools/run_dev_tools_tests.sh" -q --tb=short