Add symbolic expression

.clang-tidy

@@ -6,7 +6,7 @@ CheckOptions:
   - key:             bugprone-unused-return-value.AllowCastToVoid
     value:           true
   - key:             cppcoreguidelines-macro-usage.AllowedRegexp
-    value:           'DEBUG|ASSERT|ASSUME|UNREACHABLE|FALLTHROUGH|DEPRECATED|STRINGIZE|_HAS_|_THROW|_REQUIRES|_DECLARE_|_VISIT_|_REGISTER_|_GENERATE_|_DETAIL_|_TIDY_|_MANAGE_PTR|_MATCHER|DEVICE_SHARED|_WORKAROUND_|_PP_'
+    value:           'DEBUG|ASSERT|ASSUME|UNREACHABLE|FALLTHROUGH|DEPRECATED|STRINGIZE|_HAS_|_THROW|_REQUIRES|_DECLARE_|_DEFINE_|_VISIT_|_REGISTER_|_GENERATE_|_DETAIL_|_TIDY_|_MANAGE_PTR|_MATCHER|DEVICE_SHARED|_WORKAROUND_|_PP_'
   - key:             modernize-loop-convert.MinConfidence
     value:           risky   
   - key:             modernize-loop-convert.NamingStyle

.github/workflows/performance.yaml

@@ -3,7 +3,7 @@ name: MIGraphX Performance Tests
 on:
   pull_request_target:
     branches: [develop]
-    types: [opened, synchronize, closed]
+    types: [opened, synchronize, closed, labeled]
   schedule:
     - cron: "0 7 * * 1-6"
 
@@ -41,14 +41,37 @@ on:
         description: -m for Max value; -s for Std dev; -r for Threshold file
         required: true
         default: '-r'
+      model_path:
+        description: Host path to NFS-backed models (ARC runner mount)
+        required: false
+        default: "/migraphx-models"
+      perf_workspace:
+        description: Writable NFS-backed workspace for reports/scratch (ARC runner mount)
+        required: false
+        default: "/mnt/migraphx-perf-work"
+      runner_label:
+        description: Self-hosted runner label (ARC scale set linux-migraphx-mi250-1)
+        required: false
+        default: "linux-migraphx-mi250-1"
 
 concurrency: 
-  group: "perftest-${{ github.head_ref ||  github.base_ref || 'schedule' }}"
+  group: "perftest-${{ github.head_ref || github.base_ref || 'schedule' }}"
   cancel-in-progress: true
 
+env:
+  # Defaults when not overridden by config.md or workflow_dispatch
+  MIGRAPHX_MODEL_PATH_DEFAULT: /migraphx-models
+  MIGRAPHX_PERF_WORKSPACE_DEFAULT: /mnt/migraphx-perf-work
+  MIGRAPHX_RUNNER_LABEL: linux-migraphx-mi250-1
+
 jobs:
   get_config:
     runs-on: ubuntu-latest
+    if: >
+      github.event_name == 'schedule' ||
+      github.event_name == 'workflow_dispatch' ||
+      github.event.action == 'closed' ||
+      contains(github.event.pull_request.labels.*.name, 'ok-to-test')
     outputs:
       rocm_version: ${{ steps.read_config.outputs.rocm_version }}
       utils_repo: ${{ steps.read_config.outputs.utils_repo }}
@@ -58,11 +81,18 @@ jobs:
       perf_number: ${{ steps.read_config.outputs.perf_number }}
       perf_flag: ${{ steps.read_config.outputs.perf_flag }}
       perf_timeout: ${{ steps.read_config.outputs.perf_timeout }}
+      model_path: ${{ steps.read_config.outputs.model_path }}
+      perf_workspace: ${{ steps.read_config.outputs.perf_workspace }}
+      runner_label: ${{ steps.read_config.outputs.runner_label }}
     steps:
       - name: checkout
         uses: actions/checkout@v4
       - name: read_config
         id: read_config
+        env:
+          MIGRAPHX_MODEL_PATH_DEFAULT: ${{ env.MIGRAPHX_MODEL_PATH_DEFAULT }}
+          MIGRAPHX_PERF_WORKSPACE_DEFAULT: ${{ env.MIGRAPHX_PERF_WORKSPACE_DEFAULT }}
+          MIGRAPHX_RUNNER_LABEL_DEFAULT: ${{ env.MIGRAPHX_RUNNER_LABEL }}
         run: |
           ROCM_VERSION=$(grep 'ROCM_VERSION' .github/workflows/config.md | cut -d "'" -f2)
           BENCHMARK_UTILS_REPO=$(grep 'BENCHMARK_UTILS_REPO' .github/workflows/config.md | cut -d "'" -f2)
@@ -72,6 +102,23 @@ jobs:
           RESULTS_TO_COMPARE=$(grep 'RESULTS_TO_COMPARE' .github/workflows/config.md | cut -d "'" -f2)
           CALCULATION_METHOD_FLAG=$(grep 'CALCULATION_METHOD_FLAG' .github/workflows/config.md | cut -d "'" -f2)
           PERFORMANCE_TEST_TIMEOUT=$(grep 'PERFORMANCE_TEST_TIMEOUT' .github/workflows/config.md | cut -d "'" -f2)
+
+          MODEL_PATH=""
+          PERF_WS=""
+          if grep -q 'MIGRAPHX_MODEL_PATH' .github/workflows/config.md 2>/dev/null; then
+            MODEL_PATH=$(grep 'MIGRAPHX_MODEL_PATH' .github/workflows/config.md | cut -d "'" -f2)
+          fi
+          if grep -q 'MIGRAPHX_PERF_WORKSPACE' .github/workflows/config.md 2>/dev/null; then
+            PERF_WS=$(grep 'MIGRAPHX_PERF_WORKSPACE' .github/workflows/config.md | cut -d "'" -f2)
+          fi
+          RUNNER_LABEL=""
+          if grep -q 'MIGRAPHX_RUNNER_LABEL' .github/workflows/config.md 2>/dev/null; then
+            RUNNER_LABEL=$(grep 'MIGRAPHX_RUNNER_LABEL' .github/workflows/config.md | cut -d "'" -f2)
+          fi
+          MODEL_PATH="${MODEL_PATH:-$MIGRAPHX_MODEL_PATH_DEFAULT}"
+          PERF_WS="${PERF_WS:-$MIGRAPHX_PERF_WORKSPACE_DEFAULT}"
+          RUNNER_LABEL="${RUNNER_LABEL:-$MIGRAPHX_RUNNER_LABEL_DEFAULT}"
+
           echo "rocm_version=$ROCM_VERSION"
           echo "rocm_version=$ROCM_VERSION" >> $GITHUB_OUTPUT
           echo "utils_repo=$BENCHMARK_UTILS_REPO" >> $GITHUB_OUTPUT
@@ -81,10 +128,13 @@ jobs:
           echo "perf_number=$RESULTS_TO_COMPARE" >> $GITHUB_OUTPUT
           echo "perf_flag=$CALCULATION_METHOD_FLAG" >> $GITHUB_OUTPUT
           echo "perf_timeout=$PERFORMANCE_TEST_TIMEOUT" >> $GITHUB_OUTPUT
-
+          echo "model_path=$MODEL_PATH" >> $GITHUB_OUTPUT
+          echo "perf_workspace=$PERF_WS" >> $GITHUB_OUTPUT
+          echo "runner_label=$RUNNER_LABEL" >> $GITHUB_OUTPUT
+
   call_reusable:
     needs: get_config
-    uses: ROCm/migraphx-benchmark/.github/workflows/perf-test.yml@main
+    uses: ROCm/migraphx-benchmark/.github/workflows/perf-test.yml@testing
     with:
       rocm_release: ${{ github.event.inputs.rocm_release || needs.get_config.outputs.rocm_version }}
       benchmark_utils_repo: ${{ github.event.inputs.benchmark_utils_repo || needs.get_config.outputs.utils_repo }}
@@ -94,7 +144,8 @@ jobs:
       result_number: ${{ github.event.inputs.result_number || needs.get_config.outputs.perf_number }}
       flags: ${{ github.event.inputs.flags || needs.get_config.outputs.perf_flag }}
       model_timeout: ${{ github.event.inputs.model_timeout || needs.get_config.outputs.perf_timeout }}
+      model_path: ${{ github.event.inputs.model_path || needs.get_config.outputs.model_path }}
+      perf_workspace: ${{ github.event.inputs.perf_workspace || needs.get_config.outputs.perf_workspace }}
+      runs_on: ${{ github.event.inputs.runner_label || needs.get_config.outputs.runner_label }}
     secrets:
-      gh_token: ${{ secrets.MIGRAPHX_BOT_TOKEN }}
-      mail_user: ${{ secrets.MAIL_USERNAME }}
-      mail_pass: ${{ secrets.MAIL_PASSWORD }}
+      BENCHMARK_UTILS_READ_TOKEN: ${{ secrets.BENCHMARK_UTILS_READ_TOKEN }}

.github/workflows/report.yaml

@@ -0,0 +1,209 @@
+name: Generate Performance Report
+on:
+  workflow_run:
+    workflows: ["MIGraphX Performance Tests"]
+    types: [completed]
+
+env:
+  REPORTS_DIR: migraphx-reports
+  MAIL_TO: dl.dl-migraphx-perfrun@amd.com
+  MAIL_BODY: Scheduled Performance test run on develop branch
+  PERFORMANCE_DIR: performance-backup
+  REPORTS_PATH: temp_reports
+
+jobs:
+  post-report:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: Download artifacts
+        id: dl_art
+        uses: actions/download-artifact@v4
+        continue-on-error: true
+        with:
+          name: performance-run-data
+          path: test-results
+          github-token: ${{ secrets.MIGRAPHX_BOT_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
+
+      - name: Extract context safely
+        if: steps.dl_art.outcome == 'success'
+        run: |
+          if [ -f test-results/pr_number.txt ]; then
+            PR_NUMBER=$(cat test-results/pr_number.txt)
+            if [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then
+              echo "PR_NUMBER=$PR_NUMBER" >> $GITHUB_ENV
+            fi
+          fi
+          if [ -f test-results/env_vars.env ]; then
+            RESULT_NUMBER=$(grep '^RESULT_NUMBER=' test-results/env_vars.env | cut -d= -f2)
+            if [[ "$RESULT_NUMBER" =~ ^[0-9]+$ ]]; then
+              echo "RESULT_NUMBER=$RESULT_NUMBER" >> $GITHUB_ENV
+            fi
+            FLAGS=$(grep '^FLAGS=' test-results/env_vars.env | cut -d= -f2)
+            case "$FLAGS" in
+              "-r"|"-m"|"-s")
+                echo "FLAGS=$FLAGS" >> $GITHUB_ENV
+                ;;
+              *)
+                echo "FLAGS=-r" >> $GITHUB_ENV
+                ;;
+            esac
+            ACCURACY_DIR=$(grep '^ACCURACY_DIR=' test-results/env_vars.env | cut -d= -f2)
+            if [[ "$ACCURACY_DIR" =~ ^accuracy ]]; then
+              echo "ACCURACY_DIR=$ACCURACY_DIR" >> $GITHUB_ENV
+            fi
+            GIT_SHA=$(grep '^GIT_SHA=' test-results/env_vars.env | cut -d= -f2)
+            if [[ "$GIT_SHA" =~ ^[A-Za-z0-9]+$ ]]; then
+              echo "GIT_SHA=$GIT_SHA" >> $GITHUB_ENV
+            fi
+            ORGANIZATION=$(grep '^ORGANIZATION=' test-results/env_vars.env | cut -d= -f2)
+            if [[ "$ORGANIZATION" =~ ^[A-Za-z0-9_-]+$ ]]; then
+              echo "ORGANIZATION=$ORGANIZATION" >> $GITHUB_ENV
+            else
+              echo "ORGANIZATION=AMD" >> $GITHUB_ENV
+            fi
+          fi      
+
+      - name: Checkout report's repo
+        if: steps.dl_art.outcome == 'success'
+        uses: actions/checkout@v4.3.1
+        with:
+          repository: ROCm/${{ env.REPORTS_DIR }}
+          path: ${{ env.REPORTS_DIR }}
+          token: ${{ secrets.MIGRAPHX_BOT_TOKEN }}
+
+      - name: Checkout utils repo
+        if: steps.dl_art.outcome == 'success'
+        uses: actions/checkout@v4.3.1
+        with:
+          repository: ROCm/migraphx-benchmark-utils
+          path: benchmark-utils
+          token: ${{ secrets.MIGRAPHX_BOT_TOKEN }}
+
+      - name: Setup Python
+        if: steps.dl_art.outcome == 'success'
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install python libraries
+        if: steps.dl_art.outcome == 'success'
+        run: pip install tabulate==0.8.9 pandas==1.4.2 numpy==1.22.3 openpyxl==3.1.2
+
+      - name: Execute report script
+        if: steps.dl_art.outcome == 'success' && github.event.workflow_run.event == 'schedule'
+        run: |
+          mkdir -p ${{ env.REPORTS_PATH }}
+          python3 benchmark-utils/scripts/report.py -t 'test-results' -r ${{ env.REPORTS_PATH }}
+          python3 benchmark-utils/scripts/history.py -n -t 'test-results' -r ${{ env.REPORTS_PATH }}
+          mkdir -p ${{ env.REPORTS_DIR }}/nightly_reports
+          cp -r ${{ env.REPORTS_PATH }}/results-* ${{ env.REPORTS_DIR }}/nightly_reports/ || true
+
+      - name: Copy performance results to repo - PRs
+        if: steps.dl_art.outcome == 'success' && github.event.workflow_run.event != 'schedule' && env.PR_NUMBER && env.PR_NUMBER != ''
+        run: |
+          mkdir -p ${{ env.REPORTS_DIR }}/performance_results/PRs/PR-${{ env.PR_NUMBER }}/${{ env.GIT_SHA }}
+          cp -r test-results/perf-*/. ${{ env.REPORTS_DIR }}/performance_results/PRs/PR-${{ env.PR_NUMBER }}/${{ env.GIT_SHA }} || true
+          if [ "${{ env.ACCURACY_DIR }}" != "" ]; then
+            mkdir -p ${{ env.REPORTS_DIR }}/performance_results/PRs/PR-${{ env.PR_NUMBER }}/${{ env.GIT_SHA }}/accuracy_logs
+            cp -r test-results/${{ env.ACCURACY_DIR }}/. ${{ env.REPORTS_DIR }}/performance_results/PRs/PR-${{ env.PR_NUMBER }}/${{ env.GIT_SHA }}/accuracy_logs || true
+          fi
+
+      - name: Copy performance results to repo - develop - nightly or manual
+        if: steps.dl_art.outcome == 'success' && (github.event.workflow_run.event == 'schedule' || github.event.workflow_run.event == 'workflow_dispatch')
+        run: |
+          mkdir -p ${{ env.REPORTS_DIR }}/performance_results/develop/${{ env.GIT_SHA }}
+          cp -r test-results/perf-*/. ${{ env.REPORTS_DIR }}/performance_results/develop/${{ env.GIT_SHA }} || true
+          if [ "${{ env.ACCURACY_DIR }}" != "" ]; then
+            mkdir -p ${{ env.REPORTS_DIR }}/performance_results/develop/${{ env.GIT_SHA }}/accuracy_logs
+            cp -r test-results/${{ env.ACCURACY_DIR }}/. ${{ env.REPORTS_DIR }}/performance_results/develop/${{ env.GIT_SHA }}/accuracy_logs || true
+          fi
+
+      - name: Push results to repo
+        if: steps.dl_art.outcome == 'success'
+        continue-on-error: true
+        run: |
+          cd ${{ env.REPORTS_DIR }}
+          git add .
+          git config --local user.email github-actions@github.com
+          git config --local user.name github-actions
+          git commit -m "Performance results run https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" -a || true
+          git push
+
+      - name: Execute comment script
+        if: steps.dl_art.outcome == 'success' && env.PR_NUMBER && env.PR_NUMBER != ''
+        run: |
+          if [ "${{ env.FLAGS }}" == "-r" ]; then
+            flagoptions="-r benchmark-utils/scripts/"
+          else
+            flagoptions="${{ env.FLAGS }}"
+          fi
+          python3 benchmark-utils/scripts/comment.py -t 'test-results' -n "${{ env.RESULT_NUMBER }}" $flagoptions -p
+
+      - name: Create a comment on PR
+        if: steps.dl_art.outcome == 'success' && env.PR_NUMBER && env.PR_NUMBER != ''
+        uses: marocchino/sticky-pull-request-comment@v2.9.0
+        with:
+          header: performance
+          number: ${{ env.PR_NUMBER }}
+          GITHUB_TOKEN: ${{ secrets.MIGRAPHX_BOT_TOKEN }}
+          path: test-results/temp.md
+          recreate: true
+
+      - name: Create accuracy comment on PR
+        if: steps.dl_art.outcome == 'success' && env.PR_NUMBER && env.PR_NUMBER != ''
+        uses: marocchino/sticky-pull-request-comment@v2.9.0
+        with:
+          header: accuracy
+          number: ${{ env.PR_NUMBER }}
+          GITHUB_TOKEN: ${{ secrets.MIGRAPHX_BOT_TOKEN }}
+          path: test-results/${{ env.ACCURACY_DIR }}/results.md
+          recreate: true
+
+      - name : Checkout for backup
+        if: steps.dl_art.outcome == 'success' && github.event.workflow_run.event == 'schedule'
+        uses: actions/checkout@v4.3.1
+        with:
+          repository: migraphx-benchmark/${{ env.PERFORMANCE_DIR }}
+          path: ${{ env.PERFORMANCE_DIR }}
+          token: ${{ secrets.MIGRAPHX_BOT_TOKEN }}
+
+      - name: Backup
+        if: steps.dl_art.outcome == 'success' && github.event.workflow_run.event == 'schedule'
+        run : |
+          mkdir -p ${{ env.PERFORMANCE_DIR }}/${{ env.ORGANIZATION }}
+          cp -r test-results/perf-* ${{ env.PERFORMANCE_DIR }}/${{ env.ORGANIZATION }} || true
+          cd ${{ env.PERFORMANCE_DIR }}
+          git add .
+          git status
+          git config --local user.email github-actions@github.com
+          git config --local user.name github-actions
+          git commit -m "Backup latest perf results" -a || true
+          git push
+
+
+      - name: Get latest report
+        id: last_report
+        if: steps.dl_art.outcome == 'success' && github.event.workflow_run.event == 'schedule'
+        run: |
+          cd ${{ env.REPORTS_DIR }}/nightly_reports
+          latest="$(readlink -f $(ls -tp | grep -v /$ | head -1))"
+          echo "latest=$latest" >> $GITHUB_OUTPUT
+
+      - name: Send mail
+        if: steps.dl_art.outcome == 'success' && github.event.workflow_run.event == 'schedule'
+        uses: dawidd6/action-send-mail@v3
+        with:
+          server_address: smtp.gmail.com
+          server_port: 465
+          username: ${{ secrets.MAIL_USERNAME }}
+          password: ${{ secrets.MAIL_PASSWORD }}
+          subject: "Nightly Performance run"
+          to: ${{ env.MAIL_TO }}
+          from: "GH Actions"
+          secure: true
+          body: ${{ env.MAIL_BODY }}
+          ignore_cert: true
+          attachments: ${{ steps.last_report.outputs.latest}}
+          priority: normal

.gitignore

@@ -55,11 +55,18 @@ _toc.yml
 # Directories to ignore (do not add trailing '/'s, they skip symlinks).
 #==============================================================================#
 # Nested build directory
-/build*
+/build
 
 # Downloaded models
 test/onnx/models
 
+# Python virtual env
+venv_rbuild
+
+# ONNX test generated files
+test/onnx/load_save_arg.msgpack
+test/onnx/migraphx_api_load_save_argument.msgpack
+
 # VS2017 and VSCode config files.
 .vscode
 .vs