feat: Add ops endpoints and refactor sync workflow #284
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| name: 'Sync Files' | ||
| description: 'Generic file/directory synchronization with filtering and exclusions' | ||
| inputs: | ||
| type: | ||
| description: 'Type identifier for the sync operation (for artifact naming)' | ||
| required: true | ||
| title: | ||
| description: 'Human-readable title for reports' | ||
| required: true | ||
| source_path: | ||
| description: 'Source file or directory path' | ||
| required: true | ||
| target_path: | ||
| description: 'Target file or directory path' | ||
| required: true | ||
| filter_pattern: | ||
| description: 'Regex pattern to filter files (only for directories)' | ||
| required: false | ||
| default: '.*' | ||
| exclude_files: | ||
| description: 'Comma-separated list of filenames to exclude' | ||
| required: false | ||
| default: 'README,CHANGELOG,.gitignore,.gitkeep' | ||
| sanitize_script: | ||
| description: 'Optional script path to run after sync (for sanitization/post-processing)' | ||
| required: false | ||
| default: '' | ||
| outputs: | ||
| added: | ||
| description: 'Number of files added' | ||
| value: ${{ steps.sync.outputs.added }} | ||
| updated: | ||
| description: 'Number of files updated' | ||
| value: ${{ steps.sync.outputs.updated }} | ||
| deleted: | ||
| description: 'Number of files deleted' | ||
| value: ${{ steps.sync.outputs.deleted }} | ||
| total: | ||
| description: 'Total number of changes' | ||
| value: ${{ steps.sync.outputs.total }} | ||
| runs: | ||
| using: 'composite' | ||
| steps: | ||
| - name: Prepare config file | ||
| if: inputs.type == 'config' | ||
| shell: bash | ||
| run: | | ||
| # For config type, handle special case: | ||
| # 1. Copy config.yaml.example to temp location as config.yaml | ||
| # 2. Run sanitization on the renamed file | ||
| # 3. Use the sanitized file as source for sync | ||
|
|
||
| if [[ -f "${{ inputs.source_path }}" ]]; then | ||
| # Create temp directory | ||
| TEMP_DIR=$(mktemp -d) | ||
| echo "TEMP_CONFIG_DIR=$TEMP_DIR" >> $GITHUB_ENV | ||
|
|
||
| # Copy and rename config.yaml.example to config.yaml | ||
| cp "${{ inputs.source_path }}" "$TEMP_DIR/config.yaml" | ||
|
|
||
| # Run sanitization if script is provided | ||
| if [[ -n "${{ inputs.sanitize_script }}" ]] && [[ -f "${{ inputs.sanitize_script }}" ]]; then | ||
| echo "Sanitizing config file..." | ||
| bash "${{ inputs.sanitize_script }}" "$TEMP_DIR/config.yaml" | ||
| fi | ||
|
|
||
| # Update source path for sync | ||
| echo "CONFIG_SOURCE=$TEMP_DIR/config.yaml" >> $GITHUB_ENV | ||
| else | ||
| echo "Config source file not found: ${{ inputs.source_path }}" | ||
| exit 1 | ||
| fi | ||
|
|
||
| - name: Sync files | ||
| id: sync | ||
| shell: bash | ||
| run: | | ||
| # Use prepared config source if it's a config type, otherwise use original source | ||
| if [[ "${{ inputs.type }}" == "config" ]] && [[ -n "$CONFIG_SOURCE" ]]; then | ||
| SOURCE_PATH="$CONFIG_SOURCE" | ||
| else | ||
| SOURCE_PATH="${{ inputs.source_path }}" | ||
| fi | ||
|
|
||
| ${{ github.action_path }}/sync.sh \ | ||
| "${{ inputs.type }}" \ | ||
| "${{ inputs.title }}" \ | ||
| "$SOURCE_PATH" \ | ||
| "${{ inputs.target_path }}" \ | ||
| "${{ inputs.filter_pattern }}" \ | ||
| "${{ inputs.exclude_files }}" | ||
|
|
||
| - name: Cleanup config temp directory | ||
| if: inputs.type == 'config' && always() | ||
| shell: bash | ||
| run: | | ||
| if [[ -n "$TEMP_CONFIG_DIR" ]] && [[ -d "$TEMP_CONFIG_DIR" ]]; then | ||
| rm -rf "$TEMP_CONFIG_DIR" | ||
| fi | ||
|
|
||
| - name: Upload artifacts | ||
| uses: actions/upload-artifact@v4 | ||
| if: always() | ||
| with: | ||
| name: synced-${{ inputs.type }} | ||
| path: artifacts/ | ||
| retention-days: 1 | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
| @@ -0,0 +1,238 @@ | ||||||||
| #!/bin/bash | ||||||||
| set -e | ||||||||
|
|
||||||||
| # File/directory synchronization script for documentation | ||||||||
| # Syncs .md and .mdx files with pattern filtering | ||||||||
|
|
||||||||
| # Get parameters | ||||||||
| TYPE="$1" | ||||||||
| TITLE="$2" | ||||||||
| SOURCE_PATH="$3" | ||||||||
| TARGET_PATH="$4" | ||||||||
| FILTER_PATTERN="${5:-.*}" | ||||||||
| EXCLUDE_FILES="${6:-README,CHANGELOG,.gitignore,.gitkeep}" | ||||||||
|
|
||||||||
| # Initialize metrics and lists | ||||||||
| added=0 | ||||||||
| updated=0 | ||||||||
| deleted=0 | ||||||||
| added_files=() | ||||||||
| updated_files=() | ||||||||
| deleted_files=() | ||||||||
|
|
||||||||
| # Convert exclusions to array | ||||||||
| IFS=',' read -ra EXCLUSIONS <<< "$EXCLUDE_FILES" | ||||||||
|
|
||||||||
| # Check if file should be excluded | ||||||||
| is_excluded() { | ||||||||
| local filename="$1" | ||||||||
| for excluded in "${EXCLUSIONS[@]}"; do | ||||||||
| [[ "$filename" == "$excluded" ]] && return 0 | ||||||||
| done | ||||||||
| return 1 | ||||||||
| } | ||||||||
|
|
||||||||
| # Check if filename matches pattern | ||||||||
| matches_pattern() { | ||||||||
| local filename="$1" | ||||||||
| local pattern="$2" | ||||||||
|
|
||||||||
| # Try perl first (supports PCRE including negative lookahead) | ||||||||
| if command -v perl >/dev/null 2>&1; then | ||||||||
| echo "$filename" | perl -ne "exit 0 if /^($pattern)\$/; exit 1" | ||||||||
| return $? | ||||||||
| fi | ||||||||
|
|
||||||||
| # Fallback to grep -E (doesn't support negative lookahead) | ||||||||
| echo "$filename" | grep -E "^($pattern)$" >/dev/null 2>&1 | ||||||||
| return $? | ||||||||
| } | ||||||||
|
|
||||||||
| # Handle single file sync | ||||||||
| if [[ -f "$SOURCE_PATH" ]]; then | ||||||||
| echo "Syncing file: $SOURCE_PATH -> $TARGET_PATH" | ||||||||
| mkdir -p "$(dirname "$TARGET_PATH")" | ||||||||
|
|
||||||||
| if [[ -f "$TARGET_PATH" ]]; then | ||||||||
| if ! cmp -s "$SOURCE_PATH" "$TARGET_PATH"; then | ||||||||
| cp "$SOURCE_PATH" "$TARGET_PATH" | ||||||||
| updated=1 | ||||||||
| updated_files+=("$(basename "$TARGET_PATH")") | ||||||||
| echo "Updated: $(basename "$TARGET_PATH")" | ||||||||
| fi | ||||||||
| else | ||||||||
| cp "$SOURCE_PATH" "$TARGET_PATH" | ||||||||
| added=1 | ||||||||
| added_files+=("$(basename "$TARGET_PATH")") | ||||||||
| echo "Added: $(basename "$TARGET_PATH")" | ||||||||
| fi | ||||||||
|
|
||||||||
| # Handle directory sync | ||||||||
| elif [[ -d "$SOURCE_PATH" ]]; then | ||||||||
| echo "Syncing directory: $SOURCE_PATH -> $TARGET_PATH" | ||||||||
| echo "Filter pattern: $FILTER_PATTERN" | ||||||||
| echo "Exclude files: $EXCLUDE_FILES" | ||||||||
|
|
||||||||
| # Check if source directory has any files | ||||||||
| if [[ -z "$(ls -A "$SOURCE_PATH" 2>/dev/null)" ]]; then | ||||||||
| echo "Warning: Source directory is empty: $SOURCE_PATH" | ||||||||
| # Create empty target to ensure it exists | ||||||||
| mkdir -p "$TARGET_PATH" | ||||||||
| added=0 | ||||||||
| updated=0 | ||||||||
| deleted=0 | ||||||||
| else | ||||||||
| mkdir -p "$TARGET_PATH" | ||||||||
|
|
||||||||
| # Create temp directory with normalized source files | ||||||||
| TEMP_SOURCE=$(mktemp -d) | ||||||||
| trap "rm -rf $TEMP_SOURCE" EXIT | ||||||||
|
|
||||||||
|
Comment on lines
+89
to
+90
Contributor
There was a problem hiding this comment. 🛠️ Refactor suggestion Fix trap quoting to avoid SC2064 and accidental early expansion Use single quotes so the variable expands at signal time and quote the arg inside. - trap "rm -rf $TEMP_SOURCE" EXIT
+ trap 'rm -rf "$TEMP_SOURCE"' EXIT📝 Committable suggestion
Suggested change
🧰 Tools🪛 Shellcheck (0.10.0)[warning] 89-89: Use single quotes, otherwise this expands now rather than when signalled. (SC2064) 🤖 Prompt for AI Agents |
||||||||
| echo "Preparing source files..." | ||||||||
|
|
||||||||
| # Count source files | ||||||||
| source_count=0 | ||||||||
|
|
||||||||
| # Process and filter source files into temp directory | ||||||||
| shopt -s nullglob # Handle case when no .md or .mdx files exist | ||||||||
| for file in "$SOURCE_PATH"/*.md "$SOURCE_PATH"/*.mdx; do | ||||||||
| [[ ! -f "$file" ]] && continue | ||||||||
|
|
||||||||
| basename_file=$(basename "$file") | ||||||||
| basename_no_ext="${basename_file%.*}" | ||||||||
|
|
||||||||
| # Skip excluded files | ||||||||
| is_excluded "$basename_no_ext" && continue | ||||||||
|
|
||||||||
| # Skip if doesn't match pattern | ||||||||
| matches_pattern "$basename_no_ext" "$FILTER_PATTERN" || continue | ||||||||
|
|
||||||||
| # Copy to temp with .mdx extension | ||||||||
| cp "$file" "$TEMP_SOURCE/${basename_no_ext}.mdx" | ||||||||
| source_count=$((source_count + 1)) | ||||||||
| echo " Processing: $basename_file" | ||||||||
| done | ||||||||
| shopt -u nullglob | ||||||||
|
|
||||||||
| echo "Processed $source_count source files" | ||||||||
|
|
||||||||
| # Track existing target files (using simple array for compatibility) | ||||||||
| existing_files=() | ||||||||
| if [[ -d "$TARGET_PATH" ]]; then | ||||||||
| while IFS= read -r file; do | ||||||||
| [[ -n "$file" ]] && existing_files+=("$file") | ||||||||
| done < <(find "$TARGET_PATH" -type f -name "*.mdx" 2>/dev/null) | ||||||||
| fi | ||||||||
|
|
||||||||
| # Sync from temp source to target | ||||||||
| for source_file in "$TEMP_SOURCE"/*.mdx; do | ||||||||
| [[ ! -f "$source_file" ]] && continue | ||||||||
|
|
||||||||
| basename_file=$(basename "$source_file") | ||||||||
| target_file="$TARGET_PATH/$basename_file" | ||||||||
|
|
||||||||
| if [[ -f "$target_file" ]]; then | ||||||||
| if ! cmp -s "$source_file" "$target_file"; then | ||||||||
| cp "$source_file" "$target_file" | ||||||||
| updated=$((updated + 1)) | ||||||||
| updated_files+=("$basename_file") | ||||||||
| echo "Updated: $basename_file" | ||||||||
| fi | ||||||||
| # Remove from existing files array | ||||||||
| new_existing=() | ||||||||
| for ef in "${existing_files[@]}"; do | ||||||||
| [[ "$(basename "$ef")" != "$basename_file" ]] && new_existing+=("$ef") | ||||||||
| done | ||||||||
| existing_files=("${new_existing[@]}") | ||||||||
| else | ||||||||
| cp "$source_file" "$target_file" | ||||||||
| added=$((added + 1)) | ||||||||
| added_files+=("$basename_file") | ||||||||
| echo "Added: $basename_file" | ||||||||
| fi | ||||||||
| done | ||||||||
|
|
||||||||
| # Delete orphaned files only if we had source files to sync | ||||||||
| if [[ "$source_count" -gt 0 ]]; then | ||||||||
| # Safe to delete orphaned files (preserve _meta.json) | ||||||||
| for target_file in "${existing_files[@]}"; do | ||||||||
| if [[ -f "$target_file" && "$(basename "$target_file")" != "_meta.json" ]]; then | ||||||||
| rm "$target_file" | ||||||||
| deleted=$((deleted + 1)) | ||||||||
| deleted_files+=("$(basename "$target_file")") | ||||||||
| echo "Deleted: $(basename "$target_file")" | ||||||||
| fi | ||||||||
| done | ||||||||
| else | ||||||||
| echo "Warning: No source files matched filter; skipping deletion phase for safety" | ||||||||
| echo " This prevents accidental mass deletion of target files" | ||||||||
| fi | ||||||||
| fi # End of non-empty directory check | ||||||||
| else | ||||||||
| echo "Source not found: $SOURCE_PATH" | ||||||||
| exit 1 | ||||||||
| fi | ||||||||
|
|
||||||||
| # Output metrics | ||||||||
| total=$((added + updated + deleted)) | ||||||||
| echo "added=$added" >> "$GITHUB_OUTPUT" | ||||||||
| echo "updated=$updated" >> "$GITHUB_OUTPUT" | ||||||||
| echo "deleted=$deleted" >> "$GITHUB_OUTPUT" | ||||||||
| echo "total=$total" >> "$GITHUB_OUTPUT" | ||||||||
|
|
||||||||
| # Create artifact directory with report and synced files | ||||||||
| mkdir -p artifacts | ||||||||
|
|
||||||||
| # Create summary report at root of artifacts | ||||||||
| REPORT_FILE="artifacts/sync_report_${TYPE}.md" | ||||||||
| cat > "$REPORT_FILE" <<EOF | ||||||||
| ## ${TITLE} | ||||||||
|
|
||||||||
| ### Summary | ||||||||
| - **Added**: $added files | ||||||||
| - **Updated**: $updated files | ||||||||
| - **Deleted**: $deleted files | ||||||||
| - **Total changes**: $total | ||||||||
|
|
||||||||
| ### Synced Files Location | ||||||||
| Path: \`${TARGET_PATH}\` | ||||||||
| EOF | ||||||||
|
|
||||||||
| # Add file lists to report | ||||||||
| if [[ ${#added_files[@]} -gt 0 ]]; then | ||||||||
| echo "" >> "$REPORT_FILE" | ||||||||
| echo "### Added Files" >> "$REPORT_FILE" | ||||||||
| for file in "${added_files[@]}"; do | ||||||||
| echo "- $file" >> "$REPORT_FILE" | ||||||||
| done | ||||||||
| fi | ||||||||
|
|
||||||||
| if [[ ${#updated_files[@]} -gt 0 ]]; then | ||||||||
| echo "" >> "$REPORT_FILE" | ||||||||
| echo "### Updated Files" >> "$REPORT_FILE" | ||||||||
| for file in "${updated_files[@]}"; do | ||||||||
| echo "- $file" >> "$REPORT_FILE" | ||||||||
| done | ||||||||
| fi | ||||||||
|
|
||||||||
| if [[ ${#deleted_files[@]} -gt 0 ]]; then | ||||||||
| echo "" >> "$REPORT_FILE" | ||||||||
| echo "### Deleted Files" >> "$REPORT_FILE" | ||||||||
| for file in "${deleted_files[@]}"; do | ||||||||
| echo "- $file" >> "$REPORT_FILE" | ||||||||
| done | ||||||||
| fi | ||||||||
|
|
||||||||
| # Copy whatever is in target to artifacts | ||||||||
| if [[ -f "$TARGET_PATH" ]]; then | ||||||||
| # Single file | ||||||||
| mkdir -p "artifacts/$(dirname "$TARGET_PATH")" | ||||||||
| cp "$TARGET_PATH" "artifacts/${TARGET_PATH}" | ||||||||
|
|
||||||||
| elif [[ -d "$TARGET_PATH" ]]; then | ||||||||
| # Directory - just copy everything | ||||||||
| mkdir -p "artifacts/${TARGET_PATH}" | ||||||||
| cp -r "$TARGET_PATH"/* "artifacts/${TARGET_PATH}/" 2>/dev/null || true | ||||||||
| fi | ||||||||
|
|
||||||||
| echo "Sync completed: $total changes" | ||||||||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.