Sync Web Scraping Projects #3
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync Web Scraping Projects | |
| on: | |
| schedule: | |
| - cron: "0 2 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| jobs: | |
| sync: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout main repository | |
| uses: actions/checkout@v4 | |
| - name: Configure Git | |
| run: | | |
| git config --global user.name "github-actions" | |
| git config --global user.email "github-actions@github.com" | |
| - name: Fetch scraping repositories | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| USERNAME: ${{ github.repository_owner }} | |
| MAIN_REPO: ${{ github.event.repository.name }} | |
| run: | | |
| mkdir -p projects | |
| cd projects | |
| repos=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ | |
| https://api.github.com/users/$USERNAME/repos?per_page=100 \ | |
| | jq -r '.[].name | select(test("scrap"; "i"))') | |
| for repo in $repos; do | |
| if [ "$repo" = "$MAIN_REPO" ]; then | |
| echo "Skipping main repository: $repo" | |
| continue | |
| fi | |
| if [ ! -d "$repo" ]; then | |
| echo "Cloning $repo" | |
| git clone --depth 1 https://github.com/$USERNAME/$repo.git | |
| rm -rf "$repo/.git" | |
| else | |
| echo "Updating $repo" | |
| cd "$repo" | |
| git pull | |
| cd .. | |
| fi | |
| done | |
| - name: Commit and push changes | |
| run: | | |
| git add projects | |
| git diff --cached --quiet || git commit -m "Auto-sync scraping projects" | |
| git push |