knowledge/.github/workflows/database.yml at main · raphaelsty/knowledge · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
name: Extract data

on:
  push:
    branches:
      - main
  schedule:
    - cron: "0 1 * * *"

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - name: checkout repo content
        uses: actions/checkout@v4

      - name: install git lfs
        run: |
          curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
          sudo apt-get install git-lfs
          git lfs install

      - name: Install uv
        uses: astral-sh/setup-uv@v4

      - name: setup python
        run: uv python install 3.10

      - name: install python packages
        run: uv sync

      - name: track large files
        run: git lfs track "database/pipeline.pkl"

      - name: execute py script # run run.py to get the latest data
        env:
          TWITTER_AUTH_TOKEN: ${{ secrets.TWITTER_AUTH_TOKEN }}
          TWITTER_CT0: ${{ secrets.TWITTER_CT0 }}
          HACKERNEWS_PASSWORD: ${{ secrets.HACKERNEWS_PASSWORD }}
          HACKERNEWS_USERNAME: ${{ secrets.HACKERNEWS_USERNAME }}
          ZOTERO_API_KEY: ${{ secrets.ZOTERO_API_KEY }}
          ZOTERO_LIBRARY_ID: ${{ secrets.ZOTERO_LIBRARY_ID }}
          HUGGINGFACE_TOKEN: ${{ secrets.HUGGINGFACE_TOKEN }}
        run: uv run python run.py

      - name: commit files
        run: |
          git config --local user.email "action@github.com"
          git config --local user.name "GitHub Action"
          git add .gitattributes
          git add -A
          git commit -m "update data" -a

      - name: push changes
        uses: ad-m/github-push-action@v0.6.0
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          branch: main