github-team/.github/workflows/commit_logging.yml at main · google/github-team · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
# Workflow to log commit information to a BigQuery table
# BQ Schema:
# [
#     {
#       "mode": "REQUIRED",
#       "name": "commit_sha",
#       "type": "STRING"
#     },
#     {
#       "mode": "REQUIRED",
#       "name": "commit_author",
#       "type": "STRING"
#     },
#     {
#       "mode": "REQUIRED",
#       "name": "commit_message",
#       "type": "STRING"
#     },
#     {
#       "mode": "REQUIRED",
#       "name": "commit_date_time",
#       "type": "STRING"
#     }
#   ]
name: 'commit_logging'
on:
  workflow_dispatch:
jobs:
  log_commits:
    runs-on: 'ubuntu-latest'
    permissions:
      contents: 'read'
      id-token: 'write'
    steps:
      - id: 'checkout'
        uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4
      - id: 'auth'
        name: 'Authenticate to Google Cloud'
        uses: 'google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193' # ratchet:google-github-actions/auth@v2.1.10
        with:
          workload_identity_provider: 'projects/580543901798/locations/global/workloadIdentityPools/becle1b-wif-64840b/providers/becle1b-wif-64840b'
          service_account: 'github-automation-bot@bradegler-commit-log-exp-1b.iam.gserviceaccount.com'
          create_credentials_file: true
      - uses: 'actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065' # ratchet:actions/setup-python@v5
        with:
          python-version: '3.11.2'
      - name: 'install_dependencies'
        shell: 'bash'
        run: |
          pip install google-cloud-bigquery
      - name: 'read_commits'
        shell: 'bash'
        run: |-
          git log --pretty=format:"%H%x7c%aE%x7c%s%x7c%ai%x7c" > data.psv
          echo "commit_sha|commit_author|commit_message|commit_date_time|" > labels.psv

          echo "{\"commits\": [" > output.psv
          jq -nRc '
              def objectify($keys):
                  [$keys, .] | transpose | map({(.[0]): .[1]}) | add;

              (input|split("|")) as $keys
              | inputs | split("|") | objectify($keys)
              | del(..|select(. == ""))
              ' labels.psv data.psv | sed 's/}/},/g' | sed '$s/,$//' >> output.psv

          echo "]}" >> output.psv
      - name: 'read_log_write_bq'
        shell: python {0}
        run: |-
          import json

          from google.cloud import bigquery

          table_id = "bradegler-commit-log-exp-1b.commit_log_exp.commit_log"

          client = bigquery.Client(project='bradegler-commit-log-exp-1b')

          with open('output.psv', 'r') as file:
              data = json.load(file)

              print(data["commits"])
              errors = client.insert_rows_json(table_id, data["commits"])
              if errors == []:
                  print("New rows have been added.")
              else:
                  print("Encountered errors while inserting rows: {}".format(errors))
                  exit(1)