Feature: git mode

Dockerfile

@@ -15,7 +15,7 @@ COPY . .
 
 FROM --platform=$TARGETPLATFORM node:lts-alpine as runner
 
-RUN apk add --no-cache tini
+RUN apk add --no-cache tini git
 
 WORKDIR /app
 

README.md

@@ -21,7 +21,7 @@ The application requires a configuration file `./config.json` to be able to run.
 | informationLevel | The level of severity of the information outputted to the log. The severity can be 0 (everything), 1 (warnings and errors), 2 (only errors). Defaults to 2. You can override this per runtime by setting the `INFORMATION_LEVEL` environment variable. |
 | port | The port the server application will listen to. Defaults to 80. |
 | secureToken | The secret token used to validate the GitHub webhook post to /update. See [GitHub Developer - Securing your webhooks](https://developer.github.com/webhooks/securing/) for more information. |
-| token | Personal access token used to gain access to GitHub API. The token scope is requires only access to public repositories. See [GitHub Help - Creating a personal access token for the command line](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) for more information. |
+| token | Personal access token used to gain access to GitHub API (unused in git-only mode). The token scope is requires only access to public repositories. See [GitHub Help - Creating a personal access token for the command line](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) for more information. |
 | cleanInterval | How often the server should check if it is time to clean. Defaults to every 2 hours (Note that the cleanup job|
 | cleanThreshold | The amount of downloaded branches that will trigger the clean up job. Defaults to 1000 |
 | tmpLifetime | How many hours since last request to keep a branch when cleaning up |
@@ -48,6 +48,16 @@ INFORMATION_LEVEL=0 npm start
 
 Environment variables take precedence over values in `config.json`.
 
+### Git-only runtime cache
+This server now uses a local git clone under `tmp/git-cache/repo` for all source retrieval.
+The GitHub API is not used in production.
+
+Environment variables:
+- `GIT_SYNC_INTERVAL_MS`: Interval for `git fetch` + `git checkout -B master origin/master` (default 30 minutes).
+- `GIT_REF_CLEAN_INTERVAL_MS`: Interval for cleaning inactive refs in `tmp/` (default 30 minutes).
+
+The cleanup process keeps active refs based on the `info.json` timestamp in each `tmp/{ref}` directory.
+
 ## Run the application
 Open a CLI and run the command: `npm start`
 

app/dashboards.js

@@ -4,7 +4,7 @@ const { join } = require('node:path')
 const { stat, opendir, readFile, writeFile, symlink } = require('node:fs/promises')
 const { existsSync } = require('node:fs')
 const { downloadSourceFolder, getBranchInfo, getCommitInfo } = require('./download')
-const { compileTypeScript, log } = require('./utilities')
+const { compileTypeScript, log, updateBranchAccess } = require('./utilities')
 const { build } = require('@highcharts/highcharts-assembler/src/build')
 const { JobQueue } = require('./JobQueue')
 
@@ -41,7 +41,7 @@ async function assembleDashboards (pathCacheDirectory, commit) {
   try {
     build({
       // TODO: Remove trailing slash when assembler has fixed path concatenation
-      base: jsMastersDirectory + '/',
+      base: `${jsMastersDirectory}/`,
       output: pathOutputFolder,
       pretty: false,
       version: commit,
@@ -70,7 +70,7 @@ async function assembleDashboards (pathCacheDirectory, commit) {
           await modifyFiles(join(dirPath, dirent.name))
         } else if (dirent.name.endsWith('.src.js')) {
           const contents = await readFile(join(dirPath, dirent.name), 'utf-8')
-                    const toReplace = 'code.highcharts.com.*' + commit + '\/' // eslint-disable-line
+                    const toReplace = `code.highcharts.com.*${commit}\/` // eslint-disable-line
           if (contents) {
             await writeFile(
               join(dirPath, dirent.name),
@@ -142,7 +142,10 @@ async function dashboardsHandler (req, res, next) {
   }
 
   const pathCacheDirectory = join(PATH_TMP_DIRECTORY, commit)
-  const downloadURL = 'https://raw.githubusercontent.com/highcharts/highcharts/'
+
+  await updateBranchAccess(pathCacheDirectory).catch(error => {
+    log(1, `Failed to update access for ${commit}: ${error.message}`)
+  })
 
   try {
     await stat(join(pathCacheDirectory, 'js'))
@@ -155,7 +158,7 @@ async function dashboardsHandler (req, res, next) {
           func: downloadSourceFolder,
           args: [
             pathCacheDirectory,
-            downloadURL,
+            null,
             commit
           ]
         }
@@ -199,7 +202,7 @@ async function dashboardsHandler (req, res, next) {
     async () => {
       await Promise.resolve().then(() => queue.addJob(
         'compile',
-        commit + filepath,
+        `${commit}/${filepath}`,
         {
           func: compileTypeScript,
           args: [
@@ -210,19 +213,19 @@ async function dashboardsHandler (req, res, next) {
           ]
         }
       )).catch(error => {
-        log(2, `Failed to enqueue TypeScript compile for ${commit}${filepath}: ${error.message}`)
+        log(2, `Failed to enqueue TypeScript compile for ${commit}/${filepath}: ${error.message}`)
         return handleQueueError(error)
       })
 
       await Promise.resolve().then(() => queue.addJob(
         'compile',
-        commit + filepath,
+        `${commit}/${filepath}`,
         {
           func: assembleDashboards,
           args: [pathCacheDirectory, commit]
         }
       )).catch(error => {
-        log(2, `Failed to enqueue dashboard assembly for ${commit}${filepath}: ${error.message}`)
+        log(2, `Failed to enqueue dashboard assembly for ${commit}/${filepath}: ${error.message}`)
         return handleQueueError(error)
       })