Adjust/lustre quota from cronjob to grid daemon

.github/workflows/python-c-ext-sanity-check.yml

@@ -5,6 +5,11 @@
 
 name: Python C-Extension Sanity Checks
 
+# Ignore the following paths
+# Format: PATH1|PATH2|PATH3
+env:
+  PATHS_IGNORE_SPLINT: mig/src/lustreclient/.*$
+
 on:
   # Triggers the workflow on push or pull request events but only for this git branch
   push:
@@ -24,6 +29,7 @@ on:
       - 'mig/apache/**'
       - 'mig/bin/**'
       - 'mig/java-bin/**'
+      - 'mig/src/lustreclient/**'
       - '**/*.py'
       - '**/*.js'
     branches:
@@ -52,6 +58,7 @@ on:
       - 'mig/apache/**'
       - 'mig/bin/**'
       - 'mig/java-bin/**'
+      - 'mig/src/lustreclient/**'
       - '**/*.py'
       - '**/*.js'
     branches:
@@ -71,16 +78,19 @@ jobs:
     if: ${{ false }} # Disabled until we figure out how to fix system headers
     runs-on: ubuntu-latest
     steps:
+      - name: Update packages
+        run: |
+          DEBIAN_FRONTEND='noninteractive' sudo apt update -y
       - name: Set up latest stable python 3.x
         uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - name: Set up git, findutils and make with apt
         run: |
-          sudo apt install -y git findutils make
+          DEBIAN_FRONTEND='noninteractive' sudo apt install -y git findutils make
       - name: Install dependencies
         run: |
-          sudo apt install -y libnss3-dev libpam-dev splint
+          DEBIAN_FRONTEND='noninteractive' sudo apt install -y libnss3-dev libpam-dev splint
       # We may need git installed to get a full repo clone rather than unpacked archive
       - name: Check out source repository
         uses: actions/checkout@v4
@@ -90,9 +100,9 @@ jobs:
         run: |
           # NOTE: we only run splint error check for changed C files to limit noise
           # NOTE: point splint to Ubuntu's custom /usr/include/python3.x for Python.h
-          echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$')"
+          echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.PATHS_IGNORE_SPLINT }}' | grep -E '\.c$')"
           # NOTE: show splint warnings but don't fail unless it found critical errors
-          git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list $(python3-config --includes) &> splint.log || true
+          git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.PATHS_IGNORE_SPLINT }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list $(python3-config --includes) &> splint.log || true
           [ ! -e splint.log ] || cat splint.log
           [ ! -e splint.log ] || ! grep -q ' Cannot continue' splint.log
 
@@ -101,12 +111,15 @@ jobs:
     name: Sanity check c-extension module code in latest Ubuntu LTS python3
     runs-on: ubuntu-24.04
     steps:
+      - name: Update packages
+        run: |
+          DEBIAN_FRONTEND='noninteractive' sudo apt update -y
       - name: Set up git, findutils and make with apt
         run: |
-          sudo apt install -y git findutils make
+          DEBIAN_FRONTEND='noninteractive' sudo apt install -y git findutils make
       - name: Install dependencies
         run: |
-          sudo apt install -y python3-dev libnss3-dev libpam-dev splint
+          DEBIAN_FRONTEND='noninteractive' sudo apt install -y python3-dev libnss3-dev libpam-dev splint
       # We may need git installed to get a full repo clone rather than unpacked archive
       - name: Check out source repository
         uses: actions/checkout@v4
@@ -116,10 +129,10 @@ jobs:
         run: |
           # NOTE: we only run splint error check for changed C files to limit noise
           # NOTE: point splint to Ubuntu's custom /usr/include/python3.x for Python.h
-          echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$')"
+          echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.PATHS_IGNORE_SPLINT }}' | grep -E '\.c$')"
           # NOTE: splint complains about NATIVE_TSS_KEY_T in system header here
           # NOTE: show splint warnings but don't fail unless it found critical errors
-          git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list -DNATIVE_TSS_KEY_T=char $(python3-config --includes) &> splint.log || true
+          git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.PATHS_IGNORE_SPLINT }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list -DNATIVE_TSS_KEY_T=char $(python3-config --includes) &> splint.log || true
           [ ! -e splint.log ] || cat splint.log
           [ ! -e splint.log ] || ! grep -q ' Cannot continue' splint.log
 
@@ -135,6 +148,9 @@ jobs:
     container:
       image: rockylinux/rockylinux:9
     steps:
+      - name: Update packages
+        run: |
+          dnf update -y
       - name: Set up git, findutils, make and python3 with dnf and make the latter default
         run: |
           dnf install -y git findutils make python3 python3-pip python-unversioned-command
@@ -156,10 +172,10 @@ jobs:
           # NOTE: perms are not right inside container so repeat what checkout module does.
           git config --global --add safe.directory "$PWD"
           # NOTE: we only run splint error check for changed C files to limit noise
-          echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$')"
+          echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.PATHS_IGNORE_SPLINT }}' | grep -E '\.c$')"
           echo "with splint from $(which splint)"
           ls -l /bin/splint
           # NOTE: show splint warnings but don't fail unless it found critical errors
-          git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list &> splint.log || true
+          git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.PATHS_IGNORE_SPLINT }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list &> splint.log || true
           [ ! -e splint.log ] || cat splint.log
           [ ! -e splint.log ] || ! grep -q ' Cannot continue' splint.log

mig/install/MiGserver-template.conf

@@ -550,6 +550,7 @@ default_mount_re = SSHFS-2.X-1
 
 [QUOTA]
 backend = __QUOTA_BACKEND__
+update_interval = __QUOTA_UPDATE_INTERVAL__
 user_limit = __QUOTA_USER_LIMIT__
 vgrid_limit = __QUOTA_VGRID_LIMIT__
 

mig/install/generateconfs.py

@@ -1,3 +1,3 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
@@ -20,7 +20,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 # -- END_HEADER ---
 #
@@ -252,6 +252,7 @@
         'seafile_seafhttp_port',
         'seafile_client_port',
         'seafile_quota',
+        'quota_update_interval',
         'quota_user_limit',
         'quota_vgrid_limit',
         'wwwserve_max_bytes',

mig/install/migrid-init.d-deb-template

@@ -43,6 +43,13 @@ if [ -z "$PYTHONPATH" ]; then
 else
     PYTHONPATH=${MIG_PATH}:$PYTHONPATH
 fi
+# Append '/usr/local/(s)bin' to PATH if it exists otherwise define PATH
+if [ -z "$PATH" ]; then
+    export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+else
+    export PATH="$PATH:/usr/local/sbin:/usr/local/bin"
+fi
+
 # you probably do not want to modify these...
 PID_DIR=${PID_DIR:-/var/run}
 MIG_LOG=${MIG_STATE}/log
@@ -62,13 +69,14 @@ MIG_FTPS=${MIG_CODE}/server/grid_ftps.py
 MIG_NOTIFY=${MIG_CODE}/server/grid_notify.py
 MIG_IMNOTIFY=${MIG_CODE}/server/grid_imnotify.py
 MIG_VMPROXY=${MIG_CODE}/server/grid_vmproxy.py
+MIG_QUOTA=${MIG_CODE}/server/grid_quota.py
 MIG_CHKUSERROOT=${MIG_CODE}/server/chkuserroot.py
 MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py
 
 show_usage() {
     echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]"
     echo "where daemon is left out for all or given along with DAEMON as one of the following"
-    echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)"
+    echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)"
 }
 
 check_enabled() {
@@ -264,6 +272,18 @@ start_vmproxy() {
 	log_end_msg 1 || true
     fi
 }
+start_quota() {
+    check_enabled "quota" || return 0
+    DAEMON_PATH=${MIG_QUOTA}
+    SHORT_NAME=$(basename ${DAEMON_PATH})
+    PID_FILE="$PID_DIR/${SHORT_NAME}.pid"
+    log_daemon_msg "Starting MiG quota daemon" ${SHORT_NAME} || true
+    if start-stop-daemon --start --quiet --oknodo --pidfile ${PID_FILE} --make-pidfile --user root --chuid root --background --name ${SHORT_NAME} --startas ${DAEMON_PATH} ; then
+    log_end_msg 0 || true
+    else
+    log_end_msg 1 || true
+    fi
+}
 start_sftpsubsys() {
     check_enabled "sftp_subsys" || return 0
     DAEMON_PATH=${MIG_SFTPSUBSYS}
@@ -292,6 +312,7 @@ start_all() {
     start_notify
     start_imnotify
     start_vmproxy
+    start_quota
     return 0
 }
 
@@ -524,6 +545,19 @@ stop_vmproxy() {
 	log_end_msg 1 || true
     fi
 }
+stop_quota() {
+    check_enabled "quota" || return 0
+    DAEMON_PATH=${MIG_QUOTA}
+    SHORT_NAME=$(basename ${DAEMON_PATH})
+    PID_FILE="$PID_DIR/${SHORT_NAME}.pid"
+    log_daemon_msg "Stopping MiG quota" ${SHORT_NAME} || true
+    if start-stop-daemon --stop --quiet --oknodo --pidfile ${PID_FILE} ; then
+        rm -f ${PID_FILE}
+	log_end_msg 0 || true
+    else
+	log_end_msg 1 || true
+    fi
+}
 stop_sftpsubsys_workers() {
     DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER}
     SHORT_NAME=$(basename ${DAEMON_PATH})
@@ -563,6 +597,7 @@ stop_all() {
     stop_notify
     stop_imnotify
     stop_vmproxy
+    stop_quota
     return 0
 }
 
@@ -735,6 +770,18 @@ reload_vmproxy() {
 	log_end_msg 1 || true
     fi
 }
+reload_quota() {
+    check_enabled "quota" || return 0
+    DAEMON_PATH=${MIG_QUOTA}
+    SHORT_NAME=$(basename ${DAEMON_PATH})
+    PID_FILE="$PID_DIR/${SHORT_NAME}.pid"
+    log_daemon_msg "Reloading MiG quota" ${SHORT_NAME} || true
+    if start-stop-daemon --stop --signal HUP --quiet --oknodo --pidfile ${PID_FILE} ; then
+	log_end_msg 0 || true
+    else
+	log_end_msg 1 || true
+    fi
+}
 reload_sftpsubsys_workers() {
     DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER}
     SHORT_NAME=$(basename ${DAEMON_PATH})
@@ -787,6 +834,7 @@ reload_all() {
     reload_notify
     reload_imnotify
     reload_vmproxy
+    reload_quota
     # Apache helpers to verify proper chrooting
     reload_chkuserroot
     reload_chksidroot
@@ -891,6 +939,13 @@ status_vmproxy() {
     PID_FILE="$PID_DIR/${SHORT_NAME}.pid"
     status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME}
 }
+status_quota() {
+    check_enabled "quota" || return 0
+    DAEMON_PATH=${MIG_QUOTA}
+    SHORT_NAME=$(basename ${DAEMON_PATH})
+    PID_FILE="$PID_DIR/${SHORT_NAME}.pid"
+    status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME}
+}
 status_sftpsubsys_workers() {
     DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER}
     SHORT_NAME=$(basename ${DAEMON_PATH})
@@ -929,6 +984,7 @@ status_all() {
     status_notify
     status_imnotify
     status_vmproxy
+    status_quota
     return 0
 }
 
@@ -940,7 +996,7 @@ test -f ${MIG_SCRIPT} || exit 0
 
 # Force valid target
 case "$2" in
-    script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)
+    script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)
         TARGET="$2"
 	;;
     '')