fix(discord): improve advisory lock health check to verify lock ownership

- Change health check from SELECT 1 to querying pg_locks to verify lock is still held
- Add checks for classid=0, objid=lockId, pid=pg_backend_pid(), granted=true
- Reduce health check interval from 30s to 10s for faster detection
- Add unit test to verify pg_locks query structure

Author: brandonkachen

packages/internal/src/db/__tests__/advisory-lock.test.ts

@@ -116,7 +116,7 @@ describe('advisory-lock', () => {
         const result = await tryAcquireAdvisoryLock(ADVISORY_LOCK_IDS.DISCORD_BOT)
 
         expect(setIntervalSpy).toHaveBeenCalledTimes(1)
-        expect(setIntervalSpy.mock.calls[0][1]).toBe(30_000) // 30 seconds
+        expect(setIntervalSpy.mock.calls[0][1]).toBe(10_000) // 10 seconds
 
         await result.handle?.release()
       })
@@ -384,9 +384,17 @@ describe('advisory-lock', () => {
         expect(lostCallback).toHaveBeenCalledTimes(1)
       })
 
-      it('should do nothing when health check succeeds', async () => {
-        // All calls succeed
-        mockConnection.tagged.mockResolvedValue([{ acquired: true }])
+      it('should do nothing when health check succeeds and lock is still held', async () => {
+        // First call acquires lock, subsequent calls check lock ownership
+        let callCount = 0
+        mockConnection.tagged.mockImplementation(() => {
+          callCount++
+          if (callCount === 1) {
+            return Promise.resolve([{ acquired: true }])
+          }
+          // Health check returns that lock is still held
+          return Promise.resolve([{ held: true }])
+        })
 
         let healthCheckCallback: (() => Promise<void>) | null = null
         setIntervalSpy.mockImplementation((callback: () => Promise<void>) => {
@@ -408,6 +416,84 @@ describe('advisory-lock', () => {
         // Clean up
         await result.handle?.release()
       })
+
+      it('should trigger onLost when lock is no longer held', async () => {
+        // First call acquires lock, subsequent calls show lock is not held
+        let callCount = 0
+        mockConnection.tagged.mockImplementation(() => {
+          callCount++
+          if (callCount === 1) {
+            return Promise.resolve([{ acquired: true }])
+          }
+          // Health check returns that lock is no longer held (e.g., another process took it)
+          return Promise.resolve([{ held: false }])
+        })
+
+        let healthCheckCallback: (() => Promise<void>) | null = null
+        setIntervalSpy.mockImplementation((callback: () => Promise<void>) => {
+          healthCheckCallback = callback
+          return 123 as unknown as NodeJS.Timeout
+        })
+
+        const result = await tryAcquireAdvisoryLock(ADVISORY_LOCK_IDS.DISCORD_BOT)
+
+        const lostCallback = mock(() => {})
+        result.handle?.onLost(lostCallback)
+
+        // Trigger health check
+        await healthCheckCallback!()
+
+        expect(lostCallback).toHaveBeenCalledTimes(1)
+        expect(consoleErrorSpy).toHaveBeenCalledWith(
+          'Advisory lock health check failed - lock no longer held',
+        )
+      })
+
+      it('should query pg_locks with correct structure in health check', async () => {
+        // First call acquires lock, second call is the health check
+        let callCount = 0
+        mockConnection.tagged.mockImplementation(() => {
+          callCount++
+          if (callCount === 1) {
+            return Promise.resolve([{ acquired: true }])
+          }
+          return Promise.resolve([{ held: true }])
+        })
+
+        let healthCheckCallback: (() => Promise<void>) | null = null
+        setIntervalSpy.mockImplementation((callback: () => Promise<void>) => {
+          healthCheckCallback = callback
+          return 123 as unknown as NodeJS.Timeout
+        })
+
+        const result = await tryAcquireAdvisoryLock(ADVISORY_LOCK_IDS.DISCORD_BOT)
+
+        // Trigger health check
+        await healthCheckCallback!()
+
+        // Verify the health check query was called (second call)
+        expect(mockConnection.tagged).toHaveBeenCalledTimes(2)
+
+        // Get the health check query (second call)
+        const [queryStrings, lockIdArg] = mockConnection.tagged.mock.calls[1]
+        const fullQuery = queryStrings.join('')
+
+        // Verify the query checks pg_locks with all required conditions
+        expect(fullQuery).toContain('SELECT EXISTS')
+        expect(fullQuery).toContain('FROM pg_locks')
+        expect(fullQuery).toContain("locktype = 'advisory'")
+        expect(fullQuery).toContain('classid = 0')
+        expect(fullQuery).toContain('objid =')
+        expect(fullQuery).toContain('pid = pg_backend_pid()')
+        expect(fullQuery).toContain('granted = true')
+        expect(fullQuery).toContain('as held')
+
+        // Verify the lock ID is passed as a parameter
+        expect(lockIdArg).toBe(ADVISORY_LOCK_IDS.DISCORD_BOT)
+
+        // Clean up
+        await result.handle?.release()
+      })
     })
 
     describe('edge cases', () => {

packages/internal/src/db/advisory-lock.ts

@@ -12,7 +12,7 @@ export const ADVISORY_LOCK_IDS = {
 
 export type AdvisoryLockId = (typeof ADVISORY_LOCK_IDS)[keyof typeof ADVISORY_LOCK_IDS]
 
-const HEALTH_CHECK_INTERVAL_MS = 30_000 // 30 seconds
+const HEALTH_CHECK_INTERVAL_MS = 10_000 // 10 seconds
 
 export interface LockHandle {
   /** Register a callback to be called if the lock is lost (connection dies) */
@@ -67,11 +67,27 @@ export async function tryAcquireAdvisoryLock(lockId: AdvisoryLockId): Promise<{
       }
     }
 
-    // Start health check interval
+    // Start health check interval - verify we still hold the lock, not just connection liveness
     healthCheckTimer = setInterval(async () => {
       if (isReleased) return
       try {
-        await connection`SELECT 1`
+        // Query pg_locks to verify we still hold this specific advisory lock
+        // This catches cases where the lock was lost but connection stayed alive
+        const result = await connection`
+          SELECT EXISTS (
+            SELECT 1 FROM pg_locks 
+            WHERE locktype = 'advisory' 
+            AND classid = 0
+            AND objid = ${lockId}
+            AND pid = pg_backend_pid()
+            AND granted = true
+          ) as held
+        `
+        const stillHeld = result[0]?.held === true
+        if (!stillHeld) {
+          console.error('Advisory lock health check failed - lock no longer held')
+          triggerLost()
+        }
       } catch {
         console.error('Advisory lock health check failed - connection lost')
         triggerLost()