[clsi] collect disk usage more frequently (#24314)

GitOrigin-RevId: 0c5b3ebeadec6d2ac70b7adc77935aa0cea92ffc

services/clsi/app/js/ProjectPersistenceManager.js

@@ -23,12 +23,14 @@ const fs = require('node:fs')
 // projectId -> timestamp mapping.
 const LAST_ACCESS = new Map()
 
-async function refreshExpiryTimeout() {
+async function collectDiskStats() {
   const paths = [
     Settings.path.compilesDir,
     Settings.path.outputDir,
     Settings.path.clsiCacheDir,
   ]
+
+  const diskStats = {}
   for (const path of paths) {
     try {
       const stats = await diskusage.check(path)
@@ -37,23 +39,32 @@ async function refreshExpiryTimeout() {
         path,
       })
       const lowDisk = diskAvailablePercent < 10
-
-      const lowerExpiry = ProjectPersistenceManager.EXPIRY_TIMEOUT * 0.9
-      if (lowDisk && Settings.project_cache_length_ms / 2 < lowerExpiry) {
-        logger.warn(
-          {
-            stats,
-            newExpiryTimeoutInDays: (lowerExpiry / oneDay).toFixed(2),
-          },
-          'disk running low on space, modifying EXPIRY_TIMEOUT'
-        )
-        ProjectPersistenceManager.EXPIRY_TIMEOUT = lowerExpiry
-        break
-      }
+      diskStats[path] = { stats, lowDisk }
     } catch (err) {
       logger.err({ err, path }, 'error getting disk usage')
     }
   }
+  return diskStats
+}
+
+async function refreshExpiryTimeout() {
+  for (const [path, { stats, lowDisk }] of Object.entries(
+    await collectDiskStats()
+  )) {
+    const lowerExpiry = ProjectPersistenceManager.EXPIRY_TIMEOUT * 0.9
+    if (lowDisk && Settings.project_cache_length_ms / 2 < lowerExpiry) {
+      logger.warn(
+        {
+          path,
+          stats,
+          newExpiryTimeoutInDays: (lowerExpiry / oneDay).toFixed(2),
+        },
+        'disk running low on space, modifying EXPIRY_TIMEOUT'
+      )
+      ProjectPersistenceManager.EXPIRY_TIMEOUT = lowerExpiry
+      break
+    }
+  }
 }
 
 module.exports = ProjectPersistenceManager = {
@@ -108,6 +119,13 @@ module.exports = ProjectPersistenceManager = {
         }
       )
     })
+
+    // Collect disk stats frequently to have them ready the next time /metrics is scraped (60s +- jitter).
+    setInterval(() => {
+      collectDiskStats().catch(err => {
+        logger.err({ err }, 'low level error collecting disk stats')
+      })
+    }, 50_000)
   },
 
   markProjectAsJustAccessed(projectId, callback) {