diff --git a/services/clsi/app/js/ProjectPersistenceManager.js b/services/clsi/app/js/ProjectPersistenceManager.js index 10fa70e3c3..7d4f071d2c 100644 --- a/services/clsi/app/js/ProjectPersistenceManager.js +++ b/services/clsi/app/js/ProjectPersistenceManager.js @@ -23,12 +23,14 @@ const fs = require('node:fs') // projectId -> timestamp mapping. const LAST_ACCESS = new Map() -async function refreshExpiryTimeout() { +async function collectDiskStats() { const paths = [ Settings.path.compilesDir, Settings.path.outputDir, Settings.path.clsiCacheDir, ] + + const diskStats = {} for (const path of paths) { try { const stats = await diskusage.check(path) @@ -37,23 +39,32 @@ async function refreshExpiryTimeout() { path, }) const lowDisk = diskAvailablePercent < 10 - - const lowerExpiry = ProjectPersistenceManager.EXPIRY_TIMEOUT * 0.9 - if (lowDisk && Settings.project_cache_length_ms / 2 < lowerExpiry) { - logger.warn( - { - stats, - newExpiryTimeoutInDays: (lowerExpiry / oneDay).toFixed(2), - }, - 'disk running low on space, modifying EXPIRY_TIMEOUT' - ) - ProjectPersistenceManager.EXPIRY_TIMEOUT = lowerExpiry - break - } + diskStats[path] = { stats, lowDisk } } catch (err) { logger.err({ err, path }, 'error getting disk usage') } } + return diskStats +} + +async function refreshExpiryTimeout() { + for (const [path, { stats, lowDisk }] of Object.entries( + await collectDiskStats() + )) { + const lowerExpiry = ProjectPersistenceManager.EXPIRY_TIMEOUT * 0.9 + if (lowDisk && Settings.project_cache_length_ms / 2 < lowerExpiry) { + logger.warn( + { + path, + stats, + newExpiryTimeoutInDays: (lowerExpiry / oneDay).toFixed(2), + }, + 'disk running low on space, modifying EXPIRY_TIMEOUT' + ) + ProjectPersistenceManager.EXPIRY_TIMEOUT = lowerExpiry + break + } + } } module.exports = ProjectPersistenceManager = { @@ -108,6 +119,13 @@ module.exports = ProjectPersistenceManager = { } ) }) + + // Collect disk stats frequently to have them ready the next time /metrics is scraped (60s +- jitter). + setInterval(() => { + collectDiskStats().catch(err => { + logger.err({ err }, 'low level error collecting disk stats') + }) + }, 50_000) }, markProjectAsJustAccessed(projectId, callback) {