diff --git a/services/clsi/app/js/ContentCacheManager.js b/services/clsi/app/js/ContentCacheManager.js index 2036057064..c1ae2f71a2 100644 --- a/services/clsi/app/js/ContentCacheManager.js +++ b/services/clsi/app/js/ContentCacheManager.js @@ -10,7 +10,36 @@ const Settings = require('@overleaf/settings') const OError = require('@overleaf/o-error') const pLimit = require('p-limit') const { parseXrefTable } = require('../lib/pdfjs/parseXrefTable') -const { TimedOutError } = require('./Errors') +const { QueueLimitReachedError, TimedOutError } = require('./Errors') +const workerpool = require('workerpool') +const Metrics = require('@overleaf/metrics') + +let WORKER_POOL +// NOTE: Check for main thread to avoid recursive start of pool. +if (Settings.pdfCachingEnableWorkerPool && workerpool.isMainThread) { + WORKER_POOL = workerpool.pool(Path.join(__dirname, 'ContentCacheWorker.js'), { + // Cap number of worker threads. + maxWorkers: Settings.pdfCachingWorkerPoolSize, + // Warmup workers. + minWorkers: Settings.pdfCachingWorkerPoolSize, + // Limit queue back-log + maxQueueSize: Settings.pdfCachingWorkerPoolBackLogLimit, + }) + setInterval(() => { + const { + totalWorkers, + busyWorkers, + idleWorkers, + pendingTasks, + activeTasks, + } = WORKER_POOL.stats() + Metrics.gauge('pdf_caching_total_workers', totalWorkers) + Metrics.gauge('pdf_caching_busy_workers', busyWorkers) + Metrics.gauge('pdf_caching_idle_workers', idleWorkers) + Metrics.gauge('pdf_caching_pending_tasks', pendingTasks) + Metrics.gauge('pdf_caching_active_tasks', activeTasks) + }, 15 * 1000) +} /** * @@ -20,7 +49,68 @@ const { TimedOutError } = require('./Errors') * @param {number} compileTime */ async function update(contentDir, filePath, size, compileTime) { + if (Settings.pdfCachingEnableWorkerPool) { + return await updateOtherEventLoop(contentDir, filePath, size, compileTime) + } else { + return await updateSameEventLoop(contentDir, filePath, size, compileTime) + } +} + +/** + * + * @param {String} contentDir path to directory where content hash files are cached + * @param {String} filePath the pdf file to scan for streams + * @param {number} size the pdf size + * @param {number} compileTime + */ +async function updateOtherEventLoop(contentDir, filePath, size, compileTime) { + const timeout = getMaxOverhead(compileTime) + try { + return await WORKER_POOL.exec('doUpdateInternalNoDeadline', [ + contentDir, + filePath, + size, + ]).timeout(timeout) + } catch (e) { + if (e instanceof workerpool.Promise.TimeoutError) { + throw new TimedOutError('context-lost-in-worker') + } + if (e.message.includes('Max queue size of ')) { + throw new QueueLimitReachedError() + } + throw e + } +} + +/** + * + * @param {String} contentDir path to directory where content hash files are cached + * @param {String} filePath the pdf file to scan for streams + * @param {number} size the pdf size + * @param {number} compileTime + */ +async function updateSameEventLoop(contentDir, filePath, size, compileTime) { const checkDeadline = getDeadlineChecker(compileTime) + return doUpdateInternal(contentDir, filePath, size, checkDeadline) +} + +/** + * + * @param {String} contentDir path to directory where content hash files are cached + * @param {String} filePath the pdf file to scan for streams + * @param {number} size the pdf size + */ +async function doUpdateInternalNoDeadline(contentDir, filePath, size) { + return doUpdateInternal(contentDir, filePath, size, () => {}) +} +/** + * + * @param {String} contentDir path to directory where content hash files are cached + * @param {String} filePath the pdf file to scan for streams + * @param {number} size the pdf size + * @param {function} checkDeadline + */ +async function doUpdateInternal(contentDir, filePath, size, checkDeadline) { const ranges = [] const newRanges = [] // keep track of hashes expire old ones when they reach a generation > N. @@ -233,14 +323,18 @@ async function writePdfStream(dir, hash, buffer) { } } -function getDeadlineChecker(compileTime) { - const maxOverhead = Math.min( +function getMaxOverhead(compileTime) { + return Math.min( // Adding 10s to a 40s compile time is OK. // Adding 1s to a 3s compile time is OK. Math.max(compileTime / 4, 1000), // Adding 30s to a 120s compile time is not OK, limit to 10s. Settings.pdfCachingMaxProcessingTime ) +} + +function getDeadlineChecker(compileTime) { + const maxOverhead = getMaxOverhead(compileTime) const deadline = Date.now() + maxOverhead let lastStage = { stage: 'start', now: Date.now() } @@ -269,5 +363,6 @@ module.exports = { update: callbackify(update), promises: { update, + doUpdateInternalNoDeadline, }, } diff --git a/services/clsi/app/js/ContentCacheMetrics.js b/services/clsi/app/js/ContentCacheMetrics.js index 3550de70e8..a7830982da 100644 --- a/services/clsi/app/js/ContentCacheMetrics.js +++ b/services/clsi/app/js/ContentCacheMetrics.js @@ -22,6 +22,9 @@ function emitPdfStats(stats, timings) { if (stats['pdf-caching-timed-out']) { Metrics.inc('pdf-caching-timed-out') } + if (stats['pdf-caching-queue-limit-reached']) { + Metrics.inc('pdf-caching-queue-limit-reached') + } if (timings['compute-pdf-caching']) { emitPdfCachingStats(stats, timings) } else { diff --git a/services/clsi/app/js/ContentCacheWorker.js b/services/clsi/app/js/ContentCacheWorker.js new file mode 100644 index 0000000000..9ecb5b16dc --- /dev/null +++ b/services/clsi/app/js/ContentCacheWorker.js @@ -0,0 +1,4 @@ +const workerpool = require('workerpool') +const ContentCacheManager = require('./ContentCacheManager') + +workerpool.worker(ContentCacheManager.promises) diff --git a/services/clsi/app/js/Errors.js b/services/clsi/app/js/Errors.js index 6b66c23421..07cb48baad 100644 --- a/services/clsi/app/js/Errors.js +++ b/services/clsi/app/js/Errors.js @@ -31,9 +31,11 @@ var AlreadyCompilingError = function (message) { } AlreadyCompilingError.prototype.__proto__ = Error.prototype +class QueueLimitReachedError extends OError {} class TimedOutError extends OError {} module.exports = Errors = { + QueueLimitReachedError, TimedOutError, NotFoundError, FilesOutOfSyncError, diff --git a/services/clsi/app/js/OutputCacheManager.js b/services/clsi/app/js/OutputCacheManager.js index af85f49661..c9d346fa44 100644 --- a/services/clsi/app/js/OutputCacheManager.js +++ b/services/clsi/app/js/OutputCacheManager.js @@ -26,7 +26,7 @@ const Metrics = require('./Metrics') const OutputFileOptimiser = require('./OutputFileOptimiser') const ContentCacheManager = require('./ContentCacheManager') -const { TimedOutError } = require('./Errors') +const { QueueLimitReachedError, TimedOutError } = require('./Errors') module.exports = OutputCacheManager = { CONTENT_SUBDIR: 'content', @@ -288,6 +288,11 @@ module.exports = OutputCacheManager = { pdfSize, timings.compile, function (err, result) { + if (err && err instanceof QueueLimitReachedError) { + logger.warn({ err, outputDir }, 'pdf caching queue limit reached') + stats['pdf-caching-queue-limit-reached'] = 1 + return callback(null, outputFiles) + } if (err && err instanceof TimedOutError) { logger.warn( { err, outputDir, stats, timings }, diff --git a/services/clsi/buildscript.txt b/services/clsi/buildscript.txt index 5cc225fd87..8e5ca6e94e 100644 --- a/services/clsi/buildscript.txt +++ b/services/clsi/buildscript.txt @@ -2,7 +2,7 @@ clsi --data-dirs=cache,compiles,db,output --dependencies= --docker-repos=gcr.io/overleaf-ops ---env-add=ENABLE_PDF_CACHING="true" +--env-add=ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true" --env-pass-through=TEXLIVE_IMAGE --has-custom-cloudbuild=True --node-version=12.22.3 diff --git a/services/clsi/config/settings.defaults.js b/services/clsi/config/settings.defaults.js index e36334df1b..cd22f7bed4 100644 --- a/services/clsi/config/settings.defaults.js +++ b/services/clsi/config/settings.defaults.js @@ -76,6 +76,12 @@ module.exports = { parseInt(process.env.PDF_CACHING_MIN_CHUNK_SIZE, 10) || 1024, pdfCachingMaxProcessingTime: parseInt(process.env.PDF_CACHING_MAX_PROCESSING_TIME, 10) || 10 * 1000, + pdfCachingEnableWorkerPool: + process.env.PDF_CACHING_ENABLE_WORKER_POOL === 'true', + pdfCachingWorkerPoolSize: + parseInt(process.env.PDF_CACHING_WORKER_POOL_SIZE, 10) || 4, + pdfCachingWorkerPoolBackLogLimit: + parseInt(process.env.PDF_CACHING_WORKER_POOL_BACK_LOG_LIMIT, 10) || 40, } if (process.env.ALLOWED_COMPILE_GROUPS) { diff --git a/services/clsi/docker-compose.ci.yml b/services/clsi/docker-compose.ci.yml index 245059ad98..d70197f669 100644 --- a/services/clsi/docker-compose.ci.yml +++ b/services/clsi/docker-compose.ci.yml @@ -30,6 +30,7 @@ services: NODE_OPTIONS: "--unhandled-rejections=strict" TEXLIVE_IMAGE: ENABLE_PDF_CACHING: "true" + PDF_CACHING_ENABLE_WORKER_POOL: "true" command: npm run test:acceptance:_run diff --git a/services/clsi/docker-compose.yml b/services/clsi/docker-compose.yml index ee68a7c0ee..0c3b4eb52a 100644 --- a/services/clsi/docker-compose.yml +++ b/services/clsi/docker-compose.yml @@ -39,5 +39,6 @@ services: NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" ENABLE_PDF_CACHING: "true" + PDF_CACHING_ENABLE_WORKER_POOL: "true" command: npm run --silent test:acceptance diff --git a/services/clsi/package-lock.json b/services/clsi/package-lock.json index 86789cafbc..ac32db7be4 100644 --- a/services/clsi/package-lock.json +++ b/services/clsi/package-lock.json @@ -1703,7 +1703,52 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", - "dev": true + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } }, "charenc": { "version": "0.0.2", @@ -1755,7 +1800,43 @@ "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", "dev": true, "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", "wrap-ansi": "^7.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "string-width": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + } + }, + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.0" + } + } } }, "clone-response": { @@ -2246,14 +2327,40 @@ "function-bind": "^1.1.1", "get-intrinsic": "^1.1.1", "has": "^1.0.3", + "has-symbols": "^1.0.2", + "is-callable": "^1.2.3", "is-negative-zero": "^2.0.1", "is-regex": "^1.1.3", + "is-string": "^1.0.6", "object-inspect": "^1.10.3", "object-keys": "^1.1.1", "object.assign": "^4.1.2", "string.prototype.trimend": "^1.0.4", "string.prototype.trimstart": "^1.0.4", "unbox-primitive": "^1.0.1" + }, + "dependencies": { + "has-symbols": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", + "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==", + "dev": true + }, + "is-callable": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.4.tgz", + "integrity": "sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==", + "dev": true + }, + "is-string": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", + "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", + "dev": true, + "requires": { + "has-tostringtag": "^1.0.0" + } + } } }, "es-to-primitive": { @@ -2760,7 +2867,18 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==", - "dev": true + "dev": true, + "requires": { + "estraverse": "^5.1.0" + }, + "dependencies": { + "estraverse": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", + "dev": true + } + } }, "esrecurse": { "version": "4.3.0", @@ -2954,7 +3072,19 @@ "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==", "dev": true, "requires": { - "flatted": "^3.1.0" + "flatted": "^3.1.0", + "rimraf": "^3.0.2" + }, + "dependencies": { + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + } } }, "flatted": { @@ -3154,7 +3284,18 @@ "version": "13.10.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.10.0.tgz", "integrity": "sha512-piHC3blgLGFjvOuMmWZX60f+na1lXFDhQXBf1UYp2fXPXqvEUbOhNwi6BsQ0bQishwedgnjkwv1d9zKf+MWw3g==", - "dev": true + "dev": true, + "requires": { + "type-fest": "^0.20.2" + }, + "dependencies": { + "type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "dev": true + } + } }, "google-auth-library": { "version": "6.0.6", @@ -3419,6 +3560,23 @@ "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", "dev": true }, + "has-tostringtag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", + "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "dev": true, + "requires": { + "has-symbols": "^1.0.2" + }, + "dependencies": { + "has-symbols": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", + "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==", + "dev": true + } + } + }, "has-unicode": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", @@ -3771,7 +3929,16 @@ "integrity": "sha512-qSVXFz28HM7y+IWX6vLCsexdlvzT1PJNFSBuaQLQ5o0IEw8UDYW6/2+eCMVyIsbM8CNLX2a/QWmSpyxYEHY7CQ==", "dev": true, "requires": { - "call-bind": "^1.0.2" + "call-bind": "^1.0.2", + "has-symbols": "^1.0.2" + }, + "dependencies": { + "has-symbols": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", + "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==", + "dev": true + } } }, "is-stream": { @@ -4333,6 +4500,7 @@ "requires": { "anymatch": "~3.1.1", "braces": "~3.0.2", + "fsevents": "~2.3.1", "glob-parent": "~5.1.0", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", @@ -4386,6 +4554,13 @@ "path-exists": "^4.0.0" } }, + "fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "optional": true + }, "glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", @@ -4491,6 +4666,12 @@ "requires": { "has-flag": "^4.0.0" } + }, + "workerpool": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz", + "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==", + "dev": true } } }, @@ -5850,13 +6031,38 @@ "dev": true, "requires": { "ansi-styles": "^4.0.0", - "astral-regex": "^2.0.0" + "astral-regex": "^2.0.0", + "is-fullwidth-code-point": "^3.0.0" }, "dependencies": { "ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "dev": true } } @@ -6093,7 +6299,9 @@ "ajv": "^8.0.1", "lodash.clonedeep": "^4.5.0", "lodash.truncate": "^4.4.2", - "slice-ansi": "^4.0.0" + "slice-ansi": "^4.0.0", + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0" }, "dependencies": { "ajv": { @@ -6103,9 +6311,48 @@ "dev": true, "requires": { "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2", "uri-js": "^4.2.2" } + }, + "ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true + }, + "string-width": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + } + }, + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.0" + } } } }, @@ -6671,16 +6918,78 @@ "dev": true }, "workerpool": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz", - "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==", - "dev": true + "version": "6.1.5", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.5.tgz", + "integrity": "sha512-XdKkCK0Zqc6w3iTxLckiuJ81tiD/o5rBE/m+nXpRCB+/Sq4DqkfXZ/x0jW02DG1tGsfUGXbTJyZDP+eu67haSw==" }, "wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true + "dev": true, + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "string-width": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + } + }, + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.0" + } + } + } }, "wrappy": { "version": "1.0.2", @@ -6731,8 +7040,43 @@ "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", + "string-width": "^4.2.0", "y18n": "^5.0.5", "yargs-parser": "^20.2.2" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "string-width": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + } + }, + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.0" + } + } } }, "yargs-parser": { diff --git a/services/clsi/package.json b/services/clsi/package.json index 0b2b011d49..a063b5f52f 100644 --- a/services/clsi/package.json +++ b/services/clsi/package.json @@ -35,6 +35,7 @@ "lodash": "^4.17.21", "logger-sharelatex": "^2.2.0", "mysql": "^2.18.1", + "workerpool": "^6.1.5", "p-limit": "^3.1.0", "pdfjs-dist": "^2.7.570", "request": "^2.88.2",