diff --git a/server-ce/bin/import_pr_patch.sh b/server-ce/bin/import_pr_patch.sh new file mode 100755 index 0000000000..b25e917e2a --- /dev/null +++ b/server-ce/bin/import_pr_patch.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +for PR in "$@"; do + gh pr diff "$PR" --patch \ + | node -e 'const blob = require("fs").readFileSync("/dev/stdin", "utf-8"); console.log(blob.replace(/From [\s\S]+?\d+ files? changed,.+/g, ""))' \ + > "pr_$PR.patch" +done diff --git a/server-ce/hotfix/5.5.3/Dockerfile b/server-ce/hotfix/5.5.3/Dockerfile new file mode 100644 index 0000000000..33d77af255 --- /dev/null +++ b/server-ce/hotfix/5.5.3/Dockerfile @@ -0,0 +1,25 @@ +FROM sharelatex/sharelatex:5.5.2 + +# ../../bin/import_pr_patch.sh 27147 27173 27230 27240 27249 27257 27273 27397 +# Remove CE tests +# Remove tests +# Remove cloudbuild changes +# Remove SaaS changes +# Fixup package.json and toolbar-items.tsx +# Fix cron paths +COPY *.patch . +RUN --mount=type=cache,target=/root/.cache \ + --mount=type=cache,target=/root/.npm \ + --mount=type=cache,target=/overleaf/services/web/node_modules/.cache,id=server-ce-webpack-cache \ + --mount=type=tmpfs,target=/tmp true \ + && bash -ec 'for p in *.patch; do echo "=== Applying $p ==="; patch -p1 < "$p" && rm $p; done' \ + && npm audit --audit-level=high \ + && node genScript compile | bash \ + && npm prune --omit=dev \ + && apt remove -y linux-libc-dev + +# ../../bin/import_pr_patch.sh 27476 +# Remove tests +# Remove SaaS changes +COPY pr_27476.patch-stage-2 . +RUN patch -p1 < pr_27476.patch-stage-2 && rm pr_27476.patch-stage-2 diff --git a/server-ce/hotfix/5.5.3/NOTES.md b/server-ce/hotfix/5.5.3/NOTES.md new file mode 100644 index 0000000000..71f3f185a5 --- /dev/null +++ b/server-ce/hotfix/5.5.3/NOTES.md @@ -0,0 +1,54 @@ +# Get the base container running +docker build -t base . + +CONTAINER_NAME=new + +# Start the container +docker run -t -i --entrypoint /bin/bash --name $CONTAINER_NAME base + +# Clean any existing directories +rm -rf /tmp/{a,b} + +# Take snapshot of initial container +mkdir /tmp/a ; docker export $CONTAINER_NAME | tar --exclude node_modules -x -C /tmp/a --strip-components=1 overleaf + +# In the container, run the following commands +docker exec -i $CONTAINER_NAME /bin/bash <<'EOF' +npm install -g json +json -I -f package.json -c 'this.overrides["swagger-tools"].multer="2.0.2"' +json -I -f package.json -c 'this.overrides["request@2.88.2"]["form-data"]="2.5.5"' +json -I -f package.json -c 'this.overrides["superagent@7.1.6"] ??= {}' +json -I -f package.json -c 'this.overrides["superagent@7.1.6"]["form-data"]="4.0.4"' +json -I -f package.json -c 'this.overrides["superagent@3.8.3"] ??= {}' +json -I -f package.json -c 'this.overrides["superagent@3.8.3"]["form-data"]="2.5.5"' + +npm uninstall -w libraries/metrics @google-cloud/opentelemetry-cloud-trace-exporter @google-cloud/profiler +npm uninstall -w libraries/logger @google-cloud/logging-bunyan +npm uninstall -w services/web @slack/webhook contentful @contentful/rich-text-types @contentful/rich-text-html-renderer +npm uninstall -w services/history-v1 @google-cloud/secret-manager + +npm uninstall -w services/web "@node-saml/passport-saml" +npm install -w services/web "@node-saml/passport-saml@^5.1.0" + +npm uninstall -w services/web multer +npm install -w services/web "multer@2.0.2" + +npm uninstall -w services/history-v1 swagger-tools +npm install -w services/history-v1 swagger-tools@0.10.4 + +npm uninstall -w services/clsi request +npm install -w services/clsi request@2.88.2 +npm install + +npm audit --audit-level=high +EOF + +# Take snapshot of final container +mkdir /tmp/b ; docker export $CONTAINER_NAME | tar --exclude node_modules -x -C /tmp/b --strip-components=1 overleaf + +# Find the diff excluding node modules directories +# The sec_ prefix ensures it applies after pr_* patches. +(cd /tmp ; diff -u -x 'node_modules' -r a/ b/) > sec-npm.patch + +# In the docker file we also need to remove linux-libc-dev +apt remove -y linux-libc-dev diff --git a/server-ce/hotfix/5.5.3/multer.patch b/server-ce/hotfix/5.5.3/multer.patch new file mode 100644 index 0000000000..9e941204d4 --- /dev/null +++ b/server-ce/hotfix/5.5.3/multer.patch @@ -0,0 +1,27 @@ +commit 43d0476e489cdf8e2e7261eb419810140d252a6d +Author: Andrew Rumble +Date: Fri Jul 25 12:18:26 2025 +0100 + + Add patch for multer 2.0.2 + + Co-authored-by: Ersun Warncke + +diff --git a/patches/multer+2.0.2.patch b/patches/multer+2.0.2.patch +new file mode 100644 +index 00000000000..f9959effe15 +--- /dev/null ++++ b/patches/multer+2.0.2.patch +@@ -0,0 +1,13 @@ ++diff --git a/node_modules/multer/lib/make-middleware.js b/node_modules/multer/lib/make-middleware.js ++index 260dcb4..895b4b2 100644 ++--- a/node_modules/multer/lib/make-middleware.js +++++ b/node_modules/multer/lib/make-middleware.js ++@@ -113,7 +113,7 @@ function makeMiddleware (setup) { ++ if (fieldname == null) return abortWithCode('MISSING_FIELD_NAME') ++ ++ // don't attach to the files object, if there is no file ++- if (!filename) return fileStream.resume() +++ if (!filename) filename = 'undefined' ++ ++ // Work around bug in Busboy (https://github.com/mscdex/busboy/issues/6) ++ if (limits && Object.prototype.hasOwnProperty.call(limits, 'fieldNameSize')) { diff --git a/server-ce/hotfix/5.5.3/pr_27147.patch b/server-ce/hotfix/5.5.3/pr_27147.patch new file mode 100644 index 0000000000..c7ea9fb3f6 --- /dev/null +++ b/server-ce/hotfix/5.5.3/pr_27147.patch @@ -0,0 +1,351 @@ + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index ba3e0d43598e..feb4612ddc23 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -33,7 +33,6 @@ import { + makeProjectKey, + } from '../lib/blob_store/index.js' + import { backedUpBlobs as backedUpBlobsCollection, db } from '../lib/mongodb.js' +-import filestorePersistor from '../lib/persistor.js' + import commandLineArgs from 'command-line-args' + import readline from 'node:readline' + +@@ -179,6 +178,37 @@ const STREAM_HIGH_WATER_MARK = parseInt( + const LOGGING_INTERVAL = parseInt(process.env.LOGGING_INTERVAL || '60000', 10) + const SLEEP_BEFORE_EXIT = parseInt(process.env.SLEEP_BEFORE_EXIT || '1000', 10) + ++// Filestore endpoint location, the port is always hardcoded ++const FILESTORE_HOST = process.env.FILESTORE_HOST || '127.0.0.1' ++const FILESTORE_PORT = process.env.FILESTORE_PORT || '3009' ++ ++async function fetchFromFilestore(projectId, fileId) { ++ const url = `http://${FILESTORE_HOST}:${FILESTORE_PORT}/project/${projectId}/file/${fileId}` ++ const response = await fetch(url) ++ if (!response.ok) { ++ if (response.status === 404) { ++ throw new NotFoundError('file not found in filestore', { ++ status: response.status, ++ }) ++ } ++ const body = await response.text() ++ throw new OError('fetchFromFilestore failed', { ++ projectId, ++ fileId, ++ status: response.status, ++ body, ++ }) ++ } ++ if (!response.body) { ++ throw new OError('fetchFromFilestore response has no body', { ++ projectId, ++ fileId, ++ status: response.status, ++ }) ++ } ++ return response.body ++} ++ + const projectsCollection = db.collection('projects') + /** @type {ProjectsCollection} */ + const typedProjectsCollection = db.collection('projects') +@@ -348,8 +378,7 @@ async function processFile(entry, filePath) { + } catch (err) { + if (gracefulShutdownInitiated) throw err + if (err instanceof NotFoundError) { +- const { bucketName } = OError.getFullInfo(err) +- if (bucketName === USER_FILES_BUCKET_NAME && !RETRY_FILESTORE_404) { ++ if (!RETRY_FILESTORE_404) { + throw err // disable retries for not found in filestore bucket case + } + } +@@ -416,10 +445,8 @@ async function processFileOnce(entry, filePath) { + } + + STATS.readFromGCSCount++ +- const src = await filestorePersistor.getObjectStream( +- USER_FILES_BUCKET_NAME, +- `${projectId}/${fileId}` +- ) ++ // make a fetch request to filestore itself ++ const src = await fetchFromFilestore(projectId, fileId) + const dst = fs.createWriteStream(filePath, { + highWaterMark: STREAM_HIGH_WATER_MARK, + }) +@@ -1327,14 +1354,21 @@ async function processDeletedProjects() { + } + + async function main() { ++ console.log('Starting project file backup...') + await loadGlobalBlobs() ++ console.log('Loaded global blobs:', GLOBAL_BLOBS.size) + if (PROJECT_IDS_FROM) { ++ console.log( ++ `Processing projects from file: ${PROJECT_IDS_FROM}, this may take a while...` ++ ) + await processProjectsFromFile() + } else { + if (PROCESS_NON_DELETED_PROJECTS) { ++ console.log('Processing non-deleted projects...') + await processNonDeletedProjects() + } + if (PROCESS_DELETED_PROJECTS) { ++ console.log('Processing deleted projects...') + await processDeletedProjects() + } + } +diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +index fd39369a7189..4e697b8bec2c 100644 +--- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs ++++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +@@ -15,7 +15,6 @@ import { execFile } from 'node:child_process' + import chai, { expect } from 'chai' + import chaiExclude from 'chai-exclude' + import config from 'config' +-import ObjectPersistor from '@overleaf/object-persistor' + import { WritableBuffer } from '@overleaf/stream-utils' + import { + backupPersistor, +@@ -27,6 +26,9 @@ import { + makeProjectKey, + } from '../../../../storage/lib/blob_store/index.js' + ++import express from 'express' ++import bodyParser from 'body-parser' ++ + chai.use(chaiExclude) + const TIMEOUT = 20 * 1_000 + +@@ -36,15 +38,60 @@ const { tieringStorageClass } = config.get('backupPersistor') + const projectsCollection = db.collection('projects') + const deletedProjectsCollection = db.collection('deletedProjects') + +-const FILESTORE_PERSISTOR = ObjectPersistor({ +- backend: 'gcs', +- gcs: { +- endpoint: { +- apiEndpoint: process.env.GCS_API_ENDPOINT, +- projectId: process.env.GCS_PROJECT_ID, +- }, +- }, +-}) ++class MockFilestore { ++ constructor() { ++ this.host = process.env.FILESTORE_HOST || '127.0.0.1' ++ this.port = process.env.FILESTORE_PORT || 3009 ++ // create a server listening on this.host and this.port ++ this.files = {} ++ ++ this.app = express() ++ this.app.use(bodyParser.json()) ++ this.app.use(bodyParser.urlencoded({ extended: true })) ++ ++ this.app.get('/project/:projectId/file/:fileId', (req, res) => { ++ const { projectId, fileId } = req.params ++ const content = this.files[projectId]?.[fileId] ++ if (!content) return res.status(404).end() ++ res.status(200).end(content) ++ }) ++ } ++ ++ start() { ++ // reset stored files ++ this.files = {} ++ // start the server ++ if (this.serverPromise) { ++ return this.serverPromise ++ } else { ++ this.serverPromise = new Promise((resolve, reject) => { ++ this.server = this.app.listen(this.port, this.host, err => { ++ if (err) return reject(err) ++ resolve() ++ }) ++ }) ++ return this.serverPromise ++ } ++ } ++ ++ addFile(projectId, fileId, fileContent) { ++ if (!this.files[projectId]) { ++ this.files[projectId] = {} ++ } ++ this.files[projectId][fileId] = fileContent ++ } ++ ++ deleteObject(projectId, fileId) { ++ if (this.files[projectId]) { ++ delete this.files[projectId][fileId] ++ if (Object.keys(this.files[projectId]).length === 0) { ++ delete this.files[projectId] ++ } ++ } ++ } ++} ++ ++const mockFilestore = new MockFilestore() + + /** + * @param {ObjectId} objectId +@@ -472,67 +519,36 @@ describe('back_fill_file_hash script', function () { + } + + async function populateFilestore() { +- await FILESTORE_PERSISTOR.sendStream( +- USER_FILES_BUCKET_NAME, +- `${projectId0}/${fileId0}`, +- Stream.Readable.from([fileId0.toString()]) +- ) +- await FILESTORE_PERSISTOR.sendStream( +- USER_FILES_BUCKET_NAME, +- `${projectId0}/${fileId6}`, +- Stream.Readable.from([fileId6.toString()]) +- ) +- await FILESTORE_PERSISTOR.sendStream( +- USER_FILES_BUCKET_NAME, +- `${projectId0}/${fileId7}`, +- Stream.Readable.from([contentFile7]) +- ) +- await FILESTORE_PERSISTOR.sendStream( +- USER_FILES_BUCKET_NAME, +- `${projectId1}/${fileId1}`, +- Stream.Readable.from([fileId1.toString()]) +- ) +- await FILESTORE_PERSISTOR.sendStream( +- USER_FILES_BUCKET_NAME, +- `${projectId2}/${fileId2}`, +- Stream.Readable.from([fileId2.toString()]) +- ) +- await FILESTORE_PERSISTOR.sendStream( +- USER_FILES_BUCKET_NAME, +- `${projectId3}/${fileId3}`, +- Stream.Readable.from([fileId3.toString()]) +- ) +- await FILESTORE_PERSISTOR.sendStream( +- USER_FILES_BUCKET_NAME, +- `${projectId3}/${fileId10}`, ++ await mockFilestore.addFile(projectId0, fileId0, fileId0.toString()) ++ await mockFilestore.addFile(projectId0, fileId6, fileId6.toString()) ++ await mockFilestore.addFile(projectId0, fileId7, contentFile7) ++ await mockFilestore.addFile(projectId1, fileId1, fileId1.toString()) ++ await mockFilestore.addFile(projectId2, fileId2, fileId2.toString()) ++ await mockFilestore.addFile(projectId3, fileId3, fileId3.toString()) ++ await mockFilestore.addFile( ++ projectId3, ++ fileId10, + // fileId10 is dupe of fileId3 +- Stream.Readable.from([fileId3.toString()]) ++ fileId3.toString() + ) +- await FILESTORE_PERSISTOR.sendStream( +- USER_FILES_BUCKET_NAME, +- `${projectId3}/${fileId11}`, ++ await mockFilestore.addFile( ++ projectId3, ++ fileId11, + // fileId11 is dupe of fileId3 +- Stream.Readable.from([fileId3.toString()]) +- ) +- await FILESTORE_PERSISTOR.sendStream( +- USER_FILES_BUCKET_NAME, +- `${projectIdDeleted0}/${fileId4}`, +- Stream.Readable.from([fileId4.toString()]) ++ fileId3.toString() + ) +- await FILESTORE_PERSISTOR.sendStream( +- USER_FILES_BUCKET_NAME, +- `${projectIdDeleted1}/${fileId5}`, +- Stream.Readable.from([fileId5.toString()]) +- ) +- await FILESTORE_PERSISTOR.sendStream( +- USER_FILES_BUCKET_NAME, +- `${projectIdBadFileTree3}/${fileId9}`, +- Stream.Readable.from([fileId9.toString()]) ++ await mockFilestore.addFile(projectIdDeleted0, fileId4, fileId4.toString()) ++ await mockFilestore.addFile(projectIdDeleted1, fileId5, fileId5.toString()) ++ await mockFilestore.addFile( ++ projectIdBadFileTree3, ++ fileId9, ++ fileId9.toString() + ) + } + + async function prepareEnvironment() { + await cleanup.everything() ++ await mockFilestore.start() + await populateMongo() + await populateHistoryV1() + await populateFilestore() +@@ -1117,10 +1133,7 @@ describe('back_fill_file_hash script', function () { + beforeEach('prepare environment', prepareEnvironment) + + it('should gracefully handle fatal errors', async function () { +- await FILESTORE_PERSISTOR.deleteObject( +- USER_FILES_BUCKET_NAME, +- `${projectId0}/${fileId0}` +- ) ++ mockFilestore.deleteObject(projectId0, fileId0) + const t0 = Date.now() + const { stats, result } = await tryRunScript([], { + RETRIES: '10', +@@ -1148,17 +1161,10 @@ describe('back_fill_file_hash script', function () { + }) + + it('should retry on error', async function () { +- await FILESTORE_PERSISTOR.deleteObject( +- USER_FILES_BUCKET_NAME, +- `${projectId0}/${fileId0}` +- ) ++ mockFilestore.deleteObject(projectId0, fileId0) + const restoreFileAfter5s = async () => { + await setTimeout(5_000) +- await FILESTORE_PERSISTOR.sendStream( +- USER_FILES_BUCKET_NAME, +- `${projectId0}/${fileId0}`, +- Stream.Readable.from([fileId0.toString()]) +- ) ++ mockFilestore.addFile(projectId0, fileId0, fileId0.toString()) + } + // use Promise.allSettled to ensure the above sendStream call finishes before this test completes + const [ + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index feb4612ddc23..5a590e347a94 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -178,7 +178,7 @@ const STREAM_HIGH_WATER_MARK = parseInt( + const LOGGING_INTERVAL = parseInt(process.env.LOGGING_INTERVAL || '60000', 10) + const SLEEP_BEFORE_EXIT = parseInt(process.env.SLEEP_BEFORE_EXIT || '1000', 10) + +-// Filestore endpoint location, the port is always hardcoded ++// Filestore endpoint location + const FILESTORE_HOST = process.env.FILESTORE_HOST || '127.0.0.1' + const FILESTORE_PORT = process.env.FILESTORE_PORT || '3009' + + + + +diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +index 4e697b8bec2c..8f861d393451 100644 +--- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs ++++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +@@ -27,7 +27,6 @@ import { + } from '../../../../storage/lib/blob_store/index.js' + + import express from 'express' +-import bodyParser from 'body-parser' + + chai.use(chaiExclude) + const TIMEOUT = 20 * 1_000 +@@ -46,8 +45,6 @@ class MockFilestore { + this.files = {} + + this.app = express() +- this.app.use(bodyParser.json()) +- this.app.use(bodyParser.urlencoded({ extended: true })) + + this.app.get('/project/:projectId/file/:fileId', (req, res) => { + const { projectId, fileId } = req.params + diff --git a/server-ce/hotfix/5.5.3/pr_27173.patch b/server-ce/hotfix/5.5.3/pr_27173.patch new file mode 100644 index 0000000000..e1c0e08c64 --- /dev/null +++ b/server-ce/hotfix/5.5.3/pr_27173.patch @@ -0,0 +1,961 @@ + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 5a590e347a9..3be1c8a5407 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -1,28 +1,20 @@ + // @ts-check +-import Crypto from 'node:crypto' + import Events from 'node:events' + import fs from 'node:fs' + import Path from 'node:path' + import { performance } from 'node:perf_hooks' + import Stream from 'node:stream' +-import zLib from 'node:zlib' + import { setTimeout } from 'node:timers/promises' +-import { Binary, ObjectId } from 'mongodb' ++import { ObjectId } from 'mongodb' + import pLimit from 'p-limit' + import logger from '@overleaf/logger' + import { + batchedUpdate, + objectIdFromInput, + renderObjectId, +- READ_PREFERENCE_SECONDARY, + } from '@overleaf/mongo-utils/batchedUpdate.js' + import OError from '@overleaf/o-error' +-import { +- AlreadyWrittenError, +- NoKEKMatchedError, +- NotFoundError, +-} from '@overleaf/object-persistor/src/Errors.js' +-import { backupPersistor, projectBlobsBucket } from '../lib/backupPersistor.mjs' ++import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js' + import { + BlobStore, + GLOBAL_BLOBS, +@@ -30,9 +22,8 @@ import { + getProjectBlobsBatch, + getStringLengthOfFile, + makeBlobForFile, +- makeProjectKey, + } from '../lib/blob_store/index.js' +-import { backedUpBlobs as backedUpBlobsCollection, db } from '../lib/mongodb.js' ++import { db } from '../lib/mongodb.js' + import commandLineArgs from 'command-line-args' + import readline from 'node:readline' + +@@ -88,7 +79,7 @@ ObjectId.cacheHexString = true + */ + + /** +- * @return {{PROJECT_IDS_FROM: string, PROCESS_HASHED_FILES: boolean, LOGGING_IDENTIFIER: string, BATCH_RANGE_START: string, PROCESS_BLOBS: boolean, BATCH_RANGE_END: string, PROCESS_NON_DELETED_PROJECTS: boolean, PROCESS_DELETED_PROJECTS: boolean, COLLECT_BACKED_UP_BLOBS: boolean}} ++ * @return {{PROJECT_IDS_FROM: string, PROCESS_HASHED_FILES: boolean, LOGGING_IDENTIFIER: string, BATCH_RANGE_START: string, PROCESS_BLOBS: boolean, BATCH_RANGE_END: string, PROCESS_NON_DELETED_PROJECTS: boolean, PROCESS_DELETED_PROJECTS: boolean}} + */ + function parseArgs() { + const PUBLIC_LAUNCH_DATE = new Date('2012-01-01T00:00:00Z') +@@ -98,7 +89,6 @@ function parseArgs() { + { name: 'processHashedFiles', type: String, defaultValue: 'false' }, + { name: 'processBlobs', type: String, defaultValue: 'true' }, + { name: 'projectIdsFrom', type: String, defaultValue: '' }, +- { name: 'collectBackedUpBlobs', type: String, defaultValue: 'true' }, + { + name: 'BATCH_RANGE_START', + type: String, +@@ -130,7 +120,6 @@ function parseArgs() { + PROCESS_DELETED_PROJECTS: boolVal('processDeletedProjects'), + PROCESS_BLOBS: boolVal('processBlobs'), + PROCESS_HASHED_FILES: boolVal('processHashedFiles'), +- COLLECT_BACKED_UP_BLOBS: boolVal('collectBackedUpBlobs'), + BATCH_RANGE_START, + BATCH_RANGE_END, + LOGGING_IDENTIFIER: args['LOGGING_IDENTIFIER'] || BATCH_RANGE_START, +@@ -143,7 +132,6 @@ const { + PROCESS_DELETED_PROJECTS, + PROCESS_BLOBS, + PROCESS_HASHED_FILES, +- COLLECT_BACKED_UP_BLOBS, + BATCH_RANGE_START, + BATCH_RANGE_END, + LOGGING_IDENTIFIER, +@@ -232,7 +220,6 @@ async function processConcurrently(array, fn) { + const STATS = { + projects: 0, + blobs: 0, +- backedUpBlobs: 0, + filesWithHash: 0, + filesWithoutHash: 0, + filesDuplicated: 0, +@@ -246,14 +233,8 @@ const STATS = { + projectHardDeleted: 0, + fileHardDeleted: 0, + mongoUpdates: 0, +- deduplicatedWriteToAWSLocalCount: 0, +- deduplicatedWriteToAWSLocalEgress: 0, +- deduplicatedWriteToAWSRemoteCount: 0, +- deduplicatedWriteToAWSRemoteEgress: 0, + readFromGCSCount: 0, + readFromGCSIngress: 0, +- writeToAWSCount: 0, +- writeToAWSEgress: 0, + writeToGCSCount: 0, + writeToGCSEgress: 0, + } +@@ -275,7 +256,7 @@ function toMiBPerSecond(v, ms) { + /** + * @param {any} stats + * @param {number} ms +- * @return {{writeToAWSThroughputMiBPerSecond: number, readFromGCSThroughputMiBPerSecond: number}} ++ * @return {{readFromGCSThroughputMiBPerSecond: number}} + */ + function bandwidthStats(stats, ms) { + return { +@@ -283,10 +264,6 @@ function bandwidthStats(stats, ms) { + stats.readFromGCSIngress, + ms + ), +- writeToAWSThroughputMiBPerSecond: toMiBPerSecond( +- stats.writeToAWSEgress, +- ms +- ), + } + } + +@@ -382,9 +359,6 @@ async function processFile(entry, filePath) { + throw err // disable retries for not found in filestore bucket case + } + } +- if (err instanceof NoKEKMatchedError) { +- throw err // disable retries when upload to S3 will fail again +- } + STATS.filesRetries++ + const { + ctx: { projectId }, +@@ -417,32 +391,8 @@ async function processFileOnce(entry, filePath) { + if (entry.blob) { + const { blob } = entry + const hash = blob.getHash() +- if (entry.ctx.hasBackedUpBlob(hash)) { +- STATS.deduplicatedWriteToAWSLocalCount++ +- STATS.deduplicatedWriteToAWSLocalEgress += estimateBlobSize(blob) +- return hash +- } +- entry.ctx.recordPendingBlob(hash) +- STATS.readFromGCSCount++ +- const src = await blobStore.getStream(hash) +- const dst = fs.createWriteStream(filePath, { +- highWaterMark: STREAM_HIGH_WATER_MARK, +- }) +- try { +- await Stream.promises.pipeline(src, dst) +- } finally { +- STATS.readFromGCSIngress += dst.bytesWritten +- } +- await uploadBlobToAWS(entry, blob, filePath) + return hash + } +- if (entry.hash && entry.ctx.hasBackedUpBlob(entry.hash)) { +- STATS.deduplicatedWriteToAWSLocalCount++ +- const blob = entry.ctx.getCachedHistoryBlob(entry.hash) +- // blob might not exist on re-run with --PROCESS_BLOBS=false +- if (blob) STATS.deduplicatedWriteToAWSLocalEgress += estimateBlobSize(blob) +- return entry.hash +- } + + STATS.readFromGCSCount++ + // make a fetch request to filestore itself +@@ -469,16 +419,14 @@ async function processFileOnce(entry, filePath) { + STATS.globalBlobsEgress += estimateBlobSize(blob) + return hash + } +- if (entry.ctx.hasBackedUpBlob(hash)) { +- STATS.deduplicatedWriteToAWSLocalCount++ +- STATS.deduplicatedWriteToAWSLocalEgress += estimateBlobSize(blob) ++ if (entry.ctx.hasCompletedBlob(hash)) { + return hash + } + entry.ctx.recordPendingBlob(hash) + + try { + await uploadBlobToGCS(blobStore, entry, blob, hash, filePath) +- await uploadBlobToAWS(entry, blob, filePath) ++ entry.ctx.recordCompletedBlob(hash) // mark upload as completed + } catch (err) { + entry.ctx.recordFailedBlob(hash) + throw err +@@ -515,76 +463,6 @@ async function uploadBlobToGCS(blobStore, entry, blob, hash, filePath) { + + const GZ_SUFFIX = '.gz' + +-/** +- * @param {QueueEntry} entry +- * @param {Blob} blob +- * @param {string} filePath +- * @return {Promise} +- */ +-async function uploadBlobToAWS(entry, blob, filePath) { +- const { historyId } = entry.ctx +- let backupSource +- let contentEncoding +- const md5 = Crypto.createHash('md5') +- let size +- if (blob.getStringLength()) { +- const filePathCompressed = filePath + GZ_SUFFIX +- backupSource = filePathCompressed +- contentEncoding = 'gzip' +- size = 0 +- await Stream.promises.pipeline( +- fs.createReadStream(filePath, { highWaterMark: STREAM_HIGH_WATER_MARK }), +- zLib.createGzip(), +- async function* (source) { +- for await (const chunk of source) { +- size += chunk.byteLength +- md5.update(chunk) +- yield chunk +- } +- }, +- fs.createWriteStream(filePathCompressed, { +- highWaterMark: STREAM_HIGH_WATER_MARK, +- }) +- ) +- } else { +- backupSource = filePath +- size = blob.getByteLength() +- await Stream.promises.pipeline( +- fs.createReadStream(filePath, { highWaterMark: STREAM_HIGH_WATER_MARK }), +- md5 +- ) +- } +- const backendKeyPath = makeProjectKey(historyId, blob.getHash()) +- const persistor = await entry.ctx.getCachedPersistor(backendKeyPath) +- try { +- STATS.writeToAWSCount++ +- await persistor.sendStream( +- projectBlobsBucket, +- backendKeyPath, +- fs.createReadStream(backupSource, { +- highWaterMark: STREAM_HIGH_WATER_MARK, +- }), +- { +- contentEncoding, +- contentType: 'application/octet-stream', +- contentLength: size, +- sourceMd5: md5.digest('hex'), +- ifNoneMatch: '*', // de-duplicate write (we pay for the request, but avoid egress) +- } +- ) +- STATS.writeToAWSEgress += size +- } catch (err) { +- if (err instanceof AlreadyWrittenError) { +- STATS.deduplicatedWriteToAWSRemoteCount++ +- STATS.deduplicatedWriteToAWSRemoteEgress += size +- } else { +- STATS.writeToAWSEgress += size +- throw err +- } +- } +- entry.ctx.recordBackedUpBlob(blob.getHash()) +-} +- + /** + * @param {Array} files + * @return {Promise} +@@ -670,23 +548,18 @@ async function queueNextBatch(batch, prefix = 'rootFolder.0') { + * @return {Promise} + */ + async function processBatch(batch, prefix = 'rootFolder.0') { +- const [{ nBlobs, blobs }, { nBackedUpBlobs, backedUpBlobs }] = +- await Promise.all([collectProjectBlobs(batch), collectBackedUpBlobs(batch)]) +- const files = Array.from(findFileInBatch(batch, prefix, blobs, backedUpBlobs)) ++ const { nBlobs, blobs } = await collectProjectBlobs(batch) ++ const files = Array.from(findFileInBatch(batch, prefix, blobs)) + STATS.projects += batch.length + STATS.blobs += nBlobs +- STATS.backedUpBlobs += nBackedUpBlobs + + // GC + batch.length = 0 + blobs.clear() +- backedUpBlobs.clear() + + // The files are currently ordered by project-id. + // Order them by file-id ASC then blobs ASC to + // - process files before blobs +- // - avoid head-of-line blocking from many project-files waiting on the generation of the projects DEK (round trip to AWS) +- // - bonus: increase chance of de-duplicating write to AWS + files.sort( + /** + * @param {QueueEntry} a +@@ -903,23 +776,15 @@ function* findFiles(ctx, folder, path, isInputLoop = false) { + * @param {Array} projects + * @param {string} prefix + * @param {Map>} blobs +- * @param {Map>} backedUpBlobs + * @return Generator + */ +-function* findFileInBatch(projects, prefix, blobs, backedUpBlobs) { ++function* findFileInBatch(projects, prefix, blobs) { + for (const project of projects) { + const projectIdS = project._id.toString() + const historyIdS = project.overleaf.history.id.toString() + const projectBlobs = blobs.get(historyIdS) || [] +- const projectBackedUpBlobs = new Set(backedUpBlobs.get(projectIdS) || []) +- const ctx = new ProjectContext( +- project._id, +- historyIdS, +- projectBlobs, +- projectBackedUpBlobs +- ) ++ const ctx = new ProjectContext(project._id, historyIdS, projectBlobs) + for (const blob of projectBlobs) { +- if (projectBackedUpBlobs.has(blob.getHash())) continue + ctx.remainingQueueEntries++ + yield { + ctx, +@@ -951,42 +816,11 @@ async function collectProjectBlobs(batch) { + return await getProjectBlobsBatch(batch.map(p => p.overleaf.history.id)) + } + +-/** +- * @param {Array} projects +- * @return {Promise<{nBackedUpBlobs:number,backedUpBlobs:Map>}>} +- */ +-async function collectBackedUpBlobs(projects) { +- let nBackedUpBlobs = 0 +- const backedUpBlobs = new Map() +- if (!COLLECT_BACKED_UP_BLOBS) return { nBackedUpBlobs, backedUpBlobs } +- +- const cursor = backedUpBlobsCollection.find( +- { _id: { $in: projects.map(p => p._id) } }, +- { +- readPreference: READ_PREFERENCE_SECONDARY, +- sort: { _id: 1 }, +- } +- ) +- for await (const record of cursor) { +- const blobs = record.blobs.map(b => b.toString('hex')) +- backedUpBlobs.set(record._id.toString(), blobs) +- nBackedUpBlobs += blobs.length +- } +- return { nBackedUpBlobs, backedUpBlobs } +-} +- +-const BATCH_HASH_WRITES = 1_000 + const BATCH_FILE_UPDATES = 100 + + const MONGO_PATH_SKIP_WRITE_HASH_TO_FILE_TREE = 'skip-write-to-file-tree' + + class ProjectContext { +- /** @type {Promise | null} */ +- #cachedPersistorPromise = null +- +- /** @type {Set} */ +- #backedUpBlobs +- + /** @type {Map} */ + #historyBlobs + +@@ -1000,12 +834,10 @@ class ProjectContext { + * @param {ObjectId} projectId + * @param {string} historyId + * @param {Array} blobs +- * @param {Set} backedUpBlobs + */ +- constructor(projectId, historyId, blobs, backedUpBlobs) { ++ constructor(projectId, historyId, blobs) { + this.projectId = projectId + this.historyId = historyId +- this.#backedUpBlobs = backedUpBlobs + this.#historyBlobs = new Map(blobs.map(b => [b.getHash(), b])) + } + +@@ -1034,75 +866,17 @@ class ProjectContext { + return false + } + +- /** +- * @param {string} key +- * @return {Promise} +- */ +- getCachedPersistor(key) { +- if (!this.#cachedPersistorPromise) { +- // Fetch DEK once, but only if needed -- upon the first use +- this.#cachedPersistorPromise = this.#getCachedPersistorWithRetries(key) +- } +- return this.#cachedPersistorPromise +- } +- +- /** +- * @param {string} key +- * @return {Promise} +- */ +- async #getCachedPersistorWithRetries(key) { +- // Optimization: Skip GET on DEK in case no blobs are marked as backed up yet. +- let tryGenerateDEKFirst = this.#backedUpBlobs.size === 0 +- for (let attempt = 0; attempt < RETRIES; attempt++) { +- try { +- if (tryGenerateDEKFirst) { +- try { +- return await backupPersistor.generateDataEncryptionKey( +- projectBlobsBucket, +- key +- ) +- } catch (err) { +- if (err instanceof AlreadyWrittenError) { +- tryGenerateDEKFirst = false +- // fall back to GET below +- } else { +- throw err +- } +- } +- } +- return await backupPersistor.forProject(projectBlobsBucket, key) +- } catch (err) { +- if (gracefulShutdownInitiated) throw err +- if (err instanceof NoKEKMatchedError) { +- throw err +- } else { +- logger.warn( +- { err, projectId: this.projectId, attempt }, +- 'failed to get DEK, trying again' +- ) +- const jitter = Math.random() * RETRY_DELAY_MS +- await setTimeout(RETRY_DELAY_MS + jitter) +- } +- } +- } +- return await backupPersistor.forProject(projectBlobsBucket, key) +- } +- + async flushMongoQueuesIfNeeded() { + if (this.remainingQueueEntries === 0) { + await this.flushMongoQueues() + } + +- if (this.#completedBlobs.size > BATCH_HASH_WRITES) { +- await this.#storeBackedUpBlobs() +- } + if (this.#pendingFileWrites.length > BATCH_FILE_UPDATES) { + await this.#storeFileHashes() + } + } + + async flushMongoQueues() { +- await this.#storeBackedUpBlobs() + await this.#storeFileHashes() + } + +@@ -1111,20 +885,6 @@ class ProjectContext { + /** @type {Set} */ + #completedBlobs = new Set() + +- async #storeBackedUpBlobs() { +- if (this.#completedBlobs.size === 0) return +- const blobs = Array.from(this.#completedBlobs).map( +- hash => new Binary(Buffer.from(hash, 'hex')) +- ) +- this.#completedBlobs.clear() +- STATS.mongoUpdates++ +- await backedUpBlobsCollection.updateOne( +- { _id: this.projectId }, +- { $addToSet: { blobs: { $each: blobs } } }, +- { upsert: true } +- ) +- } +- + /** + * @param {string} hash + */ +@@ -1142,8 +902,7 @@ class ProjectContext { + /** + * @param {string} hash + */ +- recordBackedUpBlob(hash) { +- this.#backedUpBlobs.add(hash) ++ recordCompletedBlob(hash) { + this.#completedBlobs.add(hash) + this.#pendingBlobs.delete(hash) + } +@@ -1152,12 +911,8 @@ class ProjectContext { + * @param {string} hash + * @return {boolean} + */ +- hasBackedUpBlob(hash) { +- return ( +- this.#pendingBlobs.has(hash) || +- this.#completedBlobs.has(hash) || +- this.#backedUpBlobs.has(hash) +- ) ++ hasCompletedBlob(hash) { ++ return this.#pendingBlobs.has(hash) || this.#completedBlobs.has(hash) + } + + /** @type {Array} */ +diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +index 8f861d39345..62b0b1de25f 100644 +--- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs ++++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +@@ -4,23 +4,17 @@ import Stream from 'node:stream' + import { setTimeout } from 'node:timers/promises' + import { promisify } from 'node:util' + import { ObjectId, Binary } from 'mongodb' +-import { +- db, +- backedUpBlobs, +- globalBlobs, +-} from '../../../../storage/lib/mongodb.js' ++import { db, globalBlobs } from '../../../../storage/lib/mongodb.js' + import cleanup from './support/cleanup.js' + import testProjects from '../api/support/test_projects.js' + import { execFile } from 'node:child_process' + import chai, { expect } from 'chai' + import chaiExclude from 'chai-exclude' +-import config from 'config' + import { WritableBuffer } from '@overleaf/stream-utils' + import { + backupPersistor, + projectBlobsBucket, + } from '../../../../storage/lib/backupPersistor.mjs' +-import projectKey from '../../../../storage/lib/project_key.js' + import { + BlobStore, + makeProjectKey, +@@ -31,9 +25,6 @@ import express from 'express' + chai.use(chaiExclude) + const TIMEOUT = 20 * 1_000 + +-const { deksBucket } = config.get('backupStore') +-const { tieringStorageClass } = config.get('backupPersistor') +- + const projectsCollection = db.collection('projects') + const deletedProjectsCollection = db.collection('deletedProjects') + +@@ -117,17 +108,6 @@ function binaryForGitBlobHash(gitBlobHash) { + return new Binary(Buffer.from(gitBlobHash, 'hex')) + } + +-async function listS3Bucket(bucket, wantStorageClass) { +- const client = backupPersistor._getClientForBucket(bucket) +- const response = await client.listObjectsV2({ Bucket: bucket }).promise() +- +- for (const object of response.Contents || []) { +- expect(object).to.have.property('StorageClass', wantStorageClass) +- } +- +- return (response.Contents || []).map(item => item.Key || '') +-} +- + function objectIdFromTime(timestamp) { + return ObjectId.createFromTime(new Date(timestamp).getTime() / 1000) + } +@@ -591,11 +571,7 @@ describe('back_fill_file_hash script', function () { + expect((await fs.promises.readdir('/tmp')).join(';')).to.not.match( + /back_fill_file_hash/ + ) +- const extraStatsKeys = [ +- 'eventLoop', +- 'readFromGCSThroughputMiBPerSecond', +- 'writeToAWSThroughputMiBPerSecond', +- ] ++ const extraStatsKeys = ['eventLoop', 'readFromGCSThroughputMiBPerSecond'] + const stats = JSON.parse( + result.stderr + .split('\n') +@@ -610,7 +586,6 @@ describe('back_fill_file_hash script', function () { + delete stats.time + if (shouldHaveWritten) { + expect(stats.readFromGCSThroughputMiBPerSecond).to.be.greaterThan(0) +- expect(stats.writeToAWSThroughputMiBPerSecond).to.be.greaterThan(0) + } + for (const key of extraStatsKeys) { + delete stats[key] +@@ -856,109 +831,6 @@ describe('back_fill_file_hash script', function () { + }, + }, + ]) +- expect( +- (await backedUpBlobs.find({}, { sort: { _id: 1 } }).toArray()).map( +- entry => { +- // blobs are pushed unordered into mongo. Sort the list for consistency. +- entry.blobs.sort() +- return entry +- } +- ) +- ).to.deep.equal([ +- { +- _id: projectId0, +- blobs: [ +- binaryForGitBlobHash(gitBlobHash(fileId0)), +- binaryForGitBlobHash(hashFile7), +- binaryForGitBlobHash(hashTextBlob0), +- ].sort(), +- }, +- { +- _id: projectId1, +- blobs: [ +- binaryForGitBlobHash(gitBlobHash(fileId1)), +- binaryForGitBlobHash(hashTextBlob1), +- ].sort(), +- }, +- { +- _id: projectId2, +- blobs: [binaryForGitBlobHash(hashTextBlob2)] +- .concat( +- processHashedFiles +- ? [binaryForGitBlobHash(gitBlobHash(fileId2))] +- : [] +- ) +- .sort(), +- }, +- { +- _id: projectIdDeleted0, +- blobs: [binaryForGitBlobHash(gitBlobHash(fileId4))].sort(), +- }, +- { +- _id: projectId3, +- blobs: [binaryForGitBlobHash(gitBlobHash(fileId3))].sort(), +- }, +- ...(processHashedFiles +- ? [ +- { +- _id: projectIdDeleted1, +- blobs: [binaryForGitBlobHash(gitBlobHash(fileId5))].sort(), +- }, +- ] +- : []), +- { +- _id: projectIdBadFileTree0, +- blobs: [binaryForGitBlobHash(hashTextBlob3)].sort(), +- }, +- { +- _id: projectIdBadFileTree3, +- blobs: [binaryForGitBlobHash(gitBlobHash(fileId9))].sort(), +- }, +- ]) +- }) +- it('should have backed up all the files', async function () { +- expect(tieringStorageClass).to.exist +- const blobs = await listS3Bucket(projectBlobsBucket, tieringStorageClass) +- expect(blobs.sort()).to.deep.equal( +- Array.from( +- new Set( +- writtenBlobs +- .map(({ historyId, fileId, hash }) => +- makeProjectKey(historyId, hash || gitBlobHash(fileId)) +- ) +- .sort() +- ) +- ) +- ) +- for (let { historyId, fileId, hash, content } of writtenBlobs) { +- hash = hash || gitBlobHash(fileId.toString()) +- const s = await backupPersistor.getObjectStream( +- projectBlobsBucket, +- makeProjectKey(historyId, hash), +- { autoGunzip: true } +- ) +- const buf = new WritableBuffer() +- await Stream.promises.pipeline(s, buf) +- expect(gitBlobHashBuffer(buf.getContents())).to.equal(hash) +- if (content) { +- expect(buf.getContents()).to.deep.equal(content) +- } else { +- const id = buf.getContents().toString('utf-8') +- expect(id).to.equal(fileId.toString()) +- // double check we are not comparing 'undefined' or '[object Object]' above +- expect(id).to.match(/^[a-f0-9]{24}$/) +- } +- } +- const deks = await listS3Bucket(deksBucket, 'STANDARD') +- expect(deks.sort()).to.deep.equal( +- Array.from( +- new Set( +- writtenBlobs.map( +- ({ historyId }) => projectKey.format(historyId) + '/dek' +- ) +- ) +- ).sort() +- ) + }) + it('should have written the back filled files to history v1', async function () { + for (const { historyId, hash, fileId, content } of writtenBlobs) { +@@ -991,14 +863,13 @@ describe('back_fill_file_hash script', function () { + // We still need to iterate over all the projects and blobs. + projects: 10, + blobs: 10, +- backedUpBlobs: 10, ++ + badFileTrees: 4, + } + if (processHashedFiles) { + stats = sumStats(stats, { + ...STATS_ALL_ZERO, + blobs: 2, +- backedUpBlobs: 2, + }) + } + expect(rerun.stats).deep.equal(stats) +@@ -1024,7 +895,6 @@ describe('back_fill_file_hash script', function () { + const STATS_ALL_ZERO = { + projects: 0, + blobs: 0, +- backedUpBlobs: 0, + filesWithHash: 0, + filesWithoutHash: 0, + filesDuplicated: 0, +@@ -1038,21 +908,14 @@ describe('back_fill_file_hash script', function () { + fileHardDeleted: 0, + badFileTrees: 0, + mongoUpdates: 0, +- deduplicatedWriteToAWSLocalCount: 0, +- deduplicatedWriteToAWSLocalEgress: 0, +- deduplicatedWriteToAWSRemoteCount: 0, +- deduplicatedWriteToAWSRemoteEgress: 0, + readFromGCSCount: 0, + readFromGCSIngress: 0, +- writeToAWSCount: 0, +- writeToAWSEgress: 0, + writeToGCSCount: 0, + writeToGCSEgress: 0, + } + const STATS_UP_TO_PROJECT1 = { + projects: 2, + blobs: 2, +- backedUpBlobs: 0, + filesWithHash: 0, + filesWithoutHash: 5, + filesDuplicated: 1, +@@ -1065,22 +928,15 @@ describe('back_fill_file_hash script', function () { + projectHardDeleted: 0, + fileHardDeleted: 0, + badFileTrees: 0, +- mongoUpdates: 4, +- deduplicatedWriteToAWSLocalCount: 0, +- deduplicatedWriteToAWSLocalEgress: 0, +- deduplicatedWriteToAWSRemoteCount: 0, +- deduplicatedWriteToAWSRemoteEgress: 0, +- readFromGCSCount: 6, +- readFromGCSIngress: 4000086, +- writeToAWSCount: 5, +- writeToAWSEgress: 4026, ++ mongoUpdates: 2, // 4-2 blobs written to backedUpBlobs collection ++ readFromGCSCount: 4, ++ readFromGCSIngress: 4000072, + writeToGCSCount: 3, + writeToGCSEgress: 4000048, + } + const STATS_UP_FROM_PROJECT1_ONWARD = { + projects: 8, + blobs: 2, +- backedUpBlobs: 0, + filesWithHash: 0, + filesWithoutHash: 4, + filesDuplicated: 0, +@@ -1093,26 +949,18 @@ describe('back_fill_file_hash script', function () { + projectHardDeleted: 0, + fileHardDeleted: 0, + badFileTrees: 4, +- mongoUpdates: 8, +- deduplicatedWriteToAWSLocalCount: 1, +- deduplicatedWriteToAWSLocalEgress: 30, +- deduplicatedWriteToAWSRemoteCount: 0, +- deduplicatedWriteToAWSRemoteEgress: 0, +- readFromGCSCount: 6, +- readFromGCSIngress: 110, +- writeToAWSCount: 5, +- writeToAWSEgress: 143, ++ mongoUpdates: 3, // previously 5 blobs written to backedUpBlobs collection ++ readFromGCSCount: 4, ++ readFromGCSIngress: 96, + writeToGCSCount: 3, + writeToGCSEgress: 72, + } + const STATS_FILES_HASHED_EXTRA = { + ...STATS_ALL_ZERO, + filesWithHash: 2, +- mongoUpdates: 2, ++ mongoUpdates: 0, // previously 2 blobs written to backedUpBlobs collection + readFromGCSCount: 2, + readFromGCSIngress: 48, +- writeToAWSCount: 2, +- writeToAWSEgress: 60, + writeToGCSCount: 2, + writeToGCSEgress: 48, + } +@@ -1144,8 +992,6 @@ describe('back_fill_file_hash script', function () { + ...STATS_ALL_ZERO, + filesFailed: 1, + readFromGCSIngress: -24, +- writeToAWSCount: -1, +- writeToAWSEgress: -28, + writeToGCSCount: -1, + writeToGCSEgress: -24, + }) +@@ -1269,13 +1115,14 @@ describe('back_fill_file_hash script', function () { + before('run script with hashed files', async function () { + output2 = await runScript(['--processHashedFiles=true'], {}) + }) +- it('should print stats', function () { ++ it('should print stats for the first run without hashed files', function () { + expect(output1.stats).deep.equal(STATS_ALL) ++ }) ++ it('should print stats for the hashed files run', function () { + expect(output2.stats).deep.equal({ + ...STATS_FILES_HASHED_EXTRA, + projects: 10, + blobs: 10, +- backedUpBlobs: 10, + badFileTrees: 4, + }) + }) +@@ -1322,9 +1169,7 @@ describe('back_fill_file_hash script', function () { + ...STATS_FILES_HASHED_EXTRA, + readFromGCSCount: 3, + readFromGCSIngress: 72, +- deduplicatedWriteToAWSLocalCount: 1, +- deduplicatedWriteToAWSLocalEgress: 30, +- mongoUpdates: 1, ++ mongoUpdates: 0, + filesWithHash: 3, + }) + ) +@@ -1354,48 +1199,6 @@ describe('back_fill_file_hash script', function () { + expect(output.stats).deep.equal( + sumStats(STATS_ALL, { + ...STATS_ALL_ZERO, +- // one remote deduplicate +- deduplicatedWriteToAWSRemoteCount: 1, +- deduplicatedWriteToAWSRemoteEgress: 28, +- writeToAWSEgress: -28, // subtract skipped egress +- }) +- ) +- }) +- commonAssertions() +- }) +- +- describe('with something in the bucket and marked as processed', function () { +- before('prepare environment', prepareEnvironment) +- before('create a file in s3', async function () { +- await backupPersistor.sendStream( +- projectBlobsBucket, +- makeProjectKey(historyId0, hashTextBlob0), +- Stream.Readable.from([contentTextBlob0]), +- { contentLength: contentTextBlob0.byteLength } +- ) +- await backedUpBlobs.insertMany([ +- { +- _id: projectId0, +- blobs: [binaryForGitBlobHash(hashTextBlob0)], +- }, +- ]) +- }) +- let output +- before('run script', async function () { +- output = await runScript([], { +- CONCURRENCY: '1', +- }) +- }) +- +- it('should print stats', function () { +- expect(output.stats).deep.equal( +- sumStats(STATS_ALL, { +- ...STATS_ALL_ZERO, +- backedUpBlobs: 1, +- writeToAWSCount: -1, +- writeToAWSEgress: -27, +- readFromGCSCount: -1, +- readFromGCSIngress: -7, + }) + ) + }) +@@ -1418,8 +1221,10 @@ describe('back_fill_file_hash script', function () { + }) + }) + +- it('should print stats', function () { ++ it('should print stats for part 0', function () { + expect(outputPart0.stats).to.deep.equal(STATS_UP_TO_PROJECT1) ++ }) ++ it('should print stats for part 1', function () { + expect(outputPart1.stats).to.deep.equal(STATS_UP_FROM_PROJECT1_ONWARD) + }) + commonAssertions() + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 3be1c8a5407..c9ed13c6cb4 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -388,12 +388,6 @@ async function processFileOnce(entry, filePath) { + fileId, + } = entry + const blobStore = new BlobStore(historyId) +- if (entry.blob) { +- const { blob } = entry +- const hash = blob.getHash() +- return hash +- } +- + STATS.readFromGCSCount++ + // make a fetch request to filestore itself + const src = await fetchFromFilestore(projectId, fileId) +@@ -784,16 +778,6 @@ function* findFileInBatch(projects, prefix, blobs) { + const historyIdS = project.overleaf.history.id.toString() + const projectBlobs = blobs.get(historyIdS) || [] + const ctx = new ProjectContext(project._id, historyIdS, projectBlobs) +- for (const blob of projectBlobs) { +- ctx.remainingQueueEntries++ +- yield { +- ctx, +- cacheKey: blob.getHash(), +- path: MONGO_PATH_SKIP_WRITE_HASH_TO_FILE_TREE, +- blob, +- hash: blob.getHash(), +- } +- } + try { + yield* findFiles(ctx, project.rootFolder?.[0], prefix, true) + } catch (err) { + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index c9ed13c6cb4..f24ce4a6605 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -387,6 +387,13 @@ async function processFileOnce(entry, filePath) { + ctx: { projectId, historyId }, + fileId, + } = entry ++ if (entry.hash && entry.ctx.hasCompletedBlob(entry.hash)) { ++ // We can enter this case for two identical files in the same project, ++ // one with hash, the other without. When the one without hash gets ++ // processed first, we can skip downloading the other one we already ++ // know the hash of. ++ return entry.hash ++ } + const blobStore = new BlobStore(historyId) + STATS.readFromGCSCount++ + // make a fetch request to filestore itself + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index f24ce4a6605..0ccadaf5a95 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -559,8 +559,9 @@ async function processBatch(batch, prefix = 'rootFolder.0') { + blobs.clear() + + // The files are currently ordered by project-id. +- // Order them by file-id ASC then blobs ASC to +- // - process files before blobs ++ // Order them by file-id ASC then hash ASC to ++ // increase the hit rate on the "already processed ++ // hash for project" checks. + files.sort( + /** + * @param {QueueEntry} a + diff --git a/server-ce/hotfix/5.5.3/pr_27230.patch b/server-ce/hotfix/5.5.3/pr_27230.patch new file mode 100644 index 0000000000..79d16f32f4 --- /dev/null +++ b/server-ce/hotfix/5.5.3/pr_27230.patch @@ -0,0 +1,191 @@ + + +diff --git a/services/web/app.mjs b/services/web/app.mjs +index b7c723da3d77..3f54cc36a8c3 100644 +--- a/services/web/app.mjs ++++ b/services/web/app.mjs +@@ -56,14 +56,8 @@ if (Settings.catchErrors) { + // Create ./data/dumpFolder if needed + FileWriter.ensureDumpFolderExists() + +-if ( +- !Features.hasFeature('project-history-blobs') && +- !Features.hasFeature('filestore') +-) { +- throw new Error( +- 'invalid config: must enable either project-history-blobs (Settings.enableProjectHistoryBlobs=true) or enable filestore (Settings.disableFilestore=false)' +- ) +-} ++// Validate combination of feature flags. ++Features.validateSettings() + + // handle SIGTERM for graceful shutdown in kubernetes + process.on('SIGTERM', function (signal) { +diff --git a/services/web/app/src/Features/History/HistoryURLHelper.js b/services/web/app/src/Features/History/HistoryURLHelper.js +index 8b8d8cbdd730..acb43ced68e0 100644 +--- a/services/web/app/src/Features/History/HistoryURLHelper.js ++++ b/services/web/app/src/Features/History/HistoryURLHelper.js +@@ -8,7 +8,7 @@ function projectHistoryURLWithFilestoreFallback( + ) { + const filestoreURL = `${Settings.apis.filestore.url}/project/${projectId}/file/${fileRef._id}?from=${origin}` + // TODO: When this file is converted to ES modules we will be able to use Features.hasFeature('project-history-blobs'). Currently we can't stub the feature return value in tests. +- if (fileRef.hash && Settings.enableProjectHistoryBlobs) { ++ if (fileRef.hash && Settings.filestoreMigrationLevel >= 1) { + return { + url: `${Settings.apis.project_history.url}/project/${historyId}/blob/${fileRef.hash}`, + fallbackURL: filestoreURL, +diff --git a/services/web/app/src/infrastructure/Features.js b/services/web/app/src/infrastructure/Features.js +index aaf51103b9b8..89c8e6b841d0 100644 +--- a/services/web/app/src/infrastructure/Features.js ++++ b/services/web/app/src/infrastructure/Features.js +@@ -19,8 +19,7 @@ const trackChangesModuleAvailable = + * @property {boolean | undefined} enableGithubSync + * @property {boolean | undefined} enableGitBridge + * @property {boolean | undefined} enableHomepage +- * @property {boolean | undefined} enableProjectHistoryBlobs +- * @property {boolean | undefined} disableFilestore ++ * @property {number} filestoreMigrationLevel + * @property {boolean | undefined} enableSaml + * @property {boolean | undefined} ldap + * @property {boolean | undefined} oauth +@@ -29,7 +28,39 @@ const trackChangesModuleAvailable = + * @property {boolean | undefined} saml + */ + ++/** ++ * @return {{'project-history-blobs': boolean, filestore: boolean}} ++ */ ++function getFilestoreMigrationOptions() { ++ switch (Settings.filestoreMigrationLevel) { ++ case 0: ++ return { ++ 'project-history-blobs': false, ++ filestore: true, ++ } ++ case 1: ++ return { ++ 'project-history-blobs': true, ++ filestore: true, ++ } ++ ++ case 2: ++ return { ++ 'project-history-blobs': true, ++ filestore: false, ++ } ++ default: ++ throw new Error( ++ `invalid OVERLEAF_FILESTORE_MIGRATION_LEVEL=${Settings.filestoreMigrationLevel}, expected 0, 1 or 2` ++ ) ++ } ++} ++ + const Features = { ++ validateSettings() { ++ getFilestoreMigrationOptions() // throws for invalid settings ++ }, ++ + /** + * @returns {boolean} + */ +@@ -89,9 +120,9 @@ const Features = { + Settings.enabledLinkedFileTypes.includes('url') + ) + case 'project-history-blobs': +- return Boolean(Settings.enableProjectHistoryBlobs) ++ return getFilestoreMigrationOptions()['project-history-blobs'] + case 'filestore': +- return Boolean(Settings.disableFilestore) === false ++ return getFilestoreMigrationOptions().filestore + case 'support': + return supportModuleAvailable + case 'symbol-palette': +diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js +index bd0730d5d00c..4df63ebd7c6c 100644 +--- a/services/web/config/settings.defaults.js ++++ b/services/web/config/settings.defaults.js +@@ -440,6 +440,9 @@ module.exports = { + ',' + ), + ++ filestoreMigrationLevel: ++ parseInt(process.env.OVERLEAF_FILESTORE_MIGRATION_LEVEL, 10) || 0, ++ + // i18n + // ------ + // + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 0ccadaf5a955..2e12328e5c49 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -150,10 +150,6 @@ const CONCURRENT_BATCHES = parseInt(process.env.CONCURRENT_BATCHES || '2', 10) + const RETRIES = parseInt(process.env.RETRIES || '10', 10) + const RETRY_DELAY_MS = parseInt(process.env.RETRY_DELAY_MS || '100', 10) + +-const USER_FILES_BUCKET_NAME = process.env.USER_FILES_BUCKET_NAME || '' +-if (!USER_FILES_BUCKET_NAME) { +- throw new Error('env var USER_FILES_BUCKET_NAME is missing') +-} + const RETRY_FILESTORE_404 = process.env.RETRY_FILESTORE_404 === 'true' + const BUFFER_DIR = fs.mkdtempSync( + process.env.BUFFER_DIR_PREFIX || '/tmp/back_fill_file_hash-' + +diff --git a/services/web/app/src/infrastructure/Features.js b/services/web/app/src/infrastructure/Features.js +index 89c8e6b841d0..6147e70e0faf 100644 +--- a/services/web/app/src/infrastructure/Features.js ++++ b/services/web/app/src/infrastructure/Features.js +@@ -28,37 +28,13 @@ const trackChangesModuleAvailable = + * @property {boolean | undefined} saml + */ + +-/** +- * @return {{'project-history-blobs': boolean, filestore: boolean}} +- */ +-function getFilestoreMigrationOptions() { +- switch (Settings.filestoreMigrationLevel) { +- case 0: +- return { +- 'project-history-blobs': false, +- filestore: true, +- } +- case 1: +- return { +- 'project-history-blobs': true, +- filestore: true, +- } +- +- case 2: +- return { +- 'project-history-blobs': true, +- filestore: false, +- } +- default: ++const Features = { ++ validateSettings() { ++ if (![0, 1, 2].includes(Settings.filestoreMigrationLevel)) { + throw new Error( + `invalid OVERLEAF_FILESTORE_MIGRATION_LEVEL=${Settings.filestoreMigrationLevel}, expected 0, 1 or 2` + ) +- } +-} +- +-const Features = { +- validateSettings() { +- getFilestoreMigrationOptions() // throws for invalid settings ++ } + }, + + /** +@@ -120,9 +96,9 @@ const Features = { + Settings.enabledLinkedFileTypes.includes('url') + ) + case 'project-history-blobs': +- return getFilestoreMigrationOptions()['project-history-blobs'] ++ return Settings.filestoreMigrationLevel > 0 + case 'filestore': +- return getFilestoreMigrationOptions().filestore ++ return Settings.filestoreMigrationLevel < 2 + case 'support': + return supportModuleAvailable + case 'symbol-palette': diff --git a/server-ce/hotfix/5.5.3/pr_27240.patch b/server-ce/hotfix/5.5.3/pr_27240.patch new file mode 100644 index 0000000000..f205bf2091 --- /dev/null +++ b/server-ce/hotfix/5.5.3/pr_27240.patch @@ -0,0 +1,84 @@ +diff --git a/cron/deactivate-projects.sh b/cron/deactivate-projects.sh +index fab0fbfbf667..a391f99a5bd8 100755 +--- a/cron/deactivate-projects.sh ++++ b/cron/deactivate-projects.sh +@@ -1,6 +1,6 @@ + #!/usr/bin/env bash + +-set -eux ++set -eu + + echo "-------------------------" + echo "Deactivating old projects" +diff --git a/cron/delete-projects.sh b/cron/delete-projects.sh +index e1ea5ac5e621..7cd45771716a 100755 +--- a/cron/delete-projects.sh ++++ b/cron/delete-projects.sh +@@ -1,6 +1,6 @@ + #!/usr/bin/env bash + +-set -eux ++set -eu + + echo "-------------------------" + echo "Expiring deleted projects" +diff --git a/cron/delete-users.sh b/cron/delete-users.sh +index fe97bffeea0b..30872ac55657 100755 +--- a/cron/delete-users.sh ++++ b/cron/delete-users.sh +@@ -1,6 +1,6 @@ + #!/usr/bin/env bash + +-set -eux ++set -eu + + echo "----------------------" + echo "Expiring deleted users" +diff --git a/cron/project-history-flush-all.sh b/cron/project-history-flush-all.sh +index d8bbb184aa37..8fe9eea5fc55 100755 +--- a/cron/project-history-flush-all.sh ++++ b/cron/project-history-flush-all.sh +@@ -1,6 +1,6 @@ + #!/usr/bin/env bash + +-set -eux ++set -eu + + echo "---------------------------------" + echo "Flush all project-history changes" +diff --git a/cron/project-history-periodic-flush.sh b/cron/project-history-periodic-flush.sh +index 76feae410e26..1b8efff6cc7c 100755 +--- a/cron/project-history-periodic-flush.sh ++++ b/cron/project-history-periodic-flush.sh +@@ -1,6 +1,6 @@ + #!/usr/bin/env bash + +-set -eux ++set -eu + + echo "--------------------------" + echo "Flush project-history queue" +diff --git a/cron/project-history-retry-hard.sh b/cron/project-history-retry-hard.sh +index 651a6615f22d..df9b4703a58e 100755 +--- a/cron/project-history-retry-hard.sh ++++ b/cron/project-history-retry-hard.sh +@@ -1,6 +1,6 @@ + #!/usr/bin/env bash + +-set -eux ++set -eu + + echo "-----------------------------------" + echo "Retry project-history errors (hard)" +diff --git a/cron/project-history-retry-soft.sh b/cron/project-history-retry-soft.sh +index 70c597021b28..cbb6e714cae7 100755 +--- a/cron/project-history-retry-soft.sh ++++ b/cron/project-history-retry-soft.sh +@@ -1,6 +1,6 @@ + #!/usr/bin/env bash + +-set -eux ++set -eu + + echo "-----------------------------------" + echo "Retry project-history errors (soft)" diff --git a/server-ce/hotfix/5.5.3/pr_27249.patch b/server-ce/hotfix/5.5.3/pr_27249.patch new file mode 100644 index 0000000000..60014f6b99 --- /dev/null +++ b/server-ce/hotfix/5.5.3/pr_27249.patch @@ -0,0 +1,76 @@ + + +diff --git a/package-lock.json b/package-lock.json +index 2b3a5868a20..d9d8285618d 100644 +--- a/package-lock.json ++++ b/package-lock.json +@@ -35581,6 +35581,7 @@ + "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", + "deprecated": "request has been deprecated, see https://github.com/request/request/issues/3142", ++ "license": "Apache-2.0", + "dependencies": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", +@@ -35638,15 +35639,15 @@ + } + }, + "node_modules/request/node_modules/tough-cookie": { +- "version": "2.5.0", +- "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", +- "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", ++ "version": "5.1.2", ++ "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz", ++ "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==", ++ "license": "BSD-3-Clause", + "dependencies": { +- "psl": "^1.1.28", +- "punycode": "^2.1.1" ++ "tldts": "^6.1.32" + }, + "engines": { +- "node": ">=0.8" ++ "node": ">=16" + } + }, + "node_modules/requestretry": { +@@ -39612,6 +39613,24 @@ + "tlds": "bin.js" + } + }, ++ "node_modules/tldts": { ++ "version": "6.1.86", ++ "resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz", ++ "integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==", ++ "license": "MIT", ++ "dependencies": { ++ "tldts-core": "^6.1.86" ++ }, ++ "bin": { ++ "tldts": "bin/cli.js" ++ } ++ }, ++ "node_modules/tldts-core": { ++ "version": "6.1.86", ++ "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz", ++ "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==", ++ "license": "MIT" ++ }, + "node_modules/tmp": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", +diff --git a/package.json b/package.json +index 388b750c3d2..44fffc4664a 100644 +--- a/package.json ++++ b/package.json +@@ -33,6 +33,9 @@ + "multer": "2.0.1", + "path-to-regexp": "3.3.0", + "qs": "6.13.0" ++ }, ++ "request@2.88.2": { ++ "tough-cookie": "5.1.2" + } + }, + "scripts": { + diff --git a/server-ce/hotfix/5.5.3/pr_27257.patch b/server-ce/hotfix/5.5.3/pr_27257.patch new file mode 100644 index 0000000000..ca7e47c84b --- /dev/null +++ b/server-ce/hotfix/5.5.3/pr_27257.patch @@ -0,0 +1,1469 @@ + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 0ccadaf5a95..4111c42c4d1 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -111,10 +111,8 @@ function parseArgs() { + if (['true', 'false'].includes(v)) return v === 'true' + throw new Error(`expected "true" or "false" for boolean option ${name}`) + } +- const BATCH_RANGE_START = objectIdFromInput( +- args['BATCH_RANGE_START'] +- ).toString() +- const BATCH_RANGE_END = objectIdFromInput(args['BATCH_RANGE_END']).toString() ++ const BATCH_RANGE_START = objectIdFromInput(args.BATCH_RANGE_START).toString() ++ const BATCH_RANGE_END = objectIdFromInput(args.BATCH_RANGE_END).toString() + return { + PROCESS_NON_DELETED_PROJECTS: boolVal('processNonDeletedProjects'), + PROCESS_DELETED_PROJECTS: boolVal('processDeletedProjects'), +@@ -122,8 +120,8 @@ function parseArgs() { + PROCESS_HASHED_FILES: boolVal('processHashedFiles'), + BATCH_RANGE_START, + BATCH_RANGE_END, +- LOGGING_IDENTIFIER: args['LOGGING_IDENTIFIER'] || BATCH_RANGE_START, +- PROJECT_IDS_FROM: args['projectIdsFrom'], ++ LOGGING_IDENTIFIER: args.LOGGING_IDENTIFIER || BATCH_RANGE_START, ++ PROJECT_IDS_FROM: args.projectIdsFrom, + } + } + +@@ -249,8 +247,8 @@ let lastEventLoopStats = performance.eventLoopUtilization() + * @param {number} ms + */ + function toMiBPerSecond(v, ms) { +- const ONE_MiB = 1024 * 1024 +- return v / ONE_MiB / (ms / 1000) ++ const MiB = 1024 * 1024 ++ return v / MiB / (ms / 1000) + } + + /** + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 4111c42c4d1..2d55b41b43e 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -84,11 +84,11 @@ ObjectId.cacheHexString = true + function parseArgs() { + const PUBLIC_LAUNCH_DATE = new Date('2012-01-01T00:00:00Z') + const args = commandLineArgs([ +- { name: 'processNonDeletedProjects', type: String, defaultValue: 'false' }, +- { name: 'processDeletedProjects', type: String, defaultValue: 'false' }, +- { name: 'processHashedFiles', type: String, defaultValue: 'false' }, +- { name: 'processBlobs', type: String, defaultValue: 'true' }, +- { name: 'projectIdsFrom', type: String, defaultValue: '' }, ++ { name: 'projects', type: Boolean }, ++ { name: 'deleted-projects', type: Boolean }, ++ { name: 'include-hashed-files', type: Boolean }, ++ { name: 'skip-existing-blobs', type: Boolean }, ++ { name: 'from-file', type: String, defaultValue: '' }, + { + name: 'BATCH_RANGE_START', + type: String, +@@ -99,29 +99,20 @@ function parseArgs() { + type: String, + defaultValue: new Date().toISOString(), + }, +- { name: 'LOGGING_IDENTIFIER', type: String, defaultValue: '' }, ++ { name: 'logging-id', type: String, defaultValue: '' }, + ]) +- /** +- * commandLineArgs cannot handle --foo=false, so go the long way +- * @param {string} name +- * @return {boolean} +- */ +- function boolVal(name) { +- const v = args[name] +- if (['true', 'false'].includes(v)) return v === 'true' +- throw new Error(`expected "true" or "false" for boolean option ${name}`) +- } ++ + const BATCH_RANGE_START = objectIdFromInput(args.BATCH_RANGE_START).toString() + const BATCH_RANGE_END = objectIdFromInput(args.BATCH_RANGE_END).toString() + return { +- PROCESS_NON_DELETED_PROJECTS: boolVal('processNonDeletedProjects'), +- PROCESS_DELETED_PROJECTS: boolVal('processDeletedProjects'), +- PROCESS_BLOBS: boolVal('processBlobs'), +- PROCESS_HASHED_FILES: boolVal('processHashedFiles'), ++ PROCESS_NON_DELETED_PROJECTS: args.projects, ++ PROCESS_DELETED_PROJECTS: args['deleted-projects'], ++ PROCESS_HASHED_FILES: args['include-hashed-files'], ++ PROCESS_BLOBS: !args['skip-existing-blobs'], + BATCH_RANGE_START, + BATCH_RANGE_END, +- LOGGING_IDENTIFIER: args.LOGGING_IDENTIFIER || BATCH_RANGE_START, +- PROJECT_IDS_FROM: args.projectIdsFrom, ++ LOGGING_IDENTIFIER: args['logging-id'] || BATCH_RANGE_START, ++ PROJECT_IDS_FROM: args['from-file'], + } + } + +diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +index 62b0b1de25f..0f8bdbf3e1a 100644 +--- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs ++++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +@@ -544,8 +544,8 @@ describe('back_fill_file_hash script', function () { + process.argv0, + [ + 'storage/scripts/back_fill_file_hash.mjs', +- '--processNonDeletedProjects=true', +- '--processDeletedProjects=true', ++ '--projects', ++ '--deleted-projects', + ...args, + ], + { +@@ -854,7 +854,7 @@ describe('back_fill_file_hash script', function () { + // Practically, this is slow and moving it to the end of the tests gets us there most of the way. + it('should process nothing on re-run', async function () { + const rerun = await runScript( +- processHashedFiles ? ['--processHashedFiles=true'] : [], ++ processHashedFiles ? ['--include-hashed-files'] : [], + {}, + false + ) +@@ -1113,7 +1113,7 @@ describe('back_fill_file_hash script', function () { + output1 = await runScript([], {}) + }) + before('run script with hashed files', async function () { +- output2 = await runScript(['--processHashedFiles=true'], {}) ++ output2 = await runScript(['--include-hashed-files'], {}) + }) + it('should print stats for the first run without hashed files', function () { + expect(output1.stats).deep.equal(STATS_ALL) +@@ -1161,7 +1161,7 @@ describe('back_fill_file_hash script', function () { + let output + before('prepare environment', prepareEnvironment) + before('run script', async function () { +- output = await runScript(['--processHashedFiles=true'], {}) ++ output = await runScript(['--include-hashed-files'], {}) + }) + it('should print stats', function () { + expect(output.stats).deep.equal( +@@ -1263,10 +1263,10 @@ describe('back_fill_file_hash script', function () { + + let outputPart0, outputPart1 + before('run script on part 0', async function () { +- outputPart0 = await runScript([`--projectIdsFrom=${path0}`]) ++ outputPart0 = await runScript([`--from-file=${path0}`]) + }) + before('run script on part 1', async function () { +- outputPart1 = await runScript([`--projectIdsFrom=${path1}`]) ++ outputPart1 = await runScript([`--from-file=${path1}`]) + }) + + /** + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 2d55b41b43e..68ce4b67aa2 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -79,7 +79,7 @@ ObjectId.cacheHexString = true + */ + + /** +- * @return {{PROJECT_IDS_FROM: string, PROCESS_HASHED_FILES: boolean, LOGGING_IDENTIFIER: string, BATCH_RANGE_START: string, PROCESS_BLOBS: boolean, BATCH_RANGE_END: string, PROCESS_NON_DELETED_PROJECTS: boolean, PROCESS_DELETED_PROJECTS: boolean}} ++ * @return {{PROJECT_IDS_FROM: string, PROCESS_HASHED_FILES: boolean, LOGGING_IDENTIFIER: string, BATCH_RANGE_START: string, PROCESS_BLOBS: boolean, BATCH_RANGE_END: string, PROCESS_NON_DELETED_PROJECTS: boolean, PROCESS_DELETED_PROJECTS: boolean, DRY_RUN: boolean}} + */ + function parseArgs() { + const PUBLIC_LAUNCH_DATE = new Date('2012-01-01T00:00:00Z') +@@ -89,6 +89,7 @@ function parseArgs() { + { name: 'include-hashed-files', type: Boolean }, + { name: 'skip-existing-blobs', type: Boolean }, + { name: 'from-file', type: String, defaultValue: '' }, ++ { name: 'dry-run', type: Boolean }, + { + name: 'BATCH_RANGE_START', + type: String, +@@ -109,6 +110,7 @@ function parseArgs() { + PROCESS_DELETED_PROJECTS: args['deleted-projects'], + PROCESS_HASHED_FILES: args['include-hashed-files'], + PROCESS_BLOBS: !args['skip-existing-blobs'], ++ DRY_RUN: args['dry-run'], + BATCH_RANGE_START, + BATCH_RANGE_END, + LOGGING_IDENTIFIER: args['logging-id'] || BATCH_RANGE_START, +@@ -121,6 +123,7 @@ const { + PROCESS_DELETED_PROJECTS, + PROCESS_BLOBS, + PROCESS_HASHED_FILES, ++ DRY_RUN, + BATCH_RANGE_START, + BATCH_RANGE_END, + LOGGING_IDENTIFIER, +@@ -325,10 +328,12 @@ async function processFileWithCleanup(entry) { + try { + return await processFile(entry, filePath) + } finally { +- await Promise.all([ +- fs.promises.rm(filePath, { force: true }), +- fs.promises.rm(filePath + GZ_SUFFIX, { force: true }), +- ]) ++ if (!DRY_RUN) { ++ await Promise.all([ ++ fs.promises.rm(filePath, { force: true }), ++ fs.promises.rm(filePath + GZ_SUFFIX, { force: true }), ++ ]) ++ } + } + } + +@@ -383,6 +388,12 @@ async function processFileOnce(entry, filePath) { + // know the hash of. + return entry.hash + } ++ if (DRY_RUN) { ++ console.log( ++ `DRY-RUN: would process file ${fileId} for project ${projectId}` ++ ) ++ return 'dry-run' ++ } + const blobStore = new BlobStore(historyId) + STATS.readFromGCSCount++ + // make a fetch request to filestore itself + + + +diff --git a/libraries/logger/logging-manager.js b/libraries/logger/logging-manager.js +index edf922be72b..9fb4f284053 100644 +--- a/libraries/logger/logging-manager.js ++++ b/libraries/logger/logging-manager.js +@@ -11,7 +11,7 @@ const LoggingManager = { + /** + * @param {string} name - The name of the logger + */ +- initialize(name) { ++ initialize(name, options = {}) { + this.isProduction = + (process.env.NODE_ENV || '').toLowerCase() === 'production' + const isTest = (process.env.NODE_ENV || '').toLowerCase() === 'test' +@@ -27,7 +27,7 @@ const LoggingManager = { + req: Serializers.req, + res: Serializers.res, + }, +- streams: [this._getOutputStreamConfig()], ++ streams: options.streams ?? [this._getOutputStreamConfig()], + }) + this._setupRingBuffer() + this._setupLogLevelChecker() + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 68ce4b67aa2..a7f220ec362 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -79,10 +79,14 @@ ObjectId.cacheHexString = true + */ + + /** +- * @return {{PROJECT_IDS_FROM: string, PROCESS_HASHED_FILES: boolean, LOGGING_IDENTIFIER: string, BATCH_RANGE_START: string, PROCESS_BLOBS: boolean, BATCH_RANGE_END: string, PROCESS_NON_DELETED_PROJECTS: boolean, PROCESS_DELETED_PROJECTS: boolean, DRY_RUN: boolean}} ++ * @return {{PROJECT_IDS_FROM: string, PROCESS_HASHED_FILES: boolean, LOGGING_IDENTIFIER: string, BATCH_RANGE_START: string, PROCESS_BLOBS: boolean, BATCH_RANGE_END: string, PROCESS_NON_DELETED_PROJECTS: boolean, PROCESS_DELETED_PROJECTS: boolean, DRY_RUN: boolean, OUTPUT_FILE: string, PROCESS_BLOBS: boolean}} + */ + function parseArgs() { + const PUBLIC_LAUNCH_DATE = new Date('2012-01-01T00:00:00Z') ++ const DEFAULT_OUTPUT_FILE = `file-migration-${new Date() ++ .toISOString() ++ .replace(/[:.]/g, '_')}.log` ++ + const args = commandLineArgs([ + { name: 'projects', type: Boolean }, + { name: 'deleted-projects', type: Boolean }, +@@ -90,6 +94,7 @@ function parseArgs() { + { name: 'skip-existing-blobs', type: Boolean }, + { name: 'from-file', type: String, defaultValue: '' }, + { name: 'dry-run', type: Boolean }, ++ { name: 'output', type: String, defaultValue: DEFAULT_OUTPUT_FILE }, + { + name: 'BATCH_RANGE_START', + type: String, +@@ -111,6 +116,7 @@ function parseArgs() { + PROCESS_HASHED_FILES: args['include-hashed-files'], + PROCESS_BLOBS: !args['skip-existing-blobs'], + DRY_RUN: args['dry-run'], ++ OUTPUT_FILE: args.output, + BATCH_RANGE_START, + BATCH_RANGE_END, + LOGGING_IDENTIFIER: args['logging-id'] || BATCH_RANGE_START, +@@ -124,6 +130,7 @@ const { + PROCESS_BLOBS, + PROCESS_HASHED_FILES, + DRY_RUN, ++ OUTPUT_FILE, + BATCH_RANGE_START, + BATCH_RANGE_END, + LOGGING_IDENTIFIER, +@@ -158,6 +165,21 @@ const STREAM_HIGH_WATER_MARK = parseInt( + const LOGGING_INTERVAL = parseInt(process.env.LOGGING_INTERVAL || '60000', 10) + const SLEEP_BEFORE_EXIT = parseInt(process.env.SLEEP_BEFORE_EXIT || '1000', 10) + ++// Log output to a file ++logger.initialize('file-migration', { ++ streams: [ ++ { ++ stream: ++ OUTPUT_FILE === '-' ++ ? process.stdout ++ : fs.createWriteStream(OUTPUT_FILE, { flags: 'a' }), ++ }, ++ ], ++}) ++async function trackProgress(progress) { ++ logger.info({}, progress) ++} ++ + // Filestore endpoint location + const FILESTORE_HOST = process.env.FILESTORE_HOST || '127.0.0.1' + const FILESTORE_PORT = process.env.FILESTORE_PORT || '3009' +@@ -525,8 +547,9 @@ async function queueNextBatch(batch, prefix = 'rootFolder.0') { + const end = renderObjectId(batch[batch.length - 1]._id) + const deferred = processBatch(batch, prefix) + .then(() => { +- console.error(`Actually completed batch ending ${end}`) ++ logger.info({ end }, 'actually completed batch') + }) ++ + .catch(err => { + logger.error({ err, start, end }, 'fatal error processing batch') + throw err +@@ -1062,6 +1085,7 @@ async function processNonDeletedProjects() { + { + BATCH_RANGE_START, + BATCH_RANGE_END, ++ trackProgress, + } + ) + } catch (err) { +diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +index 0f8bdbf3e1a..117352d6164 100644 +--- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs ++++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +@@ -544,6 +544,7 @@ describe('back_fill_file_hash script', function () { + process.argv0, + [ + 'storage/scripts/back_fill_file_hash.mjs', ++ '--output=-', + '--projects', + '--deleted-projects', + ...args, + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index a7f220ec362..4beba19cf4c 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -88,6 +88,7 @@ function parseArgs() { + .replace(/[:.]/g, '_')}.log` + + const args = commandLineArgs([ ++ { name: 'all', alias: 'a', type: Boolean }, + { name: 'projects', type: Boolean }, + { name: 'deleted-projects', type: Boolean }, + { name: 'include-hashed-files', type: Boolean }, +@@ -108,6 +109,36 @@ function parseArgs() { + { name: 'logging-id', type: String, defaultValue: '' }, + ]) + ++ // If no arguments are provided, display a usage message ++ if (process.argv.length <= 2) { ++ console.error( ++ 'Usage: node back_fill_file_hash.mjs --all | --projects | --deleted-projects' ++ ) ++ process.exit(1) ++ } ++ ++ // Require at least one of --projects, --deleted-projects and --all ++ if (!args.projects && !args['deleted-projects'] && !args.all) { ++ console.error( ++ 'Must specify at least one of --projects and --deleted-projects, or --all' ++ ) ++ process.exit(1) ++ } ++ ++ // Forbid --all with --projects or --deleted-projects ++ if (args.all && (args.projects || args['deleted-projects'])) { ++ console.error('Cannot use --all with --projects or --deleted-projects') ++ process.exit(1) ++ } ++ ++ // The --all option processes all projects, including deleted ones ++ // and checks existing hashed files are present in the blob store. ++ if (args.all) { ++ args.projects = true ++ args['deleted-projects'] = true ++ args['include-hashed-files'] = true ++ } ++ + const BATCH_RANGE_START = objectIdFromInput(args.BATCH_RANGE_START).toString() + const BATCH_RANGE_END = objectIdFromInput(args.BATCH_RANGE_END).toString() + return { + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 4beba19cf4c..492c5ad939d 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -88,6 +88,7 @@ function parseArgs() { + .replace(/[:.]/g, '_')}.log` + + const args = commandLineArgs([ ++ { name: 'help', alias: 'h', type: Boolean }, + { name: 'all', alias: 'a', type: Boolean }, + { name: 'projects', type: Boolean }, + { name: 'deleted-projects', type: Boolean }, +@@ -117,6 +118,48 @@ function parseArgs() { + process.exit(1) + } + ++ // If --help is provided, display the help message ++ if (args.help) { ++ console.log(`Usage: node back_fill_file_hash.mjs [options] ++ ++Project selection options: ++ --all, -a Process all projects, including deleted ones ++ --projects Process projects (excluding deleted ones) ++ --deleted-projects Process deleted projects ++ --from-file Process selected projects ids from file ++ ++File selection options: ++ --include-hashed-files Process files that already have a hash ++ --skip-existing-blobs Skip processing files already in the blob store ++ ++Logging options: ++ --output Output log to the specified file ++ (default: file-migration-.log) ++ --logging-id Identifier for logging ++ (default: BATCH_RANGE_START) ++ ++Batch range options: ++ --BATCH_RANGE_START Start date for processing ++ (default: ${args.BATCH_RANGE_START}) ++ --BATCH_RANGE_END End date for processing ++ (default: ${args.BATCH_RANGE_END}) ++ ++Other options: ++ --dry-run Perform a dry run without making changes ++ --help, -h Show this help message ++ ++Typical usage: ++ ++ node back_fill_file_hash.mjs --all ++ ++is equivalent to ++ ++ node back_fill_file_hash.mjs --projects --deleted-projects \\ ++ --include-hashed-files ++`) ++ process.exit(0) ++ } ++ + // Require at least one of --projects, --deleted-projects and --all + if (!args.projects && !args['deleted-projects'] && !args.all) { + console.error( + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 492c5ad939d..b20e365c4ff 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -79,7 +79,7 @@ ObjectId.cacheHexString = true + */ + + /** +- * @return {{PROJECT_IDS_FROM: string, PROCESS_HASHED_FILES: boolean, LOGGING_IDENTIFIER: string, BATCH_RANGE_START: string, PROCESS_BLOBS: boolean, BATCH_RANGE_END: string, PROCESS_NON_DELETED_PROJECTS: boolean, PROCESS_DELETED_PROJECTS: boolean, DRY_RUN: boolean, OUTPUT_FILE: string, PROCESS_BLOBS: boolean}} ++ * @return {{PROJECT_IDS_FROM: string, PROCESS_HASHED_FILES: boolean, LOGGING_IDENTIFIER: string, BATCH_RANGE_START: string, BATCH_RANGE_END: string, PROCESS_NON_DELETED_PROJECTS: boolean, PROCESS_DELETED_PROJECTS: boolean, PROCESS_BLOBS: boolean, DRY_RUN: boolean, OUTPUT_FILE: string, DISPLAY_REPORT: boolean}} + */ + function parseArgs() { + const PUBLIC_LAUNCH_DATE = new Date('2012-01-01T00:00:00Z') +@@ -97,6 +97,7 @@ function parseArgs() { + { name: 'from-file', type: String, defaultValue: '' }, + { name: 'dry-run', type: Boolean }, + { name: 'output', type: String, defaultValue: DEFAULT_OUTPUT_FILE }, ++ { name: 'report', type: Boolean }, + { + name: 'BATCH_RANGE_START', + type: String, +@@ -145,6 +146,7 @@ Batch range options: + (default: ${args.BATCH_RANGE_END}) + + Other options: ++ --report Display a report of the current status + --dry-run Perform a dry run without making changes + --help, -h Show this help message + +@@ -160,10 +162,15 @@ is equivalent to + process.exit(0) + } + +- // Require at least one of --projects, --deleted-projects and --all +- if (!args.projects && !args['deleted-projects'] && !args.all) { ++ // Require at least one of --projects, --deleted-projects and --all or --report ++ if ( ++ !args.projects && ++ !args['deleted-projects'] && ++ !args.all && ++ !args.report ++ ) { + console.error( +- 'Must specify at least one of --projects and --deleted-projects, or --all' ++ 'Must specify at least one of --projects and --deleted-projects, --all or --report' + ) + process.exit(1) + } +@@ -174,6 +181,14 @@ is equivalent to + process.exit(1) + } + ++ // Forbid --all, --projects, --deleted-projects with --report ++ if (args.report && (args.all || args.projects || args['deleted-projects'])) { ++ console.error( ++ 'Cannot use --report with --all, --projects or --deleted-projects' ++ ) ++ process.exit(1) ++ } ++ + // The --all option processes all projects, including deleted ones + // and checks existing hashed files are present in the blob store. + if (args.all) { +@@ -195,6 +210,7 @@ is equivalent to + BATCH_RANGE_END, + LOGGING_IDENTIFIER: args['logging-id'] || BATCH_RANGE_START, + PROJECT_IDS_FROM: args['from-file'], ++ DISPLAY_REPORT: args.report, + } + } + +@@ -209,6 +225,7 @@ const { + BATCH_RANGE_END, + LOGGING_IDENTIFIER, + PROJECT_IDS_FROM, ++ DISPLAY_REPORT, + } = parseArgs() + + // We need to handle the start and end differently as ids of deleted projects are created at time of deletion. +@@ -254,6 +271,108 @@ async function trackProgress(progress) { + logger.info({}, progress) + } + ++/** ++ * Display the stats for the projects or deletedProjects collections. ++ * ++ * @param {number} N - Number of samples to take from the collection. ++ * @param {string} name - Name of the collection being sampled. ++ * @param {Collection} collection - MongoDB collection to query. ++ * @param {Object} query - MongoDB query to filter documents. ++ * @param {Object} projection - MongoDB projection to select fields. ++ * @param {number} collectionCount - Total number of documents in the collection. ++ * @returns {Promise} Resolves when stats have been displayed. ++ */ ++async function getStatsForCollection( ++ N, ++ name, ++ collection, ++ query, ++ projection, ++ collectionCount ++) { ++ const stats = { ++ projectCount: 0, ++ projectsWithAllHashes: 0, ++ fileCount: 0, ++ fileWithHashCount: 0, ++ } ++ // Pick a random sample of projects and estimate the number of files without hashes ++ const result = await collection ++ .aggregate([ ++ { $sample: { size: N } }, ++ { $match: query }, ++ { ++ $project: projection, ++ }, ++ ]) ++ .toArray() ++ ++ for (const project of result) { ++ const fileTree = JSON.stringify(project, [ ++ 'rootFolder', ++ 'folders', ++ 'fileRefs', ++ 'hash', ++ '_id', ++ ]) ++ // count the number of files without a hash, these are uniquely identified ++ // by entries with {"_id":"...."} since we have filtered the file tree ++ const filesWithoutHash = fileTree.match(/\{"_id":"[0-9a-f]{24}"\}/g) || [] ++ // count the number of files with a hash, these are uniquely identified ++ // by the number of "hash" strings due to the filtering ++ const filesWithHash = fileTree.match(/"hash"/g) || [] ++ stats.fileCount += filesWithoutHash.length + filesWithHash.length ++ stats.fileWithHashCount += filesWithHash.length ++ stats.projectCount++ ++ stats.projectsWithAllHashes += filesWithoutHash.length === 0 ? 1 : 0 ++ } ++ console.log(`Sampled stats for ${name}:`) ++ const fractionSampled = stats.projectCount / collectionCount ++ const percentageSampled = (fractionSampled * 100).toFixed(1) ++ const fractionConverted = stats.projectsWithAllHashes / stats.projectCount ++ const percentageConverted = (fractionConverted * 100).toFixed(1) ++ console.log( ++ `- Sampled ${name}: ${stats.projectCount} (${percentageSampled}%)` ++ ) ++ console.log( ++ `- Sampled ${name} with all hashes present: ${stats.projectsWithAllHashes}` ++ ) ++ console.log( ++ `- Percentage of ${name} converted: ${percentageConverted}% (estimated)` ++ ) ++} ++ ++/** ++ * Displays a report of the current status of projects and deleted projects, ++ * including counts and estimated progress based on a sample. ++ */ ++async function displayReport() { ++ const projectsCountResult = await projectsCollection.countDocuments() ++ const deletedProjectsCountResult = ++ await deletedProjectsCollection.countDocuments() ++ const sampleSize = 1000 ++ console.log('Current status:') ++ console.log(`- Projects: ${projectsCountResult}`) ++ console.log(`- Deleted projects: ${deletedProjectsCountResult}`) ++ console.log(`Sampling ${sampleSize} projects to estimate progress...`) ++ await getStatsForCollection( ++ sampleSize, ++ 'projects', ++ projectsCollection, ++ { rootFolder: { $exists: true } }, ++ { rootFolder: 1 }, ++ projectsCountResult ++ ) ++ await getStatsForCollection( ++ sampleSize, ++ 'deleted projects', ++ deletedProjectsCollection, ++ { 'project.rootFolder': { $exists: true } }, ++ { 'project.rootFolder': 1 }, ++ deletedProjectsCountResult ++ ) ++} ++ + // Filestore endpoint location + const FILESTORE_HOST = process.env.FILESTORE_HOST || '127.0.0.1' + const FILESTORE_PORT = process.env.FILESTORE_PORT || '3009' +@@ -1220,6 +1339,12 @@ async function main() { + console.warn('Done.') + } + ++if (DISPLAY_REPORT) { ++ console.warn('Displaying report...') ++ await displayReport() ++ process.exit(0) ++} ++ + try { + try { + await main() + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index b20e365c4ff..2bfc4051622 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -267,8 +267,20 @@ logger.initialize('file-migration', { + }, + ], + }) ++ ++let lastElapsedTime = 0 + async function trackProgress(progress) { +- logger.info({}, progress) ++ const elapsedTime = Math.floor((performance.now() - processStart) / 1000) ++ if (lastElapsedTime === elapsedTime) { ++ // Avoid spamming the console with the same progress message ++ return ++ } ++ lastElapsedTime = elapsedTime ++ readline.clearLine(process.stdout, 0) ++ readline.cursorTo(process.stdout, 0) ++ process.stdout.write( ++ `Processed ${STATS.projects} projects, elapsed time ${elapsedTime}s` ++ ) + } + + /** +@@ -1287,7 +1299,7 @@ async function processNonDeletedProjects() { + } finally { + await waitForDeferredQueues() + } +- console.warn('Done updating live projects') ++ console.warn('\nDone updating live projects') + } + + async function processDeletedProjects() { +@@ -1306,7 +1318,9 @@ async function processDeletedProjects() { + 'project.rootFolder': 1, + 'project._id': 1, + 'project.overleaf.history.id': 1, +- } ++ }, ++ {}, ++ { trackProgress } + ) + } catch (err) { + gracefulShutdownInitiated = true +@@ -1314,7 +1328,7 @@ async function processDeletedProjects() { + } finally { + await waitForDeferredQueues() + } +- console.warn('Done updating deleted projects') ++ console.warn('\nDone updating deleted projects') + } + + async function main() { + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 2bfc4051622..c9fd7d233a7 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -94,9 +94,14 @@ function parseArgs() { + { name: 'deleted-projects', type: Boolean }, + { name: 'include-hashed-files', type: Boolean }, + { name: 'skip-existing-blobs', type: Boolean }, +- { name: 'from-file', type: String, defaultValue: '' }, +- { name: 'dry-run', type: Boolean }, +- { name: 'output', type: String, defaultValue: DEFAULT_OUTPUT_FILE }, ++ { name: 'from-file', alias: 'f', type: String, defaultValue: '' }, ++ { name: 'dry-run', alias: 'n', type: Boolean }, ++ { ++ name: 'output', ++ alias: 'o', ++ type: String, ++ defaultValue: DEFAULT_OUTPUT_FILE, ++ }, + { name: 'report', type: Boolean }, + { + name: 'BATCH_RANGE_START', +@@ -127,14 +132,14 @@ Project selection options: + --all, -a Process all projects, including deleted ones + --projects Process projects (excluding deleted ones) + --deleted-projects Process deleted projects +- --from-file Process selected projects ids from file ++ --from-file , -f Process selected projects ids from file + + File selection options: + --include-hashed-files Process files that already have a hash + --skip-existing-blobs Skip processing files already in the blob store + + Logging options: +- --output Output log to the specified file ++ --output , -o Output log to the specified file + (default: file-migration-.log) + --logging-id Identifier for logging + (default: BATCH_RANGE_START) +@@ -147,7 +152,7 @@ Batch range options: + + Other options: + --report Display a report of the current status +- --dry-run Perform a dry run without making changes ++ --dry-run, -n Perform a dry run without making changes + --help, -h Show this help message + + Typical usage: + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index c9fd7d233a7..8f28e8a4d78 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -326,6 +326,7 @@ async function getStatsForCollection( + + for (const project of result) { + const fileTree = JSON.stringify(project, [ ++ 'project', + 'rootFolder', + 'folders', + 'fileRefs', + + + +diff --git a/libraries/mongo-utils/batchedUpdate.js b/libraries/mongo-utils/batchedUpdate.js +index 41af41f0d4a..f1253c587d3 100644 +--- a/libraries/mongo-utils/batchedUpdate.js ++++ b/libraries/mongo-utils/batchedUpdate.js +@@ -35,7 +35,7 @@ let BATCHED_UPDATE_RUNNING = false + * @property {string} [BATCH_RANGE_START] + * @property {string} [BATCH_SIZE] + * @property {string} [VERBOSE_LOGGING] +- * @property {(progress: string) => Promise} [trackProgress] ++ * @property {(progress: string, options?: object) => Promise} [trackProgress] + */ + + /** +@@ -269,9 +269,12 @@ async function batchedUpdate( + await performUpdate(collection, nextBatch, update) + } + } +- await trackProgress(`Completed batch ending ${renderObjectId(end)}`) ++ await trackProgress(`Completed batch ending ${renderObjectId(end)}`, { ++ completedBatch: true, ++ }) + start = end + } ++ await trackProgress('Completed all batches', { completedAll: true }) + return updated + } finally { + BATCHED_UPDATE_RUNNING = false + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 8f28e8a4d78..2b54fdb1687 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -274,9 +274,16 @@ logger.initialize('file-migration', { + }) + + let lastElapsedTime = 0 +-async function trackProgress(progress) { ++async function trackProgress(progress, options = {}) { ++ if (OUTPUT_FILE === '-') { ++ return // skip progress tracking when logging to stdout ++ } ++ if (options.completedAll) { ++ process.stdout.write('\n') ++ return ++ } + const elapsedTime = Math.floor((performance.now() - processStart) / 1000) +- if (lastElapsedTime === elapsedTime) { ++ if (lastElapsedTime === elapsedTime && !options.completedBatch) { + // Avoid spamming the console with the same progress message + return + } +@@ -1305,7 +1312,7 @@ async function processNonDeletedProjects() { + } finally { + await waitForDeferredQueues() + } +- console.warn('\nDone updating live projects') ++ console.warn('Done updating live projects') + } + + async function processDeletedProjects() { +@@ -1334,7 +1341,7 @@ async function processDeletedProjects() { + } finally { + await waitForDeferredQueues() + } +- console.warn('\nDone updating deleted projects') ++ console.warn('Done updating deleted projects') + } + + async function main() { +@@ -1381,7 +1388,9 @@ try { + + let code = 0 + if (STATS.filesFailed > 0) { +- console.warn('Some files could not be processed, see logs and try again') ++ console.warn( ++ `Some files could not be processed, see logs in ${OUTPUT_FILE} and try again` ++ ) + code++ + } + if (STATS.fileHardDeleted > 0) { + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 2b54fdb1687..fc46f245d1a 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -525,7 +525,7 @@ function computeDiff(nextEventLoopStats, now) { + function printStats(isLast = false) { + const now = performance.now() + const nextEventLoopStats = performance.eventLoopUtilization() +- const logLine = JSON.stringify({ ++ const logLine = { + time: new Date(), + LOGGING_IDENTIFIER, + ...STATS, +@@ -533,11 +533,11 @@ function printStats(isLast = false) { + eventLoop: nextEventLoopStats, + diff: computeDiff(nextEventLoopStats, now), + deferredBatches: Array.from(deferredBatches.keys()), +- }) +- if (isLast) { +- console.warn(logLine) ++ } ++ if (isLast && OUTPUT_FILE === '-') { ++ console.warn(JSON.stringify(logLine)) + } else { +- console.log(logLine) ++ logger.info(logLine, 'file-migration stats') + } + lastEventLoopStats = nextEventLoopStats + lastLog = Object.assign({}, STATS) + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index fc46f245d1a..4a4d93d902c 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -92,7 +92,7 @@ function parseArgs() { + { name: 'all', alias: 'a', type: Boolean }, + { name: 'projects', type: Boolean }, + { name: 'deleted-projects', type: Boolean }, +- { name: 'include-hashed-files', type: Boolean }, ++ { name: 'skip-hashed-files', type: Boolean }, + { name: 'skip-existing-blobs', type: Boolean }, + { name: 'from-file', alias: 'f', type: String, defaultValue: '' }, + { name: 'dry-run', alias: 'n', type: Boolean }, +@@ -135,7 +135,7 @@ Project selection options: + --from-file , -f Process selected projects ids from file + + File selection options: +- --include-hashed-files Process files that already have a hash ++ --skip-hashed-files Skip processing files that already have a hash + --skip-existing-blobs Skip processing files already in the blob store + + Logging options: +@@ -161,8 +161,7 @@ Typical usage: + + is equivalent to + +- node back_fill_file_hash.mjs --projects --deleted-projects \\ +- --include-hashed-files ++ node back_fill_file_hash.mjs --projects --deleted-projects + `) + process.exit(0) + } +@@ -199,7 +198,6 @@ is equivalent to + if (args.all) { + args.projects = true + args['deleted-projects'] = true +- args['include-hashed-files'] = true + } + + const BATCH_RANGE_START = objectIdFromInput(args.BATCH_RANGE_START).toString() +@@ -207,7 +205,7 @@ is equivalent to + return { + PROCESS_NON_DELETED_PROJECTS: args.projects, + PROCESS_DELETED_PROJECTS: args['deleted-projects'], +- PROCESS_HASHED_FILES: args['include-hashed-files'], ++ PROCESS_HASHED_FILES: !args['skip-hashed-files'], + PROCESS_BLOBS: !args['skip-existing-blobs'], + DRY_RUN: args['dry-run'], + OUTPUT_FILE: args.output, +diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +index 117352d6164..a95bcbabd7e 100644 +--- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs ++++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +@@ -855,7 +855,7 @@ describe('back_fill_file_hash script', function () { + // Practically, this is slow and moving it to the end of the tests gets us there most of the way. + it('should process nothing on re-run', async function () { + const rerun = await runScript( +- processHashedFiles ? ['--include-hashed-files'] : [], ++ !processHashedFiles ? ['--skip-hashed-files'] : [], + {}, + false + ) +@@ -981,7 +981,7 @@ describe('back_fill_file_hash script', function () { + it('should gracefully handle fatal errors', async function () { + mockFilestore.deleteObject(projectId0, fileId0) + const t0 = Date.now() +- const { stats, result } = await tryRunScript([], { ++ const { stats, result } = await tryRunScript(['--skip-hashed-files'], { + RETRIES: '10', + RETRY_DELAY_MS: '1000', + }) +@@ -1016,7 +1016,7 @@ describe('back_fill_file_hash script', function () { + value: { stats, result }, + }, + ] = await Promise.allSettled([ +- tryRunScript([], { ++ tryRunScript(['--skip-hashed-files'], { + RETRY_DELAY_MS: '100', + RETRIES: '60', + RETRY_FILESTORE_404: 'true', // 404s are the easiest to simulate in tests +@@ -1042,7 +1042,7 @@ describe('back_fill_file_hash script', function () { + let output + before('prepare environment', prepareEnvironment) + before('run script', async function () { +- output = await runScript([], { ++ output = await runScript(['--skip-hashed-files'], { + CONCURRENCY: '1', + }) + }) +@@ -1111,10 +1111,10 @@ describe('back_fill_file_hash script', function () { + let output1, output2 + before('prepare environment', prepareEnvironment) + before('run script without hashed files', async function () { +- output1 = await runScript([], {}) ++ output1 = await runScript(['--skip-hashed-files'], {}) + }) + before('run script with hashed files', async function () { +- output2 = await runScript(['--include-hashed-files'], {}) ++ output2 = await runScript([], {}) + }) + it('should print stats for the first run without hashed files', function () { + expect(output1.stats).deep.equal(STATS_ALL) +@@ -1134,7 +1134,7 @@ describe('back_fill_file_hash script', function () { + let output + before('prepare environment', prepareEnvironment) + before('run script', async function () { +- output = await runScript([], { ++ output = await runScript(['--skip-hashed-files'], { + CONCURRENCY: '10', + }) + }) +@@ -1148,7 +1148,7 @@ describe('back_fill_file_hash script', function () { + let output + before('prepare environment', prepareEnvironment) + before('run script', async function () { +- output = await runScript([], { ++ output = await runScript(['--skip-hashed-files'], { + STREAM_HIGH_WATER_MARK: (1024 * 1024).toString(), + }) + }) +@@ -1162,7 +1162,7 @@ describe('back_fill_file_hash script', function () { + let output + before('prepare environment', prepareEnvironment) + before('run script', async function () { +- output = await runScript(['--include-hashed-files'], {}) ++ output = await runScript([], {}) + }) + it('should print stats', function () { + expect(output.stats).deep.equal( +@@ -1191,7 +1191,7 @@ describe('back_fill_file_hash script', function () { + }) + let output + before('run script', async function () { +- output = await runScript([], { ++ output = await runScript(['--skip-hashed-files'], { + CONCURRENCY: '1', + }) + }) +@@ -1212,14 +1212,20 @@ describe('back_fill_file_hash script', function () { + let outputPart0, outputPart1 + before('prepare environment', prepareEnvironment) + before('run script on part 0', async function () { +- outputPart0 = await runScript([`--BATCH_RANGE_END=${edge}`], { +- CONCURRENCY: '1', +- }) ++ outputPart0 = await runScript( ++ ['--skip-hashed-files', `--BATCH_RANGE_END=${edge}`], ++ { ++ CONCURRENCY: '1', ++ } ++ ) + }) + before('run script on part 1', async function () { +- outputPart1 = await runScript([`--BATCH_RANGE_START=${edge}`], { +- CONCURRENCY: '1', +- }) ++ outputPart1 = await runScript( ++ ['--skip-hashed-files', `--BATCH_RANGE_START=${edge}`], ++ { ++ CONCURRENCY: '1', ++ } ++ ) + }) + + it('should print stats for part 0', function () { +@@ -1264,10 +1270,16 @@ describe('back_fill_file_hash script', function () { + + let outputPart0, outputPart1 + before('run script on part 0', async function () { +- outputPart0 = await runScript([`--from-file=${path0}`]) ++ outputPart0 = await runScript([ ++ '--skip-hashed-files', ++ `--from-file=${path0}`, ++ ]) + }) + before('run script on part 1', async function () { +- outputPart1 = await runScript([`--from-file=${path1}`]) ++ outputPart1 = await runScript([ ++ '--skip-hashed-files', ++ `--from-file=${path1}`, ++ ]) + }) + + /** + + + +diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +index a95bcbabd7e..fc6941bd7bb 100644 +--- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs ++++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +@@ -975,7 +975,7 @@ describe('back_fill_file_hash script', function () { + STATS_UP_FROM_PROJECT1_ONWARD + ) + +- describe('error cases', () => { ++ describe('error cases', function () { + beforeEach('prepare environment', prepareEnvironment) + + it('should gracefully handle fatal errors', async function () { +@@ -1237,7 +1237,7 @@ describe('back_fill_file_hash script', function () { + commonAssertions() + }) + +- describe('projectIds from file', () => { ++ describe('projectIds from file', function () { + const path0 = '/tmp/project-ids-0.txt' + const path1 = '/tmp/project-ids-1.txt' + before('prepare environment', prepareEnvironment) + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 4a4d93d902c..375e582c331 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -555,7 +555,7 @@ function handleSignal() { + + /** + * @param {QueueEntry} entry +- * @return {Promise} ++ * @return {Promise} + */ + async function processFileWithCleanup(entry) { + const { +@@ -578,7 +578,7 @@ async function processFileWithCleanup(entry) { + /** + * @param {QueueEntry} entry + * @param {string} filePath +- * @return {Promise} ++ * @return {Promise} + */ + async function processFile(entry, filePath) { + for (let attempt = 0; attempt < RETRIES; attempt++) { +@@ -612,7 +612,7 @@ async function processFile(entry, filePath) { + /** + * @param {QueueEntry} entry + * @param {string} filePath +- * @return {Promise} ++ * @return {Promise} + */ + async function processFileOnce(entry, filePath) { + const { +@@ -627,10 +627,7 @@ async function processFileOnce(entry, filePath) { + return entry.hash + } + if (DRY_RUN) { +- console.log( +- `DRY-RUN: would process file ${fileId} for project ${projectId}` +- ) +- return 'dry-run' ++ return // skip processing in dry-run mode by returning undefined + } + const blobStore = new BlobStore(historyId) + STATS.readFromGCSCount++ +@@ -843,6 +840,9 @@ async function handleDeletedFileTreeBatch(batch) { + * @return {Promise} + */ + async function tryUpdateFileRefInMongo(entry) { ++ if (DRY_RUN) { ++ return true // skip mongo updates in dry-run mode ++ } + if (entry.path.startsWith('project.')) { + return await tryUpdateFileRefInMongoInDeletedProject(entry) + } +@@ -865,6 +865,9 @@ async function tryUpdateFileRefInMongo(entry) { + * @return {Promise} + */ + async function tryUpdateFileRefInMongoInDeletedProject(entry) { ++ if (DRY_RUN) { ++ return true // skip mongo updates in dry-run mode ++ } + STATS.mongoUpdates++ + const result = await deletedProjectsCollection.updateOne( + { +@@ -1165,6 +1168,7 @@ class ProjectContext { + */ + async #tryBatchHashWrites(collection, entries, query) { + if (entries.length === 0) return [] ++ if (DRY_RUN) return [] // skip mongo updates in dry-run mode + const update = {} + for (const entry of entries) { + query[`${entry.path}._id`] = new ObjectId(entry.fileId) +@@ -1210,7 +1214,7 @@ class ProjectContext { + } + } + +- /** @type {Map>} */ ++ /** @type {Map>} */ + #pendingFiles = new Map() + + /** +@@ -1223,7 +1227,12 @@ class ProjectContext { + this.#pendingFiles.set(entry.cacheKey, processFileWithCleanup(entry)) + } + try { +- entry.hash = await this.#pendingFiles.get(entry.cacheKey) ++ const hash = await this.#pendingFiles.get(entry.cacheKey) ++ if (!hash) { ++ return // hash is undefined in dry-run mode ++ } else { ++ entry.hash = hash ++ } + } finally { + this.remainingQueueEntries-- + } +diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +index fc6941bd7bb..646e75e2b58 100644 +--- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs ++++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +@@ -1130,6 +1130,45 @@ describe('back_fill_file_hash script', function () { + commonAssertions(true) + }) + ++ describe('full run in dry-run mode', function () { ++ let output ++ before('prepare environment', prepareEnvironment) ++ before('run script', async function () { ++ output = await runScript( ++ ['--dry-run'], ++ { ++ CONCURRENCY: '1', ++ }, ++ false ++ ) ++ }) ++ ++ it('should print stats for dry-run mode', function () { ++ // Compute the stats for running the script without dry-run mode. ++ const originalStats = sumStats(STATS_ALL, { ++ ...STATS_FILES_HASHED_EXTRA, ++ readFromGCSCount: 30, ++ readFromGCSIngress: 72, ++ mongoUpdates: 0, ++ filesWithHash: 3, ++ }) ++ // For a dry-run mode, we expect the stats to be zero except for the ++ // count of projects, blobs, bad file trees, duplicated files ++ // and files with/without hash. All the other stats such as mongoUpdates ++ // and writeToGCSCount, etc should be zero. ++ const expectedDryRunStats = { ++ ...STATS_ALL_ZERO, ++ projects: originalStats.projects, ++ blobs: originalStats.blobs, ++ badFileTrees: originalStats.badFileTrees, ++ filesDuplicated: originalStats.filesDuplicated, ++ filesWithHash: originalStats.filesWithHash, ++ filesWithoutHash: originalStats.filesWithoutHash, ++ } ++ expect(output.stats).deep.equal(expectedDryRunStats) ++ }) ++ }) ++ + describe('full run CONCURRENCY=10', function () { + let output + before('prepare environment', prepareEnvironment) + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 375e582c331..85920bcf03a 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -94,7 +94,7 @@ function parseArgs() { + { name: 'deleted-projects', type: Boolean }, + { name: 'skip-hashed-files', type: Boolean }, + { name: 'skip-existing-blobs', type: Boolean }, +- { name: 'from-file', alias: 'f', type: String, defaultValue: '' }, ++ { name: 'from-file', type: String, defaultValue: '' }, + { name: 'dry-run', alias: 'n', type: Boolean }, + { + name: 'output', +@@ -132,7 +132,7 @@ Project selection options: + --all, -a Process all projects, including deleted ones + --projects Process projects (excluding deleted ones) + --deleted-projects Process deleted projects +- --from-file , -f Process selected projects ids from file ++ --from-file Process selected projects ids from file + + File selection options: + --skip-hashed-files Skip processing files that already have a hash + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 85920bcf03a..092b8f04e43 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -567,10 +567,7 @@ async function processFileWithCleanup(entry) { + return await processFile(entry, filePath) + } finally { + if (!DRY_RUN) { +- await Promise.all([ +- fs.promises.rm(filePath, { force: true }), +- fs.promises.rm(filePath + GZ_SUFFIX, { force: true }), +- ]) ++ await fs.promises.rm(filePath, { force: true }) + } + } + } +@@ -697,8 +694,6 @@ async function uploadBlobToGCS(blobStore, entry, blob, hash, filePath) { + entry.ctx.recordHistoryBlob(blob) + } + +-const GZ_SUFFIX = '.gz' +- + /** + * @param {Array} files + * @return {Promise} + + + +diff --git a/libraries/mongo-utils/batchedUpdate.js b/libraries/mongo-utils/batchedUpdate.js +index f1253c587d3..41af41f0d4a 100644 +--- a/libraries/mongo-utils/batchedUpdate.js ++++ b/libraries/mongo-utils/batchedUpdate.js +@@ -35,7 +35,7 @@ let BATCHED_UPDATE_RUNNING = false + * @property {string} [BATCH_RANGE_START] + * @property {string} [BATCH_SIZE] + * @property {string} [VERBOSE_LOGGING] +- * @property {(progress: string, options?: object) => Promise} [trackProgress] ++ * @property {(progress: string) => Promise} [trackProgress] + */ + + /** +@@ -269,12 +269,9 @@ async function batchedUpdate( + await performUpdate(collection, nextBatch, update) + } + } +- await trackProgress(`Completed batch ending ${renderObjectId(end)}`, { +- completedBatch: true, +- }) ++ await trackProgress(`Completed batch ending ${renderObjectId(end)}`) + start = end + } +- await trackProgress('Completed all batches', { completedAll: true }) + return updated + } finally { + BATCHED_UPDATE_RUNNING = false +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 092b8f04e43..755443adf52 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -1305,7 +1305,7 @@ async function processNonDeletedProjects() { + { + BATCH_RANGE_START, + BATCH_RANGE_END, +- trackProgress, ++ trackProgress: async message => {}, + } + ) + } catch (err) { +@@ -1335,7 +1335,7 @@ async function processDeletedProjects() { + 'project.overleaf.history.id': 1, + }, + {}, +- { trackProgress } ++ { trackProgress: async message => {} } + ) + } catch (err) { + gracefulShutdownInitiated = true + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 755443adf52..4ca17ddf694 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -272,7 +272,7 @@ logger.initialize('file-migration', { + }) + + let lastElapsedTime = 0 +-async function trackProgress(progress, options = {}) { ++async function displayProgress(options = {}) { + if (OUTPUT_FILE === '-') { + return // skip progress tracking when logging to stdout + } +@@ -733,6 +733,7 @@ async function waitForDeferredQueues() { + // Wait for ALL pending batches to finish, especially wait for their mongo + // writes to finish to avoid extra work when resuming the batch. + const all = await Promise.allSettled(deferredBatches.values()) ++ displayProgress({ completedAll: true }) + // Now that all batches finished, we can throw if needed. + for (const res of all) { + if (res.status === 'rejected') { +@@ -756,6 +757,7 @@ async function queueNextBatch(batch, prefix = 'rootFolder.0') { + const deferred = processBatch(batch, prefix) + .then(() => { + logger.info({ end }, 'actually completed batch') ++ displayProgress({ completedBatch: true }) + }) + + .catch(err => { + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index 4ca17ddf694..8664be21fbe 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -1226,7 +1226,11 @@ class ProjectContext { + try { + const hash = await this.#pendingFiles.get(entry.cacheKey) + if (!hash) { +- return // hash is undefined in dry-run mode ++ if (DRY_RUN) { ++ return // hash is undefined in dry-run mode ++ } else { ++ throw new Error('undefined hash outside dry-run mode') ++ } + } else { + entry.hash = hash + } + + + +diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +index 646e75e2b58..43884adbe8f 100644 +--- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs ++++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +@@ -1132,7 +1132,15 @@ describe('back_fill_file_hash script', function () { + + describe('full run in dry-run mode', function () { + let output ++ let projectRecordsBefore ++ let deletedProjectRecordsBefore + before('prepare environment', prepareEnvironment) ++ before(async function () { ++ projectRecordsBefore = await projectsCollection.find({}).toArray() ++ deletedProjectRecordsBefore = await deletedProjectsCollection ++ .find({}) ++ .toArray() ++ }) + before('run script', async function () { + output = await runScript( + ['--dry-run'], +@@ -1167,6 +1175,14 @@ describe('back_fill_file_hash script', function () { + } + expect(output.stats).deep.equal(expectedDryRunStats) + }) ++ it('should not update mongo', async function () { ++ expect(await projectsCollection.find({}).toArray()).to.deep.equal( ++ projectRecordsBefore ++ ) ++ expect(await deletedProjectsCollection.find({}).toArray()).to.deep.equal( ++ deletedProjectRecordsBefore ++ ) ++ }) + }) + + describe('full run CONCURRENCY=10', function () { + diff --git a/server-ce/hotfix/5.5.3/pr_27273.patch b/server-ce/hotfix/5.5.3/pr_27273.patch new file mode 100644 index 0000000000..b0c0822fb5 --- /dev/null +++ b/server-ce/hotfix/5.5.3/pr_27273.patch @@ -0,0 +1,82 @@ + + +diff --git a/services/web/frontend/js/features/review-panel-new/components/review-tooltip-menu.tsx b/services/web/frontend/js/features/review-panel-new/components/review-tooltip-menu.tsx +index f26542ebe909..fb6b68460bdc 100644 +--- a/services/web/frontend/js/features/review-panel-new/components/review-tooltip-menu.tsx ++++ b/services/web/frontend/js/features/review-panel-new/components/review-tooltip-menu.tsx +@@ -18,7 +18,6 @@ import { + reviewTooltipStateField, + } from '@/features/source-editor/extensions/review-tooltip' + import { EditorView, getTooltip } from '@codemirror/view' +-import useViewerPermissions from '@/shared/hooks/use-viewer-permissions' + import usePreviousValue from '@/shared/hooks/use-previous-value' + import { useLayoutContext } from '@/shared/context/layout-context' + import { useReviewPanelViewActionsContext } from '../context/review-panel-view-context' +@@ -35,6 +34,7 @@ import { useEditorPropertiesContext } from '@/features/ide-react/context/editor- + import classNames from 'classnames' + import useEventListener from '@/shared/hooks/use-event-listener' + import useReviewPanelLayout from '../hooks/use-review-panel-layout' ++import { usePermissionsContext } from '@/features/ide-react/context/permissions-context' + + const EDIT_MODE_SWITCH_WIDGET_HEIGHT = 40 + const CM_LINE_RIGHT_PADDING = 8 +@@ -43,7 +43,7 @@ const TOOLTIP_SHOW_DELAY = 120 + const ReviewTooltipMenu: FC = () => { + const state = useCodeMirrorStateContext() + const view = useCodeMirrorViewContext() +- const isViewer = useViewerPermissions() ++ const permissions = usePermissionsContext() + const [show, setShow] = useState(true) + const { setView } = useReviewPanelViewActionsContext() + const { openReviewPanel } = useReviewPanelLayout() +@@ -58,7 +58,7 @@ const ReviewTooltipMenu: FC = () => { + + const addComment = useCallback(() => { + const { main } = view.state.selection +- if (main.empty) { ++ if (main.empty || !permissions.comment) { + return + } + +@@ -74,11 +74,11 @@ const ReviewTooltipMenu: FC = () => { + + view.dispatch({ effects }) + setShow(false) +- }, [openReviewPanel, setView, setShow, view]) ++ }, [view, permissions.comment, openReviewPanel, setView]) + + useEventListener('add-new-review-comment', addComment) + +- if (isViewer || !show || !tooltipState) { ++ if (!permissions.comment || !show || !tooltipState) { + return null + } + +diff --git a/services/web/frontend/js/features/source-editor/components/toolbar/toolbar-items.tsx b/services/web/frontend/js/features/source-editor/components/toolbar/toolbar-items.tsx +index 3404976d4462..1811ccc99950 100644 +--- a/services/web/frontend/js/features/source-editor/components/toolbar/toolbar-items.tsx ++++ b/services/web/frontend/js/features/source-editor/components/toolbar/toolbar-items.tsx +@@ -16,5 +16,6 @@ import { isSplitTestEnabled } from '@/utils/splitTestUtils' + import { isMac } from '@/shared/utils/os' + import { useProjectContext } from '@/shared/context/project-context' ++import { usePermissionsContext } from '@/features/ide-react/context/permissions-context' + + export const ToolbarItems: FC<{ + state: EditorState +@@ -35,6 +36,7 @@ export const ToolbarItems: FC<{ + useEditorPropertiesContext() + const { writefullInstance } = useEditorContext() + const { features } = useProjectContext() ++ const permissions = usePermissionsContext() + const isActive = withinFormattingCommand(state) + + const symbolPaletteAvailable = getMeta('ol-symbolPaletteAvailable') +@@ -131,7 +133,7 @@ export const ToolbarItems: FC<{ + command={commands.wrapInHref} + icon="add_link" + /> +- {features.trackChangesVisible && ( ++ {features.trackChangesVisible && permissions.comment && ( + b.getHash()) ++ ) ++ const uniqueHashes = new Set(filesWithHash.map(m => m.slice(8, 48))) ++ for (const hash of uniqueHashes) { ++ if (blobs.has(hash) || GLOBAL_BLOBS.has(hash)) continue ++ stats.fileMissingInHistoryCount++ ++ } + } + console.log(`Sampled stats for ${name}:`) + const fractionSampled = stats.projectCount / collectionCount +- const percentageSampled = (fractionSampled * 100).toFixed(1) ++ const percentageSampled = (fractionSampled * 100).toFixed(0) + const fractionConverted = stats.projectsWithAllHashes / stats.projectCount +- const percentageConverted = (fractionConverted * 100).toFixed(1) ++ const percentageConverted = (fractionConverted * 100).toFixed(0) ++ const fractionMissing = stats.fileMissingInHistoryCount / stats.fileCount ++ const percentageMissing = (fractionMissing * 100).toFixed(0) + console.log( +- `- Sampled ${name}: ${stats.projectCount} (${percentageSampled}%)` ++ `- Sampled ${name}: ${stats.projectCount} (${percentageSampled}% of all ${name})` + ) + console.log( + `- Sampled ${name} with all hashes present: ${stats.projectsWithAllHashes}` + ) + console.log( +- `- Percentage of ${name} converted: ${percentageConverted}% (estimated)` ++ `- Percentage of ${name} that need back-filling hashes: ${percentageConverted}% (estimated)` ++ ) ++ console.log( ++ `- Sampled ${name} have ${stats.fileCount} files that need to be checked against the full project history system.` ++ ) ++ console.log( ++ `- Sampled ${name} have ${stats.fileMissingInHistoryCount} files that need to be uploaded to the full project history system (estimating ${percentageMissing}% of all files).` + ) + } + +@@ -369,13 +388,15 @@ async function getStatsForCollection( + * including counts and estimated progress based on a sample. + */ + async function displayReport() { +- const projectsCountResult = await projectsCollection.countDocuments() ++ const projectsCountResult = await projectsCollection.estimatedDocumentCount() + const deletedProjectsCountResult = +- await deletedProjectsCollection.countDocuments() ++ await deletedProjectsCollection.estimatedDocumentCount() + const sampleSize = 1000 + console.log('Current status:') +- console.log(`- Projects: ${projectsCountResult}`) +- console.log(`- Deleted projects: ${deletedProjectsCountResult}`) ++ console.log(`- Total number of projects: ${projectsCountResult}`) ++ console.log( ++ `- Total number of deleted projects: ${deletedProjectsCountResult}` ++ ) + console.log(`Sampling ${sampleSize} projects to estimate progress...`) + await getStatsForCollection( + sampleSize, +diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +index c661ae9bc3f..7248e74cb3f 100644 +--- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs ++++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +@@ -481,21 +481,14 @@ describe('back_fill_file_hash script', function () { + /** + * @param {Array} args + * @param {Record} env +- * @param {boolean} shouldHaveWritten +- * @return {Promise<{result, stats: any}>} ++ * @return {Promise<{result: { stdout: string, stderr: string, status: number }, stats: any}>} + */ +- async function tryRunScript(args = [], env = {}, shouldHaveWritten) { ++ async function rawRunScript(args = [], env = {}) { + let result + try { + result = await promisify(execFile)( + process.argv0, +- [ +- 'storage/scripts/back_fill_file_hash.mjs', +- '--output=-', +- '--projects', +- '--deleted-projects', +- ...args, +- ], ++ ['storage/scripts/back_fill_file_hash.mjs', ...args], + { + encoding: 'utf-8', + timeout: TIMEOUT - 500, +@@ -521,6 +514,20 @@ describe('back_fill_file_hash script', function () { + expect((await fs.promises.readdir('/tmp')).join(';')).to.not.match( + /back_fill_file_hash/ + ) ++ return result ++ } ++ ++ /** ++ * @param {Array} args ++ * @param {Record} env ++ * @param {boolean} shouldHaveWritten ++ * @return {Promise<{result, stats: any}>} ++ */ ++ async function tryRunScript(args = [], env = {}, shouldHaveWritten) { ++ const result = await rawRunScript( ++ ['--output=-', '--projects', '--deleted-projects', ...args], ++ env ++ ) + const extraStatsKeys = ['eventLoop', 'readFromGCSThroughputMiBPerSecond'] + const stats = JSON.parse( + result.stderr +@@ -1078,6 +1085,35 @@ describe('back_fill_file_hash script', function () { + }) + commonAssertions(true) + }) ++ describe('report mode', function () { ++ let output ++ before('prepare environment', prepareEnvironment) ++ before('run script', async function () { ++ output = await rawRunScript(['--report'], {}) ++ }) ++ it('should print the report', () => { ++ expect(output.status).to.equal(0) ++ console.log(output.stdout) ++ expect(output.stdout).to.equal(`\ ++Current status: ++- Total number of projects: 10 ++- Total number of deleted projects: 5 ++Sampling 1000 projects to estimate progress... ++Sampled stats for projects: ++- Sampled projects: 9 (90% of all projects) ++- Sampled projects with all hashes present: 5 ++- Percentage of projects that need back-filling hashes: 56% (estimated) ++- Sampled projects have 11 files that need to be checked against the full project history system. ++- Sampled projects have 3 files that need to be uploaded to the full project history system (estimating 27% of all files). ++Sampled stats for deleted projects: ++- Sampled deleted projects: 4 (80% of all deleted projects) ++- Sampled deleted projects with all hashes present: 3 ++- Percentage of deleted projects that need back-filling hashes: 75% (estimated) ++- Sampled deleted projects have 2 files that need to be checked against the full project history system. ++- Sampled deleted projects have 1 files that need to be uploaded to the full project history system (estimating 50% of all files). ++`) ++ }) ++ }) + + describe('full run in dry-run mode', function () { + let output + + + +diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +index e9a7721944c..9c2a9818680 100644 +--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs ++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs +@@ -79,7 +79,7 @@ ObjectId.cacheHexString = true + */ + + /** +- * @return {{PROJECT_IDS_FROM: string, PROCESS_HASHED_FILES: boolean, LOGGING_IDENTIFIER: string, BATCH_RANGE_START: string, BATCH_RANGE_END: string, PROCESS_NON_DELETED_PROJECTS: boolean, PROCESS_DELETED_PROJECTS: boolean, PROCESS_BLOBS: boolean, DRY_RUN: boolean, OUTPUT_FILE: string, DISPLAY_REPORT: boolean}} ++ * @return {{PROJECT_IDS_FROM: string, PROCESS_HASHED_FILES: boolean, LOGGING_IDENTIFIER: string, BATCH_RANGE_START: string, BATCH_RANGE_END: string, PROCESS_NON_DELETED_PROJECTS: boolean, PROCESS_DELETED_PROJECTS: boolean, PROCESS_BLOBS: boolean, DRY_RUN: boolean, OUTPUT_FILE: string, DISPLAY_REPORT: boolean, CONCURRENCY: number, CONCURRENT_BATCHES: number, RETRIES: number, RETRY_DELAY_MS: number, RETRY_FILESTORE_404: boolean, BUFFER_DIR_PREFIX: string, STREAM_HIGH_WATER_MARK: number, LOGGING_INTERVAL: number, SLEEP_BEFORE_EXIT: number }} + */ + function parseArgs() { + const PUBLIC_LAUNCH_DATE = new Date('2012-01-01T00:00:00Z') +@@ -95,6 +95,12 @@ function parseArgs() { + { name: 'skip-hashed-files', type: Boolean }, + { name: 'skip-existing-blobs', type: Boolean }, + { name: 'from-file', type: String, defaultValue: '' }, ++ { name: 'concurrency', type: Number, defaultValue: 10 }, ++ { name: 'concurrent-batches', type: Number, defaultValue: 1 }, ++ { name: 'stream-high-water-mark', type: Number, defaultValue: 1024 * 1024 }, ++ { name: 'retries', type: Number, defaultValue: 10 }, ++ { name: 'retry-delay-ms', type: Number, defaultValue: 100 }, ++ { name: 'retry-filestore-404', type: Boolean }, + { name: 'dry-run', alias: 'n', type: Boolean }, + { + name: 'output', +@@ -114,6 +120,13 @@ function parseArgs() { + defaultValue: new Date().toISOString(), + }, + { name: 'logging-id', type: String, defaultValue: '' }, ++ { name: 'logging-interval-ms', type: Number, defaultValue: 60_000 }, ++ { ++ name: 'buffer-dir-prefix', ++ type: String, ++ defaultValue: '/tmp/back_fill_file_hash-', ++ }, ++ { name: 'sleep-before-exit-ms', type: Number, defaultValue: 1_000 }, + ]) + + // If no arguments are provided, display a usage message +@@ -143,6 +156,8 @@ Logging options: + (default: file-migration-.log) + --logging-id Identifier for logging + (default: BATCH_RANGE_START) ++ --logging-interval-ms Interval for logging progres stats ++ (default: 60000, 1min) + + Batch range options: + --BATCH_RANGE_START Start date for processing +@@ -150,10 +165,30 @@ Batch range options: + --BATCH_RANGE_END End date for processing + (default: ${args.BATCH_RANGE_END}) + ++Concurrency: ++ --concurrency Number of files to process concurrently ++ (default: 10) ++ --concurrent-batches Number of project batches to process concurrently ++ (default: 1) ++ --stream-high-water-mark n In-Memory buffering threshold ++ (default: 1MiB) ++ ++Retries: ++ --retries Number of times to retry processing a file ++ (default: 10) ++ --retry-delay-ms How long to wait before processing a file again ++ (default: 100, 100ms) ++ --retry-filestore-404 Retry downloading a file when receiving a 404 ++ (default: false) ++ + Other options: + --report Display a report of the current status + --dry-run, -n Perform a dry run without making changes + --help, -h Show this help message ++ --buffer-dir-prefix

Folder/prefix for buffering files on disk ++ (default: ${args['buffer-dir-prefix']}) ++ --sleep-before-exit-ms Defer exiting from the script ++ (default: 1000, 1s) + + Typical usage: + +@@ -212,8 +247,17 @@ is equivalent to + BATCH_RANGE_START, + BATCH_RANGE_END, + LOGGING_IDENTIFIER: args['logging-id'] || BATCH_RANGE_START, ++ LOGGING_INTERVAL: args['logging-interval-ms'], + PROJECT_IDS_FROM: args['from-file'], + DISPLAY_REPORT: args.report, ++ CONCURRENCY: args.concurrency, ++ CONCURRENT_BATCHES: args['concurrent-batches'], ++ STREAM_HIGH_WATER_MARK: args['stream-high-water-mark'], ++ RETRIES: args.retries, ++ RETRY_DELAY_MS: args['retry-delay-ms'], ++ RETRY_FILESTORE_404: args['retry-filestore-404'], ++ BUFFER_DIR_PREFIX: args['buffer-dir-prefix'], ++ SLEEP_BEFORE_EXIT: args['sleep-before-exit-ms'], + } + } + +@@ -229,6 +273,15 @@ const { + LOGGING_IDENTIFIER, + PROJECT_IDS_FROM, + DISPLAY_REPORT, ++ CONCURRENCY, ++ CONCURRENT_BATCHES, ++ RETRIES, ++ RETRY_DELAY_MS, ++ RETRY_FILESTORE_404, ++ BUFFER_DIR_PREFIX, ++ STREAM_HIGH_WATER_MARK, ++ LOGGING_INTERVAL, ++ SLEEP_BEFORE_EXIT, + } = parseArgs() + + // We need to handle the start and end differently as ids of deleted projects are created at time of deletion. +@@ -236,24 +289,7 @@ if (process.env.BATCH_RANGE_START || process.env.BATCH_RANGE_END) { + throw new Error('use --BATCH_RANGE_START and --BATCH_RANGE_END') + } + +-// Concurrency for downloading from GCS and updating hashes in mongo +-const CONCURRENCY = parseInt(process.env.CONCURRENCY || '100', 10) +-const CONCURRENT_BATCHES = parseInt(process.env.CONCURRENT_BATCHES || '2', 10) +-// Retries for processing a given file +-const RETRIES = parseInt(process.env.RETRIES || '10', 10) +-const RETRY_DELAY_MS = parseInt(process.env.RETRY_DELAY_MS || '100', 10) +- +-const RETRY_FILESTORE_404 = process.env.RETRY_FILESTORE_404 === 'true' +-const BUFFER_DIR = fs.mkdtempSync( +- process.env.BUFFER_DIR_PREFIX || '/tmp/back_fill_file_hash-' +-) +-// https://nodejs.org/api/stream.html#streamgetdefaulthighwatermarkobjectmode +-const STREAM_HIGH_WATER_MARK = parseInt( +- process.env.STREAM_HIGH_WATER_MARK || (64 * 1024).toString(), +- 10 +-) +-const LOGGING_INTERVAL = parseInt(process.env.LOGGING_INTERVAL || '60000', 10) +-const SLEEP_BEFORE_EXIT = parseInt(process.env.SLEEP_BEFORE_EXIT || '1000', 10) ++const BUFFER_DIR = fs.mkdtempSync(BUFFER_DIR_PREFIX) + + // Log output to a file + if (OUTPUT_FILE !== '-') { +@@ -416,7 +452,7 @@ async function displayReport() { + ) + } + +-// Filestore endpoint location ++// Filestore endpoint location (configured by /etc/overleaf/env.sh) + const FILESTORE_HOST = process.env.FILESTORE_HOST || '127.0.0.1' + const FILESTORE_PORT = process.env.FILESTORE_PORT || '3009' + +diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +index 7248e74cb3f..601cea13b6a 100644 +--- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs ++++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +@@ -61,9 +61,8 @@ function objectIdFromTime(timestamp) { + + const PRINT_IDS_AND_HASHES_FOR_DEBUGGING = false + +-describe('back_fill_file_hash script', function () { ++describe.only('back_fill_file_hash script', function () { + this.timeout(TIMEOUT) +- const USER_FILES_BUCKET_NAME = 'fake-user-files-gcs' + + const projectId0 = objectIdFromTime('2017-01-01T00:00:00Z') + const projectId1 = objectIdFromTime('2017-01-01T00:01:00Z') +@@ -480,24 +479,24 @@ describe('back_fill_file_hash script', function () { + + /** + * @param {Array} args +- * @param {Record} env + * @return {Promise<{result: { stdout: string, stderr: string, status: number }, stats: any}>} + */ +- async function rawRunScript(args = [], env = {}) { ++ async function rawRunScript(args = []) { + let result + try { + result = await promisify(execFile)( + process.argv0, +- ['storage/scripts/back_fill_file_hash.mjs', ...args], ++ [ ++ 'storage/scripts/back_fill_file_hash.mjs', ++ '--sleep-before-exit-ms=1', ++ ...args, ++ ], + { + encoding: 'utf-8', + timeout: TIMEOUT - 500, + env: { + ...process.env, + AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE: '1', +- USER_FILES_BUCKET_NAME, +- SLEEP_BEFORE_EXIT: '1', +- ...env, + LOG_LEVEL: 'warn', // Override LOG_LEVEL of acceptance tests + }, + } +@@ -519,15 +518,16 @@ describe('back_fill_file_hash script', function () { + + /** + * @param {Array} args +- * @param {Record} env + * @param {boolean} shouldHaveWritten + * @return {Promise<{result, stats: any}>} + */ +- async function tryRunScript(args = [], env = {}, shouldHaveWritten) { +- const result = await rawRunScript( +- ['--output=-', '--projects', '--deleted-projects', ...args], +- env +- ) ++ async function tryRunScript(args = [], shouldHaveWritten) { ++ const result = await rawRunScript([ ++ '--output=-', ++ '--projects', ++ '--deleted-projects', ++ ...args, ++ ]) + const extraStatsKeys = ['eventLoop', 'readFromGCSThroughputMiBPerSecond'] + const stats = JSON.parse( + result.stderr +@@ -558,12 +558,11 @@ describe('back_fill_file_hash script', function () { + + /** + * @param {Array} args +- * @param {Record} env + * @param {boolean} shouldHaveWritten + * @return {Promise<{result, stats: any}>} + */ +- async function runScript(args = [], env = {}, shouldHaveWritten = true) { +- const { stats, result } = await tryRunScript(args, env, shouldHaveWritten) ++ async function runScript(args = [], shouldHaveWritten = true) { ++ const { stats, result } = await tryRunScript(args, shouldHaveWritten) + if (result.status !== 0) { + console.log(result) + expect(result).to.have.property('status', 0) +@@ -812,7 +811,6 @@ describe('back_fill_file_hash script', function () { + it('should process nothing on re-run', async function () { + const rerun = await runScript( + !processHashedFiles ? ['--skip-hashed-files'] : [], +- {}, + false + ) + let stats = { +@@ -937,10 +935,11 @@ describe('back_fill_file_hash script', function () { + it('should gracefully handle fatal errors', async function () { + mockFilestore.deleteObject(projectId0, fileId0) + const t0 = Date.now() +- const { stats, result } = await tryRunScript(['--skip-hashed-files'], { +- RETRIES: '10', +- RETRY_DELAY_MS: '1000', +- }) ++ const { stats, result } = await tryRunScript([ ++ '--skip-hashed-files', ++ '--retries=10', ++ '--retry-delay-ms=1000', ++ ]) + const t1 = Date.now() + expectNotFoundError(result, 'failed to process file') + expect(result.status).to.equal(1) +@@ -972,11 +971,12 @@ describe('back_fill_file_hash script', function () { + value: { stats, result }, + }, + ] = await Promise.allSettled([ +- tryRunScript(['--skip-hashed-files'], { +- RETRY_DELAY_MS: '100', +- RETRIES: '60', +- RETRY_FILESTORE_404: 'true', // 404s are the easiest to simulate in tests +- }), ++ tryRunScript([ ++ '--skip-hashed-files', ++ '--retries=60', ++ '--retry-delay-ms=1000', ++ '--retry-filestore-404', ++ ]), + restoreFileAfter5s(), + ]) + expectNotFoundError(result, 'failed to process file, trying again') +@@ -998,9 +998,7 @@ describe('back_fill_file_hash script', function () { + let output + before('prepare environment', prepareEnvironment) + before('run script', async function () { +- output = await runScript(['--skip-hashed-files'], { +- CONCURRENCY: '1', +- }) ++ output = await runScript(['--skip-hashed-files', '--concurrency=1']) + }) + + /** +@@ -1067,10 +1065,10 @@ describe('back_fill_file_hash script', function () { + let output1, output2 + before('prepare environment', prepareEnvironment) + before('run script without hashed files', async function () { +- output1 = await runScript(['--skip-hashed-files'], {}) ++ output1 = await runScript(['--skip-hashed-files']) + }) + before('run script with hashed files', async function () { +- output2 = await runScript([], {}) ++ output2 = await runScript([]) + }) + it('should print stats for the first run without hashed files', function () { + expect(output1.stats).deep.equal(STATS_ALL) +@@ -1089,7 +1087,7 @@ describe('back_fill_file_hash script', function () { + let output + before('prepare environment', prepareEnvironment) + before('run script', async function () { +- output = await rawRunScript(['--report'], {}) ++ output = await rawRunScript(['--report']) + }) + it('should print the report', () => { + expect(output.status).to.equal(0) +@@ -1127,13 +1125,7 @@ Sampled stats for deleted projects: + .toArray() + }) + before('run script', async function () { +- output = await runScript( +- ['--dry-run'], +- { +- CONCURRENCY: '1', +- }, +- false +- ) ++ output = await runScript(['--dry-run', '--concurrency=1'], false) + }) + + it('should print stats for dry-run mode', function () { +@@ -1174,9 +1166,7 @@ Sampled stats for deleted projects: + let output + before('prepare environment', prepareEnvironment) + before('run script', async function () { +- output = await runScript(['--skip-hashed-files'], { +- CONCURRENCY: '10', +- }) ++ output = await runScript(['--skip-hashed-files', '--concurrency=10']) + }) + it('should print stats', function () { + expect(output.stats).deep.equal(STATS_ALL) +@@ -1184,13 +1174,14 @@ Sampled stats for deleted projects: + commonAssertions() + }) + +- describe('full run STREAM_HIGH_WATER_MARK=1MB', function () { ++ describe('full run STREAM_HIGH_WATER_MARK=64kiB', function () { + let output + before('prepare environment', prepareEnvironment) + before('run script', async function () { +- output = await runScript(['--skip-hashed-files'], { +- STREAM_HIGH_WATER_MARK: (1024 * 1024).toString(), +- }) ++ output = await runScript([ ++ '--skip-hashed-files', ++ `--stream-high-water-mark=${64 * 1024}`, ++ ]) + }) + it('should print stats', function () { + expect(output.stats).deep.equal(STATS_ALL) +@@ -1202,7 +1193,7 @@ Sampled stats for deleted projects: + let output + before('prepare environment', prepareEnvironment) + before('run script', async function () { +- output = await runScript([], {}) ++ output = await runScript([]) + }) + it('should print stats', function () { + expect(output.stats).deep.equal( +@@ -1231,9 +1222,7 @@ Sampled stats for deleted projects: + }) + let output + before('run script', async function () { +- output = await runScript(['--skip-hashed-files'], { +- CONCURRENCY: '1', +- }) ++ output = await runScript(['--skip-hashed-files', '--concurrency=1']) + }) + + it('should print stats', function () { +@@ -1252,20 +1241,18 @@ Sampled stats for deleted projects: + let outputPart0, outputPart1 + before('prepare environment', prepareEnvironment) + before('run script on part 0', async function () { +- outputPart0 = await runScript( +- ['--skip-hashed-files', `--BATCH_RANGE_END=${edge}`], +- { +- CONCURRENCY: '1', +- } +- ) ++ outputPart0 = await runScript([ ++ '--skip-hashed-files', ++ `--BATCH_RANGE_END=${edge}`, ++ '--concurrency=1', ++ ]) + }) + before('run script on part 1', async function () { +- outputPart1 = await runScript( +- ['--skip-hashed-files', `--BATCH_RANGE_START=${edge}`], +- { +- CONCURRENCY: '1', +- } +- ) ++ outputPart1 = await runScript([ ++ '--skip-hashed-files', ++ `--BATCH_RANGE_START=${edge}`, ++ '--concurrency=1', ++ ]) + }) + + it('should print stats for part 0', function () { + diff --git a/server-ce/hotfix/5.5.3/pr_27476.patch-stage-2 b/server-ce/hotfix/5.5.3/pr_27476.patch-stage-2 new file mode 100644 index 0000000000..d0f82545d6 --- /dev/null +++ b/server-ce/hotfix/5.5.3/pr_27476.patch-stage-2 @@ -0,0 +1,165 @@ +diff --git a/services/web/app/src/Features/Collaborators/OwnershipTransferHandler.js b/services/web/app/src/Features/Collaborators/OwnershipTransferHandler.js +index e22818ebb880..81ec5ccb0aa5 100644 +--- a/services/web/app/src/Features/Collaborators/OwnershipTransferHandler.js ++++ b/services/web/app/src/Features/Collaborators/OwnershipTransferHandler.js +@@ -9,9 +9,75 @@ const PrivilegeLevels = require('../Authorization/PrivilegeLevels') + const TpdsProjectFlusher = require('../ThirdPartyDataStore/TpdsProjectFlusher') + const ProjectAuditLogHandler = require('../Project/ProjectAuditLogHandler') + const AnalyticsManager = require('../Analytics/AnalyticsManager') ++const OError = require('@overleaf/o-error') ++const TagsHandler = require('../Tags/TagsHandler') ++const { promiseMapWithLimit } = require('@overleaf/promise-utils') + + module.exports = { +- promises: { transferOwnership }, ++ promises: { ++ transferOwnership, ++ transferAllProjectsToUser, ++ }, ++} ++ ++const TAG_COLOR_BLUE = '#434AF0' ++ ++/** ++ * @param {string} fromUserId ++ * @param {string} toUserId ++ * @param {string} ipAddress ++ * @return {Promise<{projectCount: number, newTagName: string}>} ++ */ ++async function transferAllProjectsToUser({ fromUserId, toUserId, ipAddress }) { ++ // - Verify that both users exist ++ const fromUser = await UserGetter.promises.getUser(fromUserId, { ++ _id: 1, ++ email: 1, ++ }) ++ const toUser = await UserGetter.promises.getUser(toUserId, { _id: 1 }) ++ if (!fromUser) throw new OError('missing source user', { fromUserId }) ++ if (!toUser) throw new OError('missing destination user', { toUserId }) ++ if (fromUser._id.equals(toUser._id)) ++ throw new OError('rejecting transfer between identical users', { ++ fromUserId, ++ toUserId, ++ }) ++ logger.debug( ++ { fromUserId, toUserId }, ++ 'started bulk transfer of all projects from one user to another' ++ ) ++ // - Get all owned projects for fromUserId ++ const projects = await Project.find({ owner_ref: fromUserId }, { _id: 1 }) ++ ++ // - Create new tag on toUserId ++ const newTag = await TagsHandler.promises.createTag( ++ toUserId, ++ `transferred-from-${fromUser.email}`, ++ TAG_COLOR_BLUE, ++ { truncate: true } ++ ) ++ ++ // - Add tag to projects (can happen before ownership is transferred) ++ await TagsHandler.promises.addProjectsToTag( ++ toUserId, ++ newTag._id, ++ projects.map(p => p._id) ++ ) ++ ++ // - Transfer all projects ++ await promiseMapWithLimit(5, projects, async project => { ++ await transferOwnership(project._id, toUserId, { ++ allowTransferToNonCollaborators: true, ++ skipEmails: true, ++ ipAddress, ++ }) ++ }) ++ ++ logger.debug( ++ { fromUserId, toUserId }, ++ 'finished bulk transfer of all projects from one user to another' ++ ) ++ return { projectCount: projects.length, newTagName: newTag.name } + } + + async function transferOwnership(projectId, newOwnerId, options = {}) { +@@ -74,8 +140,8 @@ async function transferOwnership(projectId, newOwnerId, options = {}) { + await TpdsProjectFlusher.promises.flushProjectToTpds(projectId) + + // Send confirmation emails +- const previousOwner = await UserGetter.promises.getUser(previousOwnerId) + if (!skipEmails) { ++ const previousOwner = await UserGetter.promises.getUser(previousOwnerId) + await _sendEmails(project, previousOwner, newOwner) + } + } +diff --git a/services/web/modules/server-ce-scripts/scripts/transfer-all-projects-to-user.mjs b/services/web/modules/server-ce-scripts/scripts/transfer-all-projects-to-user.mjs +new file mode 100644 +index 000000000000..6ff1215de53b +--- /dev/null ++++ b/services/web/modules/server-ce-scripts/scripts/transfer-all-projects-to-user.mjs +@@ -0,0 +1,46 @@ ++import { ObjectId } from 'mongodb' ++import minimist from 'minimist' ++import OwnershipTransferHandler from '../../../app/src/Features/Collaborators/OwnershipTransferHandler.js' ++import UserGetter from '../../../app/src/Features/User/UserGetter.js' ++import EmailHelper from '../../../app/src/Features/Helpers/EmailHelper.js' ++ ++const args = minimist(process.argv.slice(2), { ++ string: ['from-user', 'to-user'], ++}) ++ ++/** ++ * @param {string} flag ++ * @return {Promise} ++ */ ++async function resolveUser(flag) { ++ const raw = args[flag] ++ if (!raw) throw new Error(`missing parameter --${flag}`) ++ if (ObjectId.isValid(raw)) return raw ++ const email = EmailHelper.parseEmail(raw) ++ if (!email) throw new Error(`invalid email --${flag}=${raw}`) ++ const user = await UserGetter.promises.getUser({ email: email }, { _id: 1 }) ++ if (!user) ++ throw new Error(`user with email --${flag}=${email} does not exist`) ++ return user._id.toString() ++} ++ ++async function main() { ++ const fromUserId = await resolveUser('from-user') ++ const toUserId = await resolveUser('to-user') ++ await OwnershipTransferHandler.promises.transferAllProjectsToUser({ ++ fromUserId, ++ toUserId, ++ ipAddress: '0.0.0.0', ++ }) ++} ++ ++main() ++ .then(() => { ++ console.error('Done.') ++ process.exit(0) ++ }) ++ .catch(err => { ++ console.error('---') ++ console.error(err) ++ process.exit(1) ++ }) + + +diff --git a/services/web/modules/server-ce-scripts/scripts/transfer-all-projects-to-user.mjs b/services/web/modules/server-ce-scripts/scripts/transfer-all-projects-to-user.mjs +index 6ff1215de53b..8c5951334403 100644 +--- a/services/web/modules/server-ce-scripts/scripts/transfer-all-projects-to-user.mjs ++++ b/services/web/modules/server-ce-scripts/scripts/transfer-all-projects-to-user.mjs +@@ -1,4 +1,4 @@ +-import { ObjectId } from 'mongodb' ++import { ObjectId } from '../../../app/src/infrastructure/mongodb.js' + import minimist from 'minimist' + import OwnershipTransferHandler from '../../../app/src/Features/Collaborators/OwnershipTransferHandler.js' + import UserGetter from '../../../app/src/Features/User/UserGetter.js' +@@ -18,7 +18,7 @@ async function resolveUser(flag) { + if (ObjectId.isValid(raw)) return raw + const email = EmailHelper.parseEmail(raw) + if (!email) throw new Error(`invalid email --${flag}=${raw}`) +- const user = await UserGetter.promises.getUser({ email: email }, { _id: 1 }) ++ const user = await UserGetter.promises.getUser({ email }, { _id: 1 }) + if (!user) + throw new Error(`user with email --${flag}=${email} does not exist`) + return user._id.toString() + diff --git a/server-ce/hotfix/5.5.3/sec-npm.patch b/server-ce/hotfix/5.5.3/sec-npm.patch new file mode 100644 index 0000000000..a525718866 --- /dev/null +++ b/server-ce/hotfix/5.5.3/sec-npm.patch @@ -0,0 +1,2509 @@ +diff -u -x node_modules -r a/libraries/logger/package.json b/libraries/logger/package.json +--- a/libraries/logger/package.json 2025-05-23 09:07:47.000000000 +0100 ++++ b/libraries/logger/package.json 2025-07-28 12:35:14.000000000 +0100 +@@ -20,7 +20,6 @@ + "types:check": "tsc --noEmit" + }, + "dependencies": { +- "@google-cloud/logging-bunyan": "^5.1.0", + "@overleaf/fetch-utils": "*", + "@overleaf/o-error": "*", + "bunyan": "^1.8.14" +diff -u -x node_modules -r a/libraries/metrics/package.json b/libraries/metrics/package.json +--- a/libraries/metrics/package.json 2025-05-23 09:07:47.000000000 +0100 ++++ b/libraries/metrics/package.json 2025-07-28 12:35:06.000000000 +0100 +@@ -8,8 +8,6 @@ + }, + "main": "index.js", + "dependencies": { +- "@google-cloud/opentelemetry-cloud-trace-exporter": "^2.1.0", +- "@google-cloud/profiler": "^6.0.3", + "@opentelemetry/api": "^1.4.1", + "@opentelemetry/auto-instrumentations-node": "^0.39.1", + "@opentelemetry/exporter-trace-otlp-http": "^0.41.2", +diff -u -x node_modules -r a/package.json b/package.json +--- a/package.json 2025-07-28 12:32:06.000000000 +0100 ++++ b/package.json 2025-07-28 12:34:41.000000000 +0100 +@@ -37,12 +37,19 @@ + }, + "swagger-tools": { + "body-parser": "1.20.3", +- "multer": "2.0.1", ++ "multer": "2.0.2", + "path-to-regexp": "3.3.0", + "qs": "6.13.0" + }, + "request@2.88.2": { +- "tough-cookie": "5.1.2" ++ "tough-cookie": "5.1.2", ++ "form-data": "2.5.5" ++ }, ++ "superagent@7.1.6": { ++ "form-data": "4.0.4" ++ }, ++ "superagent@3.8.3": { ++ "form-data": "2.5.5" + } + }, + "scripts": { +diff -u -x node_modules -r a/package-lock.json b/package-lock.json +--- a/package-lock.json 2025-07-28 12:32:06.000000000 +0100 ++++ b/package-lock.json 2025-07-28 12:36:33.000000000 +0100 +@@ -127,7 +127,6 @@ + "version": "3.1.1", + "license": "AGPL-3.0-only", + "dependencies": { +- "@google-cloud/logging-bunyan": "^5.1.0", + "@overleaf/fetch-utils": "*", + "@overleaf/o-error": "*", + "bunyan": "^1.8.14" +@@ -148,8 +147,6 @@ + "name": "@overleaf/metrics", + "version": "4.2.0", + "dependencies": { +- "@google-cloud/opentelemetry-cloud-trace-exporter": "^2.1.0", +- "@google-cloud/profiler": "^6.0.3", + "@opentelemetry/api": "^1.4.1", + "@opentelemetry/auto-instrumentations-node": "^0.39.1", + "@opentelemetry/exporter-trace-otlp-http": "^0.41.2", +@@ -3215,26 +3212,6 @@ + "w3c-keyname": "^2.2.4" + } + }, +- "node_modules/@contentful/rich-text-html-renderer": { +- "version": "16.0.2", +- "resolved": "https://registry.npmjs.org/@contentful/rich-text-html-renderer/-/rich-text-html-renderer-16.0.2.tgz", +- "integrity": "sha512-0flmxVixlNk5PMiHXAlABUJ2uURsWxOjbC6ZHhqpEVHU03kHMoIKfDdo6CRZc0S0rMWMO3c14Ei91E97T06T8w==", +- "dependencies": { +- "@contentful/rich-text-types": "^16.0.2", +- "escape-html": "^1.0.3" +- }, +- "engines": { +- "node": ">=6.0.0" +- } +- }, +- "node_modules/@contentful/rich-text-types": { +- "version": "16.0.2", +- "resolved": "https://registry.npmjs.org/@contentful/rich-text-types/-/rich-text-types-16.0.2.tgz", +- "integrity": "sha512-ovbmCKQjlyGek4NuABoqDesC3FBV3e5jPMMdtT2mpOy9ia31MKO0NSFMRGZu7Q+veZzmDMja8S1i/XeFCUT9Pw==", +- "engines": { +- "node": ">=6.0.0" +- } +- }, + "node_modules/@csstools/css-parser-algorithms": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-2.6.1.tgz", +@@ -3954,595 +3931,6 @@ + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, +- "node_modules/@google-cloud/logging": { +- "version": "11.1.0", +- "resolved": "https://registry.npmjs.org/@google-cloud/logging/-/logging-11.1.0.tgz", +- "integrity": "sha512-S3Zsd+HZxIdZgDZByJ+2GaSQ8rA5OLfdZoZ9Ys1iSZ4HRIhO9ZxlXbmGZgGK9JJ2GaXp7Rux4K4LpkqoYPKnEg==", +- "dependencies": { +- "@google-cloud/common": "^5.0.0", +- "@google-cloud/paginator": "^5.0.0", +- "@google-cloud/projectify": "^4.0.0", +- "@google-cloud/promisify": "^4.0.0", +- "arrify": "^2.0.1", +- "dot-prop": "^6.0.0", +- "eventid": "^2.0.0", +- "extend": "^3.0.2", +- "gcp-metadata": "^6.0.0", +- "google-auth-library": "^9.0.0", +- "google-gax": "^4.0.3", +- "on-finished": "^2.3.0", +- "pumpify": "^2.0.1", +- "stream-events": "^1.0.5", +- "uuid": "^9.0.0" +- }, +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/@google-cloud/logging-bunyan": { +- "version": "5.1.0", +- "resolved": "https://registry.npmjs.org/@google-cloud/logging-bunyan/-/logging-bunyan-5.1.0.tgz", +- "integrity": "sha512-D2Rg5nb+onjWre4eEowWyNmVF1RN7WThWdu1cCOcTMVOoVEGJphMxrBo9VQKQmkqdlAUG4NaM6i2sqieISQDsg==", +- "dependencies": { +- "@google-cloud/logging": "^11.0.0", +- "google-auth-library": "^9.0.0" +- }, +- "engines": { +- "node": ">=14.0.0" +- }, +- "peerDependencies": { +- "bunyan": "*" +- } +- }, +- "node_modules/@google-cloud/logging-bunyan/node_modules/agent-base": { +- "version": "7.1.0", +- "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", +- "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", +- "dependencies": { +- "debug": "^4.3.4" +- }, +- "engines": { +- "node": ">= 14" +- } +- }, +- "node_modules/@google-cloud/logging-bunyan/node_modules/debug": { +- "version": "4.3.4", +- "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", +- "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", +- "dependencies": { +- "ms": "2.1.2" +- }, +- "engines": { +- "node": ">=6.0" +- }, +- "peerDependenciesMeta": { +- "supports-color": { +- "optional": true +- } +- } +- }, +- "node_modules/@google-cloud/logging-bunyan/node_modules/gaxios": { +- "version": "6.1.0", +- "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.1.0.tgz", +- "integrity": "sha512-EIHuesZxNyIkUGcTQKQPMICyOpDD/bi+LJIJx+NLsSGmnS7N+xCLRX5bi4e9yAu9AlSZdVq+qlyWWVuTh/483w==", +- "dependencies": { +- "extend": "^3.0.2", +- "https-proxy-agent": "^7.0.1", +- "is-stream": "^2.0.0", +- "node-fetch": "^2.6.9" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/logging-bunyan/node_modules/gcp-metadata": { +- "version": "6.0.0", +- "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.0.0.tgz", +- "integrity": "sha512-Ozxyi23/1Ar51wjUT2RDklK+3HxqDr8TLBNK8rBBFQ7T85iIGnXnVusauj06QyqCXRFZig8LZC+TUddWbndlpQ==", +- "dependencies": { +- "gaxios": "^6.0.0", +- "json-bigint": "^1.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/logging-bunyan/node_modules/google-auth-library": { +- "version": "9.0.0", +- "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.0.0.tgz", +- "integrity": "sha512-IQGjgQoVUAfOk6khqTVMLvWx26R+yPw9uLyb1MNyMQpdKiKt0Fd9sp4NWoINjyGHR8S3iw12hMTYK7O8J07c6Q==", +- "dependencies": { +- "base64-js": "^1.3.0", +- "ecdsa-sig-formatter": "^1.0.11", +- "gaxios": "^6.0.0", +- "gcp-metadata": "^6.0.0", +- "gtoken": "^7.0.0", +- "jws": "^4.0.0", +- "lru-cache": "^6.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/logging-bunyan/node_modules/gtoken": { +- "version": "7.0.1", +- "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.0.1.tgz", +- "integrity": "sha512-KcFVtoP1CVFtQu0aSk3AyAt2og66PFhZAlkUOuWKwzMLoulHXG5W5wE5xAnHb+yl3/wEFoqGW7/cDGMU8igDZQ==", +- "dependencies": { +- "gaxios": "^6.0.0", +- "jws": "^4.0.0" +- }, +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/@google-cloud/logging-bunyan/node_modules/https-proxy-agent": { +- "version": "7.0.1", +- "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.1.tgz", +- "integrity": "sha512-Eun8zV0kcYS1g19r78osiQLEFIRspRUDd9tIfBCTBPBeMieF/EsJNL8VI3xOIdYRDEkjQnqOYPsZ2DsWsVsFwQ==", +- "dependencies": { +- "agent-base": "^7.0.2", +- "debug": "4" +- }, +- "engines": { +- "node": ">= 14" +- } +- }, +- "node_modules/@google-cloud/logging-bunyan/node_modules/lru-cache": { +- "version": "6.0.0", +- "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", +- "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", +- "dependencies": { +- "yallist": "^4.0.0" +- }, +- "engines": { +- "node": ">=10" +- } +- }, +- "node_modules/@google-cloud/logging-bunyan/node_modules/ms": { +- "version": "2.1.2", +- "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", +- "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" +- }, +- "node_modules/@google-cloud/logging-bunyan/node_modules/yallist": { +- "version": "4.0.0", +- "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", +- "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" +- }, +- "node_modules/@google-cloud/logging-min": { +- "version": "11.2.0", +- "resolved": "https://registry.npmjs.org/@google-cloud/logging-min/-/logging-min-11.2.0.tgz", +- "integrity": "sha512-o1mwzi1+9NMEjwYZJ0X3tK64obf9PzPVBAhzEJv65L0h7jVl3Fw7GswtsMUkdUvZexf96vAvlZZMvXB9jAIW2Q==", +- "license": "Apache-2.0", +- "dependencies": { +- "@google-cloud/common": "^5.0.0", +- "@google-cloud/paginator": "^5.0.0", +- "@google-cloud/projectify": "^4.0.0", +- "@google-cloud/promisify": "^4.0.0", +- "@opentelemetry/api": "^1.7.0", +- "arrify": "^2.0.1", +- "dot-prop": "^6.0.0", +- "eventid": "^2.0.0", +- "extend": "^3.0.2", +- "gcp-metadata": "^6.0.0", +- "google-auth-library": "^9.0.0", +- "google-gax": "^4.0.3", +- "on-finished": "^2.3.0", +- "pumpify": "^2.0.1", +- "stream-events": "^1.0.5", +- "uuid": "^9.0.0" +- }, +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/@google-cloud/logging-min/node_modules/@google-cloud/common": { +- "version": "5.0.2", +- "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-5.0.2.tgz", +- "integrity": "sha512-V7bmBKYQyu0eVG2BFejuUjlBt+zrya6vtsKdY+JxMM/dNntPF41vZ9+LhOshEUH01zOHEqBSvI7Dad7ZS6aUeA==", +- "license": "Apache-2.0", +- "dependencies": { +- "@google-cloud/projectify": "^4.0.0", +- "@google-cloud/promisify": "^4.0.0", +- "arrify": "^2.0.1", +- "duplexify": "^4.1.1", +- "extend": "^3.0.2", +- "google-auth-library": "^9.0.0", +- "html-entities": "^2.5.2", +- "retry-request": "^7.0.0", +- "teeny-request": "^9.0.0" +- }, +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/@google-cloud/logging-min/node_modules/@google-cloud/paginator": { +- "version": "5.0.2", +- "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.2.tgz", +- "integrity": "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==", +- "license": "Apache-2.0", +- "dependencies": { +- "arrify": "^2.0.0", +- "extend": "^3.0.2" +- }, +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/@google-cloud/logging-min/node_modules/@google-cloud/projectify": { +- "version": "4.0.0", +- "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", +- "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", +- "license": "Apache-2.0", +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/@google-cloud/logging-min/node_modules/@google-cloud/promisify": { +- "version": "4.1.0", +- "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.1.0.tgz", +- "integrity": "sha512-G/FQx5cE/+DqBbOpA5jKsegGwdPniU6PuIEMt+qxWgFxvxuFOzVmp6zYchtYuwAWV5/8Dgs0yAmjvNZv3uXLQg==", +- "license": "Apache-2.0", +- "engines": { +- "node": ">=18" +- } +- }, +- "node_modules/@google-cloud/logging-min/node_modules/@opentelemetry/api": { +- "version": "1.9.0", +- "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", +- "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", +- "license": "Apache-2.0", +- "engines": { +- "node": ">=8.0.0" +- } +- }, +- "node_modules/@google-cloud/logging-min/node_modules/agent-base": { +- "version": "7.1.3", +- "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", +- "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", +- "license": "MIT", +- "engines": { +- "node": ">= 14" +- } +- }, +- "node_modules/@google-cloud/logging-min/node_modules/gaxios": { +- "version": "6.7.1", +- "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", +- "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", +- "license": "Apache-2.0", +- "dependencies": { +- "extend": "^3.0.2", +- "https-proxy-agent": "^7.0.1", +- "is-stream": "^2.0.0", +- "node-fetch": "^2.6.9", +- "uuid": "^9.0.1" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/logging-min/node_modules/gaxios/node_modules/https-proxy-agent": { +- "version": "7.0.6", +- "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", +- "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", +- "license": "MIT", +- "dependencies": { +- "agent-base": "^7.1.2", +- "debug": "4" +- }, +- "engines": { +- "node": ">= 14" +- } +- }, +- "node_modules/@google-cloud/logging-min/node_modules/gcp-metadata": { +- "version": "6.1.1", +- "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", +- "integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==", +- "license": "Apache-2.0", +- "dependencies": { +- "gaxios": "^6.1.1", +- "google-logging-utils": "^0.0.2", +- "json-bigint": "^1.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/logging-min/node_modules/google-auth-library": { +- "version": "9.15.1", +- "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz", +- "integrity": "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==", +- "license": "Apache-2.0", +- "dependencies": { +- "base64-js": "^1.3.0", +- "ecdsa-sig-formatter": "^1.0.11", +- "gaxios": "^6.1.1", +- "gcp-metadata": "^6.1.0", +- "gtoken": "^7.0.0", +- "jws": "^4.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/logging-min/node_modules/gtoken": { +- "version": "7.1.0", +- "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", +- "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", +- "license": "MIT", +- "dependencies": { +- "gaxios": "^6.0.0", +- "jws": "^4.0.0" +- }, +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/@google-cloud/logging-min/node_modules/retry-request": { +- "version": "7.0.2", +- "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", +- "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", +- "license": "MIT", +- "dependencies": { +- "@types/request": "^2.48.8", +- "extend": "^3.0.2", +- "teeny-request": "^9.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/logging-min/node_modules/teeny-request": { +- "version": "9.0.0", +- "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", +- "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", +- "license": "Apache-2.0", +- "dependencies": { +- "http-proxy-agent": "^5.0.0", +- "https-proxy-agent": "^5.0.0", +- "node-fetch": "^2.6.9", +- "stream-events": "^1.0.5", +- "uuid": "^9.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/logging-min/node_modules/uuid": { +- "version": "9.0.1", +- "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", +- "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", +- "funding": [ +- "https://github.com/sponsors/broofa", +- "https://github.com/sponsors/ctavan" +- ], +- "license": "MIT", +- "bin": { +- "uuid": "dist/bin/uuid" +- } +- }, +- "node_modules/@google-cloud/logging/node_modules/@google-cloud/common": { +- "version": "5.0.2", +- "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-5.0.2.tgz", +- "integrity": "sha512-V7bmBKYQyu0eVG2BFejuUjlBt+zrya6vtsKdY+JxMM/dNntPF41vZ9+LhOshEUH01zOHEqBSvI7Dad7ZS6aUeA==", +- "dependencies": { +- "@google-cloud/projectify": "^4.0.0", +- "@google-cloud/promisify": "^4.0.0", +- "arrify": "^2.0.1", +- "duplexify": "^4.1.1", +- "extend": "^3.0.2", +- "google-auth-library": "^9.0.0", +- "html-entities": "^2.5.2", +- "retry-request": "^7.0.0", +- "teeny-request": "^9.0.0" +- }, +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/@google-cloud/logging/node_modules/@google-cloud/paginator": { +- "version": "5.0.2", +- "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.2.tgz", +- "integrity": "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==", +- "dependencies": { +- "arrify": "^2.0.0", +- "extend": "^3.0.2" +- }, +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/@google-cloud/logging/node_modules/@google-cloud/projectify": { +- "version": "4.0.0", +- "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", +- "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/@google-cloud/logging/node_modules/@google-cloud/promisify": { +- "version": "4.0.0", +- "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", +- "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/logging/node_modules/agent-base": { +- "version": "7.1.1", +- "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", +- "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", +- "dependencies": { +- "debug": "^4.3.4" +- }, +- "engines": { +- "node": ">= 14" +- } +- }, +- "node_modules/@google-cloud/logging/node_modules/debug": { +- "version": "4.3.5", +- "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", +- "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", +- "dependencies": { +- "ms": "2.1.2" +- }, +- "engines": { +- "node": ">=6.0" +- }, +- "peerDependenciesMeta": { +- "supports-color": { +- "optional": true +- } +- } +- }, +- "node_modules/@google-cloud/logging/node_modules/gaxios": { +- "version": "6.6.0", +- "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.6.0.tgz", +- "integrity": "sha512-bpOZVQV5gthH/jVCSuYuokRo2bTKOcuBiVWpjmTn6C5Agl5zclGfTljuGsQZxwwDBkli+YhZhP4TdlqTnhOezQ==", +- "dependencies": { +- "extend": "^3.0.2", +- "https-proxy-agent": "^7.0.1", +- "is-stream": "^2.0.0", +- "node-fetch": "^2.6.9", +- "uuid": "^9.0.1" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/logging/node_modules/gaxios/node_modules/https-proxy-agent": { +- "version": "7.0.4", +- "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", +- "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", +- "dependencies": { +- "agent-base": "^7.0.2", +- "debug": "4" +- }, +- "engines": { +- "node": ">= 14" +- } +- }, +- "node_modules/@google-cloud/logging/node_modules/gcp-metadata": { +- "version": "6.1.0", +- "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", +- "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", +- "dependencies": { +- "gaxios": "^6.0.0", +- "json-bigint": "^1.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/logging/node_modules/google-auth-library": { +- "version": "9.10.0", +- "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.10.0.tgz", +- "integrity": "sha512-ol+oSa5NbcGdDqA+gZ3G3mev59OHBZksBTxY/tYwjtcp1H/scAFwJfSQU9/1RALoyZ7FslNbke8j4i3ipwlyuQ==", +- "dependencies": { +- "base64-js": "^1.3.0", +- "ecdsa-sig-formatter": "^1.0.11", +- "gaxios": "^6.1.1", +- "gcp-metadata": "^6.1.0", +- "gtoken": "^7.0.0", +- "jws": "^4.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/logging/node_modules/gtoken": { +- "version": "7.1.0", +- "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", +- "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", +- "dependencies": { +- "gaxios": "^6.0.0", +- "jws": "^4.0.0" +- }, +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/@google-cloud/logging/node_modules/ms": { +- "version": "2.1.2", +- "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", +- "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" +- }, +- "node_modules/@google-cloud/logging/node_modules/retry-request": { +- "version": "7.0.2", +- "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", +- "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", +- "dependencies": { +- "@types/request": "^2.48.8", +- "extend": "^3.0.2", +- "teeny-request": "^9.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/logging/node_modules/teeny-request": { +- "version": "9.0.0", +- "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", +- "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", +- "dependencies": { +- "http-proxy-agent": "^5.0.0", +- "https-proxy-agent": "^5.0.0", +- "node-fetch": "^2.6.9", +- "stream-events": "^1.0.5", +- "uuid": "^9.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/logging/node_modules/uuid": { +- "version": "9.0.1", +- "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", +- "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", +- "funding": [ +- "https://github.com/sponsors/broofa", +- "https://github.com/sponsors/ctavan" +- ], +- "bin": { +- "uuid": "dist/bin/uuid" +- } +- }, +- "node_modules/@google-cloud/opentelemetry-cloud-trace-exporter": { +- "version": "2.1.0", +- "resolved": "https://registry.npmjs.org/@google-cloud/opentelemetry-cloud-trace-exporter/-/opentelemetry-cloud-trace-exporter-2.1.0.tgz", +- "integrity": "sha512-6IPFnWG4edDgNfgLxXJjTjNYGAW8ZQ7Oz7eGZJMgQsIiEALNIAk4e/MgccglL3yh5ReONY3YePcGRWQKPbxmUg==", +- "dependencies": { +- "@google-cloud/opentelemetry-resource-util": "^2.1.0", +- "@grpc/grpc-js": "^1.1.8", +- "@grpc/proto-loader": "^0.7.0", +- "google-auth-library": "^7.0.0", +- "google-proto-files": "^3.0.0" +- }, +- "engines": { +- "node": ">=14" +- }, +- "peerDependencies": { +- "@opentelemetry/api": "^1.0.0", +- "@opentelemetry/core": "^1.0.0", +- "@opentelemetry/resources": "^1.0.0", +- "@opentelemetry/sdk-trace-base": "^1.0.0" +- } +- }, +- "node_modules/@google-cloud/opentelemetry-resource-util": { +- "version": "2.1.0", +- "resolved": "https://registry.npmjs.org/@google-cloud/opentelemetry-resource-util/-/opentelemetry-resource-util-2.1.0.tgz", +- "integrity": "sha512-/Qqnm6f10e89Txt39qpIhD+LCOF80artYOVwNF1ZAzgJFxBldEniNkf19SR+q9LAp75ZZWKyhRlumM1V7fT8gw==", +- "dependencies": { +- "gcp-metadata": "^5.0.1" +- }, +- "engines": { +- "node": ">=14" +- }, +- "peerDependencies": { +- "@opentelemetry/resources": "^1.0.0", +- "@opentelemetry/semantic-conventions": "^1.0.0" +- } +- }, + "node_modules/@google-cloud/paginator": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-3.0.7.tgz", +@@ -4555,242 +3943,6 @@ + "node": ">=10" + } + }, +- "node_modules/@google-cloud/profiler": { +- "version": "6.0.3", +- "resolved": "https://registry.npmjs.org/@google-cloud/profiler/-/profiler-6.0.3.tgz", +- "integrity": "sha512-Ey8li6Vc2CbfEzOTSZaqKolxPMGacxVUQuhChNT0Wi55a3nfImMiiuDgqYw1In/a9Q3Z62O7jUg2L8f1XwMN7Q==", +- "license": "Apache-2.0", +- "dependencies": { +- "@google-cloud/common": "^5.0.0", +- "@google-cloud/logging-min": "^11.0.0", +- "@google-cloud/promisify": "~4.0.0", +- "@types/console-log-level": "^1.4.0", +- "@types/semver": "^7.0.0", +- "console-log-level": "^1.4.0", +- "delay": "^5.0.0", +- "extend": "^3.0.2", +- "gcp-metadata": "^6.0.0", +- "ms": "^2.1.3", +- "pprof": "4.0.0", +- "pretty-ms": "^7.0.0", +- "protobufjs": "~7.4.0", +- "semver": "^7.0.0", +- "teeny-request": "^9.0.0" +- }, +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/@google-cloud/profiler/node_modules/@google-cloud/common": { +- "version": "5.0.2", +- "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-5.0.2.tgz", +- "integrity": "sha512-V7bmBKYQyu0eVG2BFejuUjlBt+zrya6vtsKdY+JxMM/dNntPF41vZ9+LhOshEUH01zOHEqBSvI7Dad7ZS6aUeA==", +- "license": "Apache-2.0", +- "dependencies": { +- "@google-cloud/projectify": "^4.0.0", +- "@google-cloud/promisify": "^4.0.0", +- "arrify": "^2.0.1", +- "duplexify": "^4.1.1", +- "extend": "^3.0.2", +- "google-auth-library": "^9.0.0", +- "html-entities": "^2.5.2", +- "retry-request": "^7.0.0", +- "teeny-request": "^9.0.0" +- }, +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/@google-cloud/profiler/node_modules/@google-cloud/projectify": { +- "version": "4.0.0", +- "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", +- "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", +- "license": "Apache-2.0", +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/@google-cloud/profiler/node_modules/@google-cloud/promisify": { +- "version": "4.0.0", +- "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", +- "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", +- "license": "Apache-2.0", +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/profiler/node_modules/agent-base": { +- "version": "7.1.3", +- "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", +- "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", +- "license": "MIT", +- "engines": { +- "node": ">= 14" +- } +- }, +- "node_modules/@google-cloud/profiler/node_modules/gaxios": { +- "version": "6.7.1", +- "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", +- "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", +- "license": "Apache-2.0", +- "dependencies": { +- "extend": "^3.0.2", +- "https-proxy-agent": "^7.0.1", +- "is-stream": "^2.0.0", +- "node-fetch": "^2.6.9", +- "uuid": "^9.0.1" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/profiler/node_modules/gaxios/node_modules/https-proxy-agent": { +- "version": "7.0.6", +- "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", +- "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", +- "license": "MIT", +- "dependencies": { +- "agent-base": "^7.1.2", +- "debug": "4" +- }, +- "engines": { +- "node": ">= 14" +- } +- }, +- "node_modules/@google-cloud/profiler/node_modules/gcp-metadata": { +- "version": "6.1.1", +- "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", +- "integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==", +- "license": "Apache-2.0", +- "dependencies": { +- "gaxios": "^6.1.1", +- "google-logging-utils": "^0.0.2", +- "json-bigint": "^1.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/profiler/node_modules/google-auth-library": { +- "version": "9.15.1", +- "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz", +- "integrity": "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==", +- "license": "Apache-2.0", +- "dependencies": { +- "base64-js": "^1.3.0", +- "ecdsa-sig-formatter": "^1.0.11", +- "gaxios": "^6.1.1", +- "gcp-metadata": "^6.1.0", +- "gtoken": "^7.0.0", +- "jws": "^4.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/profiler/node_modules/gtoken": { +- "version": "7.1.0", +- "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", +- "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", +- "license": "MIT", +- "dependencies": { +- "gaxios": "^6.0.0", +- "jws": "^4.0.0" +- }, +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/@google-cloud/profiler/node_modules/protobufjs": { +- "version": "7.4.0", +- "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", +- "integrity": "sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==", +- "hasInstallScript": true, +- "license": "BSD-3-Clause", +- "dependencies": { +- "@protobufjs/aspromise": "^1.1.2", +- "@protobufjs/base64": "^1.1.2", +- "@protobufjs/codegen": "^2.0.4", +- "@protobufjs/eventemitter": "^1.1.0", +- "@protobufjs/fetch": "^1.1.0", +- "@protobufjs/float": "^1.0.2", +- "@protobufjs/inquire": "^1.1.0", +- "@protobufjs/path": "^1.1.2", +- "@protobufjs/pool": "^1.1.0", +- "@protobufjs/utf8": "^1.1.0", +- "@types/node": ">=13.7.0", +- "long": "^5.0.0" +- }, +- "engines": { +- "node": ">=12.0.0" +- } +- }, +- "node_modules/@google-cloud/profiler/node_modules/retry-request": { +- "version": "7.0.2", +- "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", +- "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", +- "license": "MIT", +- "dependencies": { +- "@types/request": "^2.48.8", +- "extend": "^3.0.2", +- "teeny-request": "^9.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/profiler/node_modules/semver": { +- "version": "7.7.1", +- "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", +- "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", +- "license": "ISC", +- "bin": { +- "semver": "bin/semver.js" +- }, +- "engines": { +- "node": ">=10" +- } +- }, +- "node_modules/@google-cloud/profiler/node_modules/teeny-request": { +- "version": "9.0.0", +- "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", +- "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", +- "license": "Apache-2.0", +- "dependencies": { +- "http-proxy-agent": "^5.0.0", +- "https-proxy-agent": "^5.0.0", +- "node-fetch": "^2.6.9", +- "stream-events": "^1.0.5", +- "uuid": "^9.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/@google-cloud/profiler/node_modules/uuid": { +- "version": "9.0.1", +- "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", +- "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", +- "funding": [ +- "https://github.com/sponsors/broofa", +- "https://github.com/sponsors/ctavan" +- ], +- "license": "MIT", +- "bin": { +- "uuid": "dist/bin/uuid" +- } +- }, +- "node_modules/@google-cloud/secret-manager": { +- "version": "5.6.0", +- "resolved": "https://registry.npmjs.org/@google-cloud/secret-manager/-/secret-manager-5.6.0.tgz", +- "integrity": "sha512-0daW/OXQEVc6VQKPyJTQNyD+563I/TYQ7GCQJx4dq3lB666R9FUPvqHx9b/o/qQtZ5pfuoCbGZl3krpxgTSW8Q==", +- "dependencies": { +- "google-gax": "^4.0.3" +- }, +- "engines": { +- "node": ">=14.0.0" +- } +- }, + "node_modules/@google-cloud/storage": { + "version": "6.10.1", + "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-6.10.1.tgz", +@@ -5963,40 +5115,44 @@ + } + }, + "node_modules/@node-saml/node-saml": { +- "version": "4.0.5", +- "resolved": "https://registry.npmjs.org/@node-saml/node-saml/-/node-saml-4.0.5.tgz", +- "integrity": "sha512-J5DglElbY1tjOuaR1NPtjOXkXY5bpUhDoKVoeucYN98A3w4fwgjIOPqIGcb6cQsqFq2zZ6vTCeKn5C/hvefSaw==", +- "dependencies": { +- "@types/debug": "^4.1.7", +- "@types/passport": "^1.0.11", +- "@types/xml-crypto": "^1.4.2", +- "@types/xml-encryption": "^1.2.1", +- "@types/xml2js": "^0.4.11", +- "@xmldom/xmldom": "^0.8.6", +- "debug": "^4.3.4", +- "xml-crypto": "^3.0.1", +- "xml-encryption": "^3.0.2", +- "xml2js": "^0.5.0", +- "xmlbuilder": "^15.1.1" ++ "version": "5.1.0", ++ "resolved": "https://registry.npmjs.org/@node-saml/node-saml/-/node-saml-5.1.0.tgz", ++ "integrity": "sha512-t3cJnZ4aC7HhPZ6MGylGZULvUtBOZ6FzuUndaHGXjmIZHXnLfC/7L8a57O9Q9V7AxJGKAiRM5zu2wNm9EsvQpw==", ++ "license": "MIT", ++ "dependencies": { ++ "@types/debug": "^4.1.12", ++ "@types/qs": "^6.9.18", ++ "@types/xml-encryption": "^1.2.4", ++ "@types/xml2js": "^0.4.14", ++ "@xmldom/is-dom-node": "^1.0.1", ++ "@xmldom/xmldom": "^0.8.10", ++ "debug": "^4.4.0", ++ "xml-crypto": "^6.1.2", ++ "xml-encryption": "^3.1.0", ++ "xml2js": "^0.6.2", ++ "xmlbuilder": "^15.1.1", ++ "xpath": "^0.0.34" + }, + "engines": { +- "node": ">= 14" ++ "node": ">= 18" + } + }, + "node_modules/@node-saml/node-saml/node_modules/@xmldom/xmldom": { + "version": "0.8.10", + "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.10.tgz", + "integrity": "sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==", ++ "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/@node-saml/node-saml/node_modules/debug": { +- "version": "4.3.4", +- "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", +- "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", ++ "version": "4.4.1", ++ "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", ++ "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", ++ "license": "MIT", + "dependencies": { +- "ms": "2.1.2" ++ "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" +@@ -6007,66 +5163,80 @@ + } + } + }, +- "node_modules/@node-saml/node-saml/node_modules/ms": { +- "version": "2.1.2", +- "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", +- "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" +- }, + "node_modules/@node-saml/node-saml/node_modules/xml-encryption": { +- "version": "3.0.2", +- "resolved": "https://registry.npmjs.org/xml-encryption/-/xml-encryption-3.0.2.tgz", +- "integrity": "sha512-VxYXPvsWB01/aqVLd6ZMPWZ+qaj0aIdF+cStrVJMcFj3iymwZeI0ABzB3VqMYv48DkSpRhnrXqTUkR34j+UDyg==", ++ "version": "3.1.0", ++ "resolved": "https://registry.npmjs.org/xml-encryption/-/xml-encryption-3.1.0.tgz", ++ "integrity": "sha512-PV7qnYpoAMXbf1kvQkqMScLeQpjCMixddAKq9PtqVrho8HnYbBOWNfG0kA4R7zxQDo7w9kiYAyzS/ullAyO55Q==", ++ "license": "MIT", + "dependencies": { + "@xmldom/xmldom": "^0.8.5", + "escape-html": "^1.0.3", + "xpath": "0.0.32" +- }, +- "engines": { +- "node": ">=12" + } + }, +- "node_modules/@node-saml/node-saml/node_modules/xml2js": { +- "version": "0.5.0", +- "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", +- "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==", +- "dependencies": { +- "sax": ">=0.6.0", +- "xmlbuilder": "~11.0.0" +- }, +- "engines": { +- "node": ">=4.0.0" +- } +- }, +- "node_modules/@node-saml/node-saml/node_modules/xml2js/node_modules/xmlbuilder": { +- "version": "11.0.1", +- "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", +- "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", ++ "node_modules/@node-saml/node-saml/node_modules/xml-encryption/node_modules/xpath": { ++ "version": "0.0.32", ++ "resolved": "https://registry.npmjs.org/xpath/-/xpath-0.0.32.tgz", ++ "integrity": "sha512-rxMJhSIoiO8vXcWvSifKqhvV96GjiD5wYb8/QHdoRyQvraTpp4IEv944nhGausZZ3u7dhQXteZuZbaqfpB7uYw==", ++ "license": "MIT", + "engines": { +- "node": ">=4.0" ++ "node": ">=0.6.0" + } + }, + "node_modules/@node-saml/node-saml/node_modules/xpath": { +- "version": "0.0.32", +- "resolved": "https://registry.npmjs.org/xpath/-/xpath-0.0.32.tgz", +- "integrity": "sha512-rxMJhSIoiO8vXcWvSifKqhvV96GjiD5wYb8/QHdoRyQvraTpp4IEv944nhGausZZ3u7dhQXteZuZbaqfpB7uYw==", ++ "version": "0.0.34", ++ "resolved": "https://registry.npmjs.org/xpath/-/xpath-0.0.34.tgz", ++ "integrity": "sha512-FxF6+rkr1rNSQrhUNYrAFJpRXNzlDoMxeXN5qI84939ylEv3qqPFKa85Oxr6tDaJKqwW6KKyo2v26TSv3k6LeA==", ++ "license": "MIT", + "engines": { + "node": ">=0.6.0" + } + }, + "node_modules/@node-saml/passport-saml": { +- "version": "4.0.4", +- "resolved": "https://registry.npmjs.org/@node-saml/passport-saml/-/passport-saml-4.0.4.tgz", +- "integrity": "sha512-xFw3gw0yo+K1mzlkW15NeBF7cVpRHN/4vpjmBKzov5YFImCWh/G0LcTZ8krH3yk2/eRPc3Or8LRPudVJBjmYaw==", ++ "version": "5.1.0", ++ "resolved": "https://registry.npmjs.org/@node-saml/passport-saml/-/passport-saml-5.1.0.tgz", ++ "integrity": "sha512-pBm+iFjv9eihcgeJuSUs4c0AuX1QEFdHwP8w1iaWCfDzXdeWZxUBU5HT2bY2S4dvNutcy+A9hYsH7ZLBGtgwDg==", ++ "license": "MIT", + "dependencies": { +- "@node-saml/node-saml": "^4.0.4", +- "@types/express": "^4.17.14", +- "@types/passport": "^1.0.11", +- "@types/passport-strategy": "^0.2.35", +- "passport": "^0.6.0", ++ "@node-saml/node-saml": "^5.1.0", ++ "@types/express": "^4.17.23", ++ "@types/passport": "^1.0.17", ++ "@types/passport-strategy": "^0.2.38", ++ "passport": "^0.7.0", + "passport-strategy": "^1.0.0" + }, + "engines": { +- "node": ">= 14" ++ "node": ">= 18" ++ } ++ }, ++ "node_modules/@node-saml/passport-saml/node_modules/@types/express": { ++ "version": "4.17.23", ++ "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.23.tgz", ++ "integrity": "sha512-Crp6WY9aTYP3qPi2wGDo9iUe/rceX01UMhnF1jmwDcKCFM6cx7YhGP/Mpr3y9AASpfHixIG0E6azCcL5OcDHsQ==", ++ "license": "MIT", ++ "dependencies": { ++ "@types/body-parser": "*", ++ "@types/express-serve-static-core": "^4.17.33", ++ "@types/qs": "*", ++ "@types/serve-static": "*" ++ } ++ }, ++ "node_modules/@node-saml/passport-saml/node_modules/passport": { ++ "version": "0.7.0", ++ "resolved": "https://registry.npmjs.org/passport/-/passport-0.7.0.tgz", ++ "integrity": "sha512-cPLl+qZpSc+ireUvt+IzqbED1cHHkDoVYMo30jbJIdOOjQ1MQYZBPiNvmi8UM6lJuOpTPXJGZQk0DtC4y61MYQ==", ++ "license": "MIT", ++ "dependencies": { ++ "passport-strategy": "1.x.x", ++ "pause": "0.0.1", ++ "utils-merge": "^1.0.1" ++ }, ++ "engines": { ++ "node": ">= 0.4.0" ++ }, ++ "funding": { ++ "type": "github", ++ "url": "https://github.com/sponsors/jaredhanson" + } + }, + "node_modules/@nodelib/fs.scandir": { +@@ -8177,29 +7347,6 @@ + "integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==", + "dev": true + }, +- "node_modules/@slack/types": { +- "version": "2.10.0", +- "resolved": "https://registry.npmjs.org/@slack/types/-/types-2.10.0.tgz", +- "integrity": "sha512-JXY9l49rf7dDgvfMZi0maFyugzGkvq0s5u+kDlD68WaRUhjZNLBDKZcsrycMsVVDFfyOK0R1UKkYGmy9Ph069Q==", +- "engines": { +- "node": ">= 12.13.0", +- "npm": ">= 6.12.0" +- } +- }, +- "node_modules/@slack/webhook": { +- "version": "7.0.2", +- "resolved": "https://registry.npmjs.org/@slack/webhook/-/webhook-7.0.2.tgz", +- "integrity": "sha512-dsrO/ow6a6+xkLm/lZKbUNTsFJlBc679tD+qwlVTztsQkDxPLH6odM7FKALz1IHa+KpLX8HKUIPV13a7y7z29w==", +- "dependencies": { +- "@slack/types": "^2.9.0", +- "@types/node": ">=18.0.0", +- "axios": "^1.6.3" +- }, +- "engines": { +- "node": ">= 18", +- "npm": ">= 8.6.0" +- } +- }, + "node_modules/@smithy/abort-controller": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-1.0.2.tgz", +@@ -10616,11 +9763,6 @@ + "@types/node": "*" + } + }, +- "node_modules/@types/caseless": { +- "version": "0.12.5", +- "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", +- "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==" +- }, + "node_modules/@types/chai": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.0.tgz", +@@ -10659,11 +9801,6 @@ + "@types/node": "*" + } + }, +- "node_modules/@types/console-log-level": { +- "version": "1.4.3", +- "resolved": "https://registry.npmjs.org/@types/console-log-level/-/console-log-level-1.4.3.tgz", +- "integrity": "sha512-B6Mzad6H4RugduMX84ehFVvGM/JRAd9lZQk4a6dztB4+zcIUehIjKrbWH/nHO2+0wwx05rgyqjXBvOjAv0uL6A==" +- }, + "node_modules/@types/content-disposition": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/@types/content-disposition/-/content-disposition-0.5.5.tgz", +@@ -10697,9 +9834,10 @@ + "dev": true + }, + "node_modules/@types/debug": { +- "version": "4.1.7", +- "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.7.tgz", +- "integrity": "sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg==", ++ "version": "4.1.12", ++ "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", ++ "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", ++ "license": "MIT", + "dependencies": { + "@types/ms": "*" + } +@@ -10950,11 +10088,6 @@ + "integrity": "sha512-0d5Wd09ItQWH1qFbEyQ7oTQ3GZrMfth5JkbN3EvTKLXcHLRDSXeLnlvlOn0wvxVIwK5o2M8JzP/OWz7T3NRsbw==", + "dev": true + }, +- "node_modules/@types/long": { +- "version": "4.0.1", +- "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.1.tgz", +- "integrity": "sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==" +- }, + "node_modules/@types/mdx": { + "version": "2.0.13", + "resolved": "https://registry.npmjs.org/@types/mdx/-/mdx-2.0.13.tgz", +@@ -11044,9 +10177,10 @@ + "dev": true + }, + "node_modules/@types/passport": { +- "version": "1.0.15", +- "resolved": "https://registry.npmjs.org/@types/passport/-/passport-1.0.15.tgz", +- "integrity": "sha512-oHOgzPBp5eLI1U/7421qYV/ZySQXMYCBSfRkDe1tQ0YrIbLY/M/76qIXE7Bs7lFyvw1x5QqiNQ9imvh0fQHe9Q==", ++ "version": "1.0.17", ++ "resolved": "https://registry.npmjs.org/@types/passport/-/passport-1.0.17.tgz", ++ "integrity": "sha512-aciLyx+wDwT2t2/kJGJR2AEeBz0nJU4WuRX04Wu9Dqc5lSUtwu0WERPHYsLhF9PtseiAMPBGNUOtFjxZ56prsg==", ++ "license": "MIT", + "dependencies": { + "@types/express": "*" + } +@@ -11055,6 +10189,7 @@ + "version": "0.2.38", + "resolved": "https://registry.npmjs.org/@types/passport-strategy/-/passport-strategy-0.2.38.tgz", + "integrity": "sha512-GC6eMqqojOooq993Tmnmp7AUTbbQSgilyvpCYQjT+H6JfG/g6RGc7nXEniZlp0zyKJ0WUdOiZWLBZft9Yug1uA==", ++ "license": "MIT", + "dependencies": { + "@types/express": "*", + "@types/passport": "*" +@@ -11092,9 +10227,10 @@ + "license": "MIT" + }, + "node_modules/@types/qs": { +- "version": "6.9.7", +- "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", +- "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" ++ "version": "6.14.0", ++ "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.14.0.tgz", ++ "integrity": "sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ==", ++ "license": "MIT" + }, + "node_modules/@types/range-parser": { + "version": "1.2.4", +@@ -11191,30 +10327,6 @@ + "integrity": "sha512-2dJ1QnwcyCmxeIAzOaBx/r1JqMIqZ7rohxJMY0UynSQidEDfb9X2x3OHMthBXDtTzSFJ1usY934wakxgm7d+Wg==", + "dev": true + }, +- "node_modules/@types/request": { +- "version": "2.48.12", +- "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.12.tgz", +- "integrity": "sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==", +- "dependencies": { +- "@types/caseless": "*", +- "@types/node": "*", +- "@types/tough-cookie": "*", +- "form-data": "^2.5.0" +- } +- }, +- "node_modules/@types/request/node_modules/form-data": { +- "version": "2.5.1", +- "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", +- "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", +- "dependencies": { +- "asynckit": "^0.4.0", +- "combined-stream": "^1.0.6", +- "mime-types": "^2.1.12" +- }, +- "engines": { +- "node": ">= 0.12" +- } +- }, + "node_modules/@types/resolve": { + "version": "1.20.6", + "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.20.6.tgz", +@@ -11231,7 +10343,8 @@ + "node_modules/@types/semver": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.0.tgz", +- "integrity": "sha512-G8hZ6XJiHnuhQKR7ZmysCeJWE08o8T0AXtk5darsCaTVsYZhhgUrq53jizaR2FvsoeCwJhlmwTjkXBY5Pn/ZHw==" ++ "integrity": "sha512-G8hZ6XJiHnuhQKR7ZmysCeJWE08o8T0AXtk5darsCaTVsYZhhgUrq53jizaR2FvsoeCwJhlmwTjkXBY5Pn/ZHw==", ++ "dev": true + }, + "node_modules/@types/send": { + "version": "0.17.1", +@@ -11325,11 +10438,6 @@ + "@types/node": "*" + } + }, +- "node_modules/@types/tough-cookie": { +- "version": "4.0.5", +- "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", +- "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==" +- }, + "node_modules/@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", +@@ -11382,27 +10490,11 @@ + "@types/node": "*" + } + }, +- "node_modules/@types/xml-crypto": { +- "version": "1.4.5", +- "resolved": "https://registry.npmjs.org/@types/xml-crypto/-/xml-crypto-1.4.5.tgz", +- "integrity": "sha512-rHc0tlw/ixu7PCqqlpmP9KDIA79IsoV+HFnhJDsdS4MkVAEhBNaazXjv92Xf9oYjWp9e4His4Qzo8fOzoTjT+Q==", +- "dependencies": { +- "@types/node": "*", +- "xpath": "0.0.27" +- } +- }, +- "node_modules/@types/xml-crypto/node_modules/xpath": { +- "version": "0.0.27", +- "resolved": "https://registry.npmjs.org/xpath/-/xpath-0.0.27.tgz", +- "integrity": "sha512-fg03WRxtkCV6ohClePNAECYsmpKKTv5L8y/X3Dn1hQrec3POx2jHZ/0P2qQ6HvsrU1BmeqXcof3NGGueG6LxwQ==", +- "engines": { +- "node": ">=0.6.0" +- } +- }, + "node_modules/@types/xml-encryption": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@types/xml-encryption/-/xml-encryption-1.2.4.tgz", + "integrity": "sha512-I69K/WW1Dv7j6O3jh13z0X8sLWJRXbu5xnHDl9yHzUNDUBtUoBY058eb5s+x/WG6yZC1h8aKdI2EoyEPjyEh+Q==", ++ "license": "MIT", + "dependencies": { + "@types/node": "*" + } +@@ -11411,6 +10503,7 @@ + "version": "0.4.14", + "resolved": "https://registry.npmjs.org/@types/xml2js/-/xml2js-0.4.14.tgz", + "integrity": "sha512-4YnrRemBShWRO2QjvUin8ESA41rH+9nQGLUGZV/1IDhi3SL9OhdpNC/MrulTWuptXKwhx/aDxE7toV0f/ypIXQ==", ++ "license": "MIT", + "dependencies": { + "@types/node": "*" + } +@@ -12658,6 +11751,15 @@ + } + } + }, ++ "node_modules/@xmldom/is-dom-node": { ++ "version": "1.0.1", ++ "resolved": "https://registry.npmjs.org/@xmldom/is-dom-node/-/is-dom-node-1.0.1.tgz", ++ "integrity": "sha512-CJDxIgE5I0FH+ttq/Fxy6nRpxP70+e2O048EPe85J2use3XKdatVM7dDVvFNjQudd9B49NPoZ+8PG49zj4Er8Q==", ++ "license": "MIT", ++ "engines": { ++ "node": ">= 16" ++ } ++ }, + "node_modules/@xmldom/xmldom": { + "version": "0.7.13", + "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.7.13.tgz", +@@ -13877,22 +12979,6 @@ + "node": ">=4" + } + }, +- "node_modules/axios": { +- "version": "1.8.4", +- "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.4.tgz", +- "integrity": "sha512-eBSYY4Y68NNlHbHBMdeDmKNtDgXWhQsJcGqzO3iLUM0GraQFSS9cVgPX5I9b3lbdFKyYoAEGAZF1DwhTaljNAw==", +- "license": "MIT", +- "dependencies": { +- "follow-redirects": "^1.15.6", +- "form-data": "^4.0.0", +- "proxy-from-env": "^1.1.0" +- } +- }, +- "node_modules/axios/node_modules/proxy-from-env": { +- "version": "1.1.0", +- "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", +- "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" +- }, + "node_modules/axobject-query": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-3.1.1.tgz", +@@ -14536,15 +13622,6 @@ + "node": ">=8" + } + }, +- "node_modules/bindings": { +- "version": "1.5.0", +- "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", +- "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", +- "license": "MIT", +- "dependencies": { +- "file-uri-to-path": "1.0.0" +- } +- }, + "node_modules/bintrees": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz", +@@ -15970,11 +15047,6 @@ + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" + }, +- "node_modules/console-log-level": { +- "version": "1.4.1", +- "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.1.tgz", +- "integrity": "sha512-VZzbIORbP+PPcN/gg3DXClTLPLg5Slwd5fL2MIc+o1qZ4BXBvWyc6QxPk6T/Mkr6IVjRpoAGf32XxP3ZWMVRcQ==" +- }, + "node_modules/constantinople": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/constantinople/-/constantinople-4.0.1.tgz", +@@ -16037,59 +15109,6 @@ + "node": ">= 0.6" + } + }, +- "node_modules/contentful": { +- "version": "10.8.5", +- "resolved": "https://registry.npmjs.org/contentful/-/contentful-10.8.5.tgz", +- "integrity": "sha512-aol2LmjRAmuOKQPITZtNasIkRFIECnrcQf2UiXGBZ2ldUpCAjyr533553fuGst2SeHZoKleLyYqFNgHcznDBUw==", +- "dependencies": { +- "@contentful/rich-text-types": "^16.0.2", +- "axios": "^1.6.7", +- "contentful-resolve-response": "^1.8.1", +- "contentful-sdk-core": "^8.1.0", +- "json-stringify-safe": "^5.0.1", +- "type-fest": "^4.0.0" +- }, +- "engines": { +- "node": ">=18" +- } +- }, +- "node_modules/contentful-resolve-response": { +- "version": "1.8.1", +- "resolved": "https://registry.npmjs.org/contentful-resolve-response/-/contentful-resolve-response-1.8.1.tgz", +- "integrity": "sha512-VXGK2c8dBIGcRCknqudKmkDr2PzsUYfjLN6hhx71T09UzoXOdA/c0kfDhsf/BBCBWPWcLaUgaJEFU0lCo45TSg==", +- "dependencies": { +- "fast-copy": "^2.1.7" +- }, +- "engines": { +- "node": ">=4.7.2" +- } +- }, +- "node_modules/contentful-sdk-core": { +- "version": "8.1.2", +- "resolved": "https://registry.npmjs.org/contentful-sdk-core/-/contentful-sdk-core-8.1.2.tgz", +- "integrity": "sha512-XZvX2JMJF4YiICXLrHFv59KBHaQJ6ElqAP8gSNgnCu4x+pPG7Y1bC2JMNOiyAgJuGQGVUOcNZ5PmK+tsNEayYw==", +- "dependencies": { +- "fast-copy": "^2.1.7", +- "lodash.isplainobject": "^4.0.6", +- "lodash.isstring": "^4.0.1", +- "p-throttle": "^4.1.1", +- "qs": "^6.11.2" +- }, +- "engines": { +- "node": ">=12" +- } +- }, +- "node_modules/contentful/node_modules/type-fest": { +- "version": "4.13.1", +- "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.13.1.tgz", +- "integrity": "sha512-ASMgM+Vf2cLwDMt1KXSkMUDSYCxtckDJs8zsaVF/mYteIsiARKCVtyXtcK38mIKbLTctZP8v6GMqdNaeI3fo7g==", +- "engines": { +- "node": ">=16" +- }, +- "funding": { +- "url": "https://github.com/sponsors/sindresorhus" +- } +- }, + "node_modules/convert-source-map": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.8.0.tgz", +@@ -17614,17 +16633,6 @@ + "dev": true, + "license": "MIT" + }, +- "node_modules/delay": { +- "version": "5.0.0", +- "resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz", +- "integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw==", +- "engines": { +- "node": ">=10" +- }, +- "funding": { +- "url": "https://github.com/sponsors/sindresorhus" +- } +- }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", +@@ -17995,20 +17003,6 @@ + "tslib": "^2.0.3" + } + }, +- "node_modules/dot-prop": { +- "version": "6.0.1", +- "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-6.0.1.tgz", +- "integrity": "sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==", +- "dependencies": { +- "is-obj": "^2.0.0" +- }, +- "engines": { +- "node": ">=10" +- }, +- "funding": { +- "url": "https://github.com/sponsors/sindresorhus" +- } +- }, + "node_modules/downshift": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/downshift/-/downshift-9.0.9.tgz", +@@ -19954,25 +18948,6 @@ + "dev": true, + "license": "MIT" + }, +- "node_modules/eventid": { +- "version": "2.0.1", +- "resolved": "https://registry.npmjs.org/eventid/-/eventid-2.0.1.tgz", +- "integrity": "sha512-sPNTqiMokAvV048P2c9+foqVJzk49o6d4e0D/sq5jog3pw+4kBgyR0gaM1FM7Mx6Kzd9dztesh9oYz1LWWOpzw==", +- "dependencies": { +- "uuid": "^8.0.0" +- }, +- "engines": { +- "node": ">=10" +- } +- }, +- "node_modules/eventid/node_modules/uuid": { +- "version": "8.3.2", +- "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", +- "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", +- "bin": { +- "uuid": "dist/bin/uuid" +- } +- }, + "node_modules/events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", +@@ -20541,11 +19516,6 @@ + "node": ">=18" + } + }, +- "node_modules/fast-copy": { +- "version": "2.1.7", +- "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-2.1.7.tgz", +- "integrity": "sha512-ozrGwyuCTAy7YgFCua8rmqmytECYk/JYAMXcswOcm0qvGoE3tPb7ivBeIHTOK2DiapBhDZgacIhzhQIKU5TCfA==" +- }, + "node_modules/fast-crc32c": { + "version": "2.0.0", + "resolved": "git+ssh://git@github.com/overleaf/node-fast-crc32c.git#aae6b2a4c7a7a159395df9cc6c38dfde702d6f51", +@@ -20779,12 +19749,6 @@ + "node": "^10.12.0 || >=12.0.0" + } + }, +- "node_modules/file-uri-to-path": { +- "version": "1.0.0", +- "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", +- "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", +- "license": "MIT" +- }, + "node_modules/filelist": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", +@@ -21001,15 +19965,6 @@ + "micromatch": "^4.0.2" + } + }, +- "node_modules/findit2": { +- "version": "2.2.3", +- "resolved": "https://registry.npmjs.org/findit2/-/findit2-2.2.3.tgz", +- "integrity": "sha512-lg/Moejf4qXovVutL0Lz4IsaPoNYMuxt4PA0nGqFxnJ1CTTGGlEO2wKgoDpwknhvZ8k4Q2F+eesgkLbG2Mxfog==", +- "license": "MIT", +- "engines": { +- "node": ">=0.8.22" +- } +- }, + "node_modules/flat": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", +@@ -21071,6 +20026,7 @@ + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", ++ "dev": true, + "funding": [ + { + "type": "individual", +@@ -21226,12 +20182,15 @@ + } + }, + "node_modules/form-data": { +- "version": "4.0.0", +- "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", +- "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", ++ "version": "4.0.4", ++ "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", ++ "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", ++ "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", ++ "es-set-tostringtag": "^2.1.0", ++ "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { +@@ -21894,352 +20853,6 @@ + "dev": true, + "peer": true + }, +- "node_modules/google-auth-library": { +- "version": "7.14.1", +- "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-7.14.1.tgz", +- "integrity": "sha512-5Rk7iLNDFhFeBYc3s8l1CqzbEBcdhwR193RlD4vSNFajIcINKI8W8P0JLmBpwymHqqWbX34pJDQu39cSy/6RsA==", +- "dependencies": { +- "arrify": "^2.0.0", +- "base64-js": "^1.3.0", +- "ecdsa-sig-formatter": "^1.0.11", +- "fast-text-encoding": "^1.0.0", +- "gaxios": "^4.0.0", +- "gcp-metadata": "^4.2.0", +- "gtoken": "^5.0.4", +- "jws": "^4.0.0", +- "lru-cache": "^6.0.0" +- }, +- "engines": { +- "node": ">=10" +- } +- }, +- "node_modules/google-auth-library/node_modules/gaxios": { +- "version": "4.3.2", +- "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-4.3.2.tgz", +- "integrity": "sha512-T+ap6GM6UZ0c4E6yb1y/hy2UB6hTrqhglp3XfmU9qbLCGRYhLVV5aRPpC4EmoG8N8zOnkYCgoBz+ScvGAARY6Q==", +- "dependencies": { +- "abort-controller": "^3.0.0", +- "extend": "^3.0.2", +- "https-proxy-agent": "^5.0.0", +- "is-stream": "^2.0.0", +- "node-fetch": "^2.6.1" +- }, +- "engines": { +- "node": ">=10" +- } +- }, +- "node_modules/google-auth-library/node_modules/gcp-metadata": { +- "version": "4.3.1", +- "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.3.1.tgz", +- "integrity": "sha512-x850LS5N7V1F3UcV7PoupzGsyD6iVwTVvsh3tbXfkctZnBnjW5yu5z1/3k3SehF7TyoTIe78rJs02GMMy+LF+A==", +- "dependencies": { +- "gaxios": "^4.0.0", +- "json-bigint": "^1.0.0" +- }, +- "engines": { +- "node": ">=10" +- } +- }, +- "node_modules/google-auth-library/node_modules/lru-cache": { +- "version": "6.0.0", +- "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", +- "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", +- "dependencies": { +- "yallist": "^4.0.0" +- }, +- "engines": { +- "node": ">=10" +- } +- }, +- "node_modules/google-auth-library/node_modules/yallist": { +- "version": "4.0.0", +- "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", +- "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" +- }, +- "node_modules/google-gax": { +- "version": "4.3.5", +- "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-4.3.5.tgz", +- "integrity": "sha512-zXRSGgHp33ottCQMdYlKEFX/MhWkzKVX5P3Vpmx+DW6rtseLILzp3V0YV5Rh4oQzzkM0BH9+nJIyX01EUgmd3g==", +- "dependencies": { +- "@grpc/grpc-js": "~1.10.3", +- "@grpc/proto-loader": "^0.7.0", +- "@types/long": "^4.0.0", +- "abort-controller": "^3.0.0", +- "duplexify": "^4.0.0", +- "google-auth-library": "^9.3.0", +- "node-fetch": "^2.6.1", +- "object-hash": "^3.0.0", +- "proto3-json-serializer": "^2.0.0", +- "protobufjs": "7.3.0", +- "retry-request": "^7.0.0", +- "uuid": "^9.0.1" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/google-gax/node_modules/@grpc/grpc-js": { +- "version": "1.10.11", +- "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.10.11.tgz", +- "integrity": "sha512-3RaoxOqkHHN2c05bwtBNVJmOf/UwMam0rZYtdl7dsRpsvDwcNpv6LkGgzltQ7xVf822LzBoKEPRvf4D7+xeIDw==", +- "dependencies": { +- "@grpc/proto-loader": "^0.7.13", +- "@js-sdsl/ordered-map": "^4.4.2" +- }, +- "engines": { +- "node": ">=12.10.0" +- } +- }, +- "node_modules/google-gax/node_modules/agent-base": { +- "version": "7.1.1", +- "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", +- "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", +- "dependencies": { +- "debug": "^4.3.4" +- }, +- "engines": { +- "node": ">= 14" +- } +- }, +- "node_modules/google-gax/node_modules/debug": { +- "version": "4.3.5", +- "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", +- "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", +- "dependencies": { +- "ms": "2.1.2" +- }, +- "engines": { +- "node": ">=6.0" +- }, +- "peerDependenciesMeta": { +- "supports-color": { +- "optional": true +- } +- } +- }, +- "node_modules/google-gax/node_modules/duplexify": { +- "version": "4.1.2", +- "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.2.tgz", +- "integrity": "sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw==", +- "dependencies": { +- "end-of-stream": "^1.4.1", +- "inherits": "^2.0.3", +- "readable-stream": "^3.1.1", +- "stream-shift": "^1.0.0" +- } +- }, +- "node_modules/google-gax/node_modules/gaxios": { +- "version": "6.6.0", +- "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.6.0.tgz", +- "integrity": "sha512-bpOZVQV5gthH/jVCSuYuokRo2bTKOcuBiVWpjmTn6C5Agl5zclGfTljuGsQZxwwDBkli+YhZhP4TdlqTnhOezQ==", +- "dependencies": { +- "extend": "^3.0.2", +- "https-proxy-agent": "^7.0.1", +- "is-stream": "^2.0.0", +- "node-fetch": "^2.6.9", +- "uuid": "^9.0.1" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/google-gax/node_modules/gcp-metadata": { +- "version": "6.1.0", +- "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", +- "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", +- "dependencies": { +- "gaxios": "^6.0.0", +- "json-bigint": "^1.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/google-gax/node_modules/google-auth-library": { +- "version": "9.10.0", +- "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.10.0.tgz", +- "integrity": "sha512-ol+oSa5NbcGdDqA+gZ3G3mev59OHBZksBTxY/tYwjtcp1H/scAFwJfSQU9/1RALoyZ7FslNbke8j4i3ipwlyuQ==", +- "dependencies": { +- "base64-js": "^1.3.0", +- "ecdsa-sig-formatter": "^1.0.11", +- "gaxios": "^6.1.1", +- "gcp-metadata": "^6.1.0", +- "gtoken": "^7.0.0", +- "jws": "^4.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/google-gax/node_modules/gtoken": { +- "version": "7.1.0", +- "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", +- "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", +- "dependencies": { +- "gaxios": "^6.0.0", +- "jws": "^4.0.0" +- }, +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/google-gax/node_modules/https-proxy-agent": { +- "version": "7.0.4", +- "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", +- "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", +- "dependencies": { +- "agent-base": "^7.0.2", +- "debug": "4" +- }, +- "engines": { +- "node": ">= 14" +- } +- }, +- "node_modules/google-gax/node_modules/ms": { +- "version": "2.1.2", +- "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", +- "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" +- }, +- "node_modules/google-gax/node_modules/object-hash": { +- "version": "3.0.0", +- "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", +- "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", +- "engines": { +- "node": ">= 6" +- } +- }, +- "node_modules/google-gax/node_modules/proto3-json-serializer": { +- "version": "2.0.2", +- "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-2.0.2.tgz", +- "integrity": "sha512-SAzp/O4Yh02jGdRc+uIrGoe87dkN/XtwxfZ4ZyafJHymd79ozp5VG5nyZ7ygqPM5+cpLDjjGnYFUkngonyDPOQ==", +- "dependencies": { +- "protobufjs": "^7.2.5" +- }, +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/google-gax/node_modules/protobufjs": { +- "version": "7.3.0", +- "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.3.0.tgz", +- "integrity": "sha512-YWD03n3shzV9ImZRX3ccbjqLxj7NokGN0V/ESiBV5xWqrommYHYiihuIyavq03pWSGqlyvYUFmfoMKd+1rPA/g==", +- "hasInstallScript": true, +- "dependencies": { +- "@protobufjs/aspromise": "^1.1.2", +- "@protobufjs/base64": "^1.1.2", +- "@protobufjs/codegen": "^2.0.4", +- "@protobufjs/eventemitter": "^1.1.0", +- "@protobufjs/fetch": "^1.1.0", +- "@protobufjs/float": "^1.0.2", +- "@protobufjs/inquire": "^1.1.0", +- "@protobufjs/path": "^1.1.2", +- "@protobufjs/pool": "^1.1.0", +- "@protobufjs/utf8": "^1.1.0", +- "@types/node": ">=13.7.0", +- "long": "^5.0.0" +- }, +- "engines": { +- "node": ">=12.0.0" +- } +- }, +- "node_modules/google-gax/node_modules/retry-request": { +- "version": "7.0.2", +- "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", +- "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", +- "dependencies": { +- "@types/request": "^2.48.8", +- "extend": "^3.0.2", +- "teeny-request": "^9.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/google-gax/node_modules/teeny-request": { +- "version": "9.0.0", +- "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", +- "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", +- "dependencies": { +- "http-proxy-agent": "^5.0.0", +- "https-proxy-agent": "^5.0.0", +- "node-fetch": "^2.6.9", +- "stream-events": "^1.0.5", +- "uuid": "^9.0.0" +- }, +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/google-gax/node_modules/teeny-request/node_modules/agent-base": { +- "version": "6.0.2", +- "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", +- "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", +- "dependencies": { +- "debug": "4" +- }, +- "engines": { +- "node": ">= 6.0.0" +- } +- }, +- "node_modules/google-gax/node_modules/teeny-request/node_modules/https-proxy-agent": { +- "version": "5.0.1", +- "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", +- "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", +- "dependencies": { +- "agent-base": "6", +- "debug": "4" +- }, +- "engines": { +- "node": ">= 6" +- } +- }, +- "node_modules/google-gax/node_modules/uuid": { +- "version": "9.0.1", +- "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", +- "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", +- "funding": [ +- "https://github.com/sponsors/broofa", +- "https://github.com/sponsors/ctavan" +- ], +- "bin": { +- "uuid": "dist/bin/uuid" +- } +- }, +- "node_modules/google-logging-utils": { +- "version": "0.0.2", +- "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz", +- "integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==", +- "license": "Apache-2.0", +- "engines": { +- "node": ">=14" +- } +- }, +- "node_modules/google-p12-pem": { +- "version": "3.1.3", +- "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.1.3.tgz", +- "integrity": "sha512-MC0jISvzymxePDVembypNefkAQp+DRP7dBE+zNUPaIjEspIlYg0++OrsNr248V9tPbz6iqtZ7rX1hxWA5B8qBQ==", +- "dependencies": { +- "node-forge": "^1.0.0" +- }, +- "bin": { +- "gp12-pem": "build/src/bin/gp12-pem.js" +- }, +- "engines": { +- "node": ">=10" +- } +- }, +- "node_modules/google-proto-files": { +- "version": "3.0.3", +- "resolved": "https://registry.npmjs.org/google-proto-files/-/google-proto-files-3.0.3.tgz", +- "integrity": "sha512-7JaU/smPA/FpNsCaXyVjitwiQyn5zYC/ETA+xag3ziovBojIWvzevyrbVqhxgnQdgMJ0p1RVSvpzQL6hkg6yGw==", +- "dependencies": { +- "protobufjs": "^7.0.0", +- "walkdir": "^0.4.0" +- }, +- "engines": { +- "node": ">=12.0.0" +- } +- }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", +@@ -22272,34 +20885,6 @@ + "lodash": "^4.17.15" + } + }, +- "node_modules/gtoken": { +- "version": "5.3.1", +- "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-5.3.1.tgz", +- "integrity": "sha512-yqOREjzLHcbzz1UrQoxhBtpk8KjrVhuqPE7od1K2uhyxG2BHjKZetlbLw/SPZak/QqTIQW+addS+EcjqQsZbwQ==", +- "dependencies": { +- "gaxios": "^4.0.0", +- "google-p12-pem": "^3.0.3", +- "jws": "^4.0.0" +- }, +- "engines": { +- "node": ">=10" +- } +- }, +- "node_modules/gtoken/node_modules/gaxios": { +- "version": "4.3.2", +- "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-4.3.2.tgz", +- "integrity": "sha512-T+ap6GM6UZ0c4E6yb1y/hy2UB6hTrqhglp3XfmU9qbLCGRYhLVV5aRPpC4EmoG8N8zOnkYCgoBz+ScvGAARY6Q==", +- "dependencies": { +- "abort-controller": "^3.0.0", +- "extend": "^3.0.2", +- "https-proxy-agent": "^5.0.0", +- "is-stream": "^2.0.0", +- "node-fetch": "^2.6.1" +- }, +- "engines": { +- "node": ">=10" +- } +- }, + "node_modules/gulp-sort": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/gulp-sort/-/gulp-sort-2.0.0.tgz", +@@ -22814,6 +21399,7 @@ + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.5.2.tgz", + "integrity": "sha512-K//PSRMQk4FZ78Kyau+mZurHn3FH0Vwr+H36eE0rPbeYkRRi9YxceYPhuN60UwWorxyKHhqoAJl2OFKa4BVtaA==", ++ "dev": true, + "funding": [ + { + "type": "github", +@@ -23998,14 +22584,6 @@ + "url": "https://github.com/sponsors/ljharb" + } + }, +- "node_modules/is-obj": { +- "version": "2.0.0", +- "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", +- "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", +- "engines": { +- "node": ">=8" +- } +- }, + "node_modules/is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", +@@ -25922,12 +24500,6 @@ + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w=" + }, +- "node_modules/lodash.sortby": { +- "version": "4.7.0", +- "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", +- "integrity": "sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==", +- "license": "MIT" +- }, + "node_modules/lodash.support": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/lodash.support/-/lodash.support-2.4.1.tgz", +@@ -27554,9 +26126,9 @@ + } + }, + "node_modules/multer": { +- "version": "2.0.1", +- "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.1.tgz", +- "integrity": "sha512-Ug8bXeTIUlxurg8xLTEskKShvcKDZALo1THEX5E41pYCD2sCVub5/kIRIGqWNoqV6szyLyQKV6mD4QUrWE5GCQ==", ++ "version": "2.0.2", ++ "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz", ++ "integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==", + "license": "MIT", + "dependencies": { + "append-field": "^1.0.0", +@@ -28323,6 +26895,7 @@ + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", ++ "dev": true, + "dependencies": { + "ee-first": "1.1.1" + }, +@@ -28591,17 +27164,6 @@ + "url": "https://github.com/sponsors/sindresorhus" + } + }, +- "node_modules/p-throttle": { +- "version": "4.1.1", +- "resolved": "https://registry.npmjs.org/p-throttle/-/p-throttle-4.1.1.tgz", +- "integrity": "sha512-TuU8Ato+pRTPJoDzYD4s7ocJYcNSEZRvlxoq3hcPI2kZDZ49IQ1Wkj7/gDJc3X7XiEAAvRGtDzdXJI0tC3IL1g==", +- "engines": { +- "node": ">=10" +- }, +- "funding": { +- "url": "https://github.com/sponsors/sindresorhus" +- } +- }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", +@@ -28683,14 +27245,6 @@ + "url": "https://github.com/sponsors/sindresorhus" + } + }, +- "node_modules/parse-ms": { +- "version": "2.1.0", +- "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", +- "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==", +- "engines": { +- "node": ">=6" +- } +- }, + "node_modules/parse-srcset": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/parse-srcset/-/parse-srcset-1.0.2.tgz", +@@ -29083,7 +27637,7 @@ + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/superagent/-/superagent-7.1.6.tgz", + "integrity": "sha512-gZkVCQR1gy/oUXr+kxJMLDjla434KmSOKbx5iGD30Ql+AkJQ/YlPKECJy2nhqOsHLjGHzoDTXNSjhnvWhzKk7g==", +- "deprecated": "Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net", ++ "deprecated": "Please upgrade to superagent v10.2.2+, see release notes at https://github.com/forwardemail/superagent/releases/tag/v10.2.2 - maintenance is supported by Forward Email @ https://forwardemail.net", + "license": "MIT", + "dependencies": { + "component-emitter": "^1.3.0", +@@ -30228,39 +28782,6 @@ + "node": ">=0.10.0" + } + }, +- "node_modules/pprof": { +- "version": "4.0.0", +- "resolved": "https://registry.npmjs.org/pprof/-/pprof-4.0.0.tgz", +- "integrity": "sha512-Yhfk7Y0G1MYsy97oXxmSG5nvbM1sCz9EALiNhW/isAv5Xf7svzP+1RfGeBlS6mLSgRJvgSLh6Mi5DaisQuPttw==", +- "hasInstallScript": true, +- "license": "Apache-2.0", +- "dependencies": { +- "@mapbox/node-pre-gyp": "^1.0.9", +- "bindings": "^1.2.1", +- "delay": "^5.0.0", +- "findit2": "^2.2.3", +- "nan": "^2.17.0", +- "p-limit": "^3.0.0", +- "protobufjs": "~7.2.4", +- "source-map": "~0.8.0-beta.0", +- "split": "^1.0.1" +- }, +- "engines": { +- "node": ">=14.0.0" +- } +- }, +- "node_modules/pprof/node_modules/source-map": { +- "version": "0.8.0-beta.0", +- "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.8.0-beta.0.tgz", +- "integrity": "sha512-2ymg6oRBpebeZi9UUNsgQ89bhx01TcTkmNTGnNO88imTmbSgy4nfujrgVEFKWpMTEGA11EDkTt7mqObTPdigIA==", +- "license": "BSD-3-Clause", +- "dependencies": { +- "whatwg-url": "^7.0.0" +- }, +- "engines": { +- "node": ">= 8" +- } +- }, + "node_modules/preact": { + "version": "10.26.5", + "resolved": "https://registry.npmjs.org/preact/-/preact-10.26.5.tgz", +@@ -30364,20 +28885,6 @@ + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, +- "node_modules/pretty-ms": { +- "version": "7.0.1", +- "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-7.0.1.tgz", +- "integrity": "sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==", +- "dependencies": { +- "parse-ms": "^2.1.0" +- }, +- "engines": { +- "node": ">=10" +- }, +- "funding": { +- "url": "https://github.com/sponsors/sindresorhus" +- } +- }, + "node_modules/private": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/private/-/private-0.1.8.tgz", +@@ -30689,16 +29196,6 @@ + "once": "^1.3.1" + } + }, +- "node_modules/pumpify": { +- "version": "2.0.1", +- "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-2.0.1.tgz", +- "integrity": "sha512-m7KOje7jZxrmutanlkS1daj1dS6z6BgslzOXmcSEpIlCxM3VJH7lG5QLeck/6hgF6F4crFf01UtQmNsJfweTAw==", +- "dependencies": { +- "duplexify": "^4.1.1", +- "inherits": "^2.0.3", +- "pump": "^3.0.0" +- } +- }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", +@@ -31838,13 +30335,17 @@ + } + }, + "node_modules/request/node_modules/form-data": { +- "version": "2.3.3", +- "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", +- "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", ++ "version": "2.5.5", ++ "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.5.tgz", ++ "integrity": "sha512-jqdObeR2rxZZbPSGL+3VckHMYtu+f9//KXBsVny6JSX/pa38Fy+bGjuG8eW/H6USNQWhLi8Num++cU2yOCNz4A==", ++ "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", +- "combined-stream": "^1.0.6", +- "mime-types": "^2.1.12" ++ "combined-stream": "^1.0.8", ++ "es-set-tostringtag": "^2.1.0", ++ "hasown": "^2.0.2", ++ "mime-types": "^2.1.35", ++ "safe-buffer": "^5.2.1" + }, + "engines": { + "node": ">= 0.12" +@@ -31858,6 +30359,26 @@ + "node": ">=0.6" + } + }, ++ "node_modules/request/node_modules/safe-buffer": { ++ "version": "5.2.1", ++ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", ++ "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", ++ "funding": [ ++ { ++ "type": "github", ++ "url": "https://github.com/sponsors/feross" ++ }, ++ { ++ "type": "patreon", ++ "url": "https://www.patreon.com/feross" ++ }, ++ { ++ "type": "consulting", ++ "url": "https://feross.org/support" ++ } ++ ], ++ "license": "MIT" ++ }, + "node_modules/request/node_modules/tough-cookie": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz", +@@ -33458,18 +31979,6 @@ + "wbuf": "^1.7.3" + } + }, +- "node_modules/split": { +- "version": "1.0.1", +- "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", +- "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", +- "license": "MIT", +- "dependencies": { +- "through": "2" +- }, +- "engines": { +- "node": "*" +- } +- }, + "node_modules/split-ca": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", +@@ -34320,7 +32829,7 @@ + "version": "3.8.3", + "resolved": "https://registry.npmjs.org/superagent/-/superagent-3.8.3.tgz", + "integrity": "sha512-GLQtLMCoEIK4eDv6OGtkOoSMt3D+oq0y3dsxMuYuDvaNUvuT8eFBuLmfR0iYYzHC1e8hpzC6ZsxbuP6DIalMFA==", +- "deprecated": "Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net", ++ "deprecated": "Please upgrade to superagent v10.2.2+, see release notes at https://github.com/forwardemail/superagent/releases/tag/v10.2.2 - maintenance is supported by Forward Email @ https://forwardemail.net", + "license": "MIT", + "dependencies": { + "component-emitter": "^1.2.0", +@@ -34347,42 +32856,6 @@ + "ms": "^2.1.1" + } + }, +- "node_modules/superagent/node_modules/form-data": { +- "version": "2.5.3", +- "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.3.tgz", +- "integrity": "sha512-XHIrMD0NpDrNM/Ckf7XJiBbLl57KEhT3+i3yY+eWm+cqYZJQTZrKo8Y8AWKnuV5GT4scfuUGt9LzNoIx3dU1nQ==", +- "license": "MIT", +- "dependencies": { +- "asynckit": "^0.4.0", +- "combined-stream": "^1.0.8", +- "es-set-tostringtag": "^2.1.0", +- "mime-types": "^2.1.35", +- "safe-buffer": "^5.2.1" +- }, +- "engines": { +- "node": ">= 0.12" +- } +- }, +- "node_modules/superagent/node_modules/form-data/node_modules/safe-buffer": { +- "version": "5.2.1", +- "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", +- "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", +- "funding": [ +- { +- "type": "github", +- "url": "https://github.com/sponsors/feross" +- }, +- { +- "type": "patreon", +- "url": "https://www.patreon.com/feross" +- }, +- { +- "type": "consulting", +- "url": "https://feross.org/support" +- } +- ], +- "license": "MIT" +- }, + "node_modules/superagent/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", +@@ -35223,7 +33696,8 @@ + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", +- "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" ++ "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", ++ "dev": true + }, + "node_modules/through2": { + "version": "4.0.2", +@@ -35502,15 +33976,6 @@ + "node": ">= 4.0.0" + } + }, +- "node_modules/tr46": { +- "version": "1.0.1", +- "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz", +- "integrity": "sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA==", +- "license": "MIT", +- "dependencies": { +- "punycode": "^2.1.0" +- } +- }, + "node_modules/traverse": { + "version": "0.6.11", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.6.11.tgz", +@@ -36519,14 +34984,6 @@ + "dev": true, + "license": "ISC" + }, +- "node_modules/walkdir": { +- "version": "0.4.1", +- "resolved": "https://registry.npmjs.org/walkdir/-/walkdir-0.4.1.tgz", +- "integrity": "sha512-3eBwRyEln6E1MSzcxcVpQIhRG8Q1jLvEqRmCZqS3dsfXEDR/AhOF4d+jHg1qvDCpYaVRZjENPQyrVxAkQqxPgQ==", +- "engines": { +- "node": ">=6.0.0" +- } +- }, + "node_modules/warning": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/warning/-/warning-4.0.3.tgz", +@@ -37250,23 +35707,6 @@ + "node": ">=12" + } + }, +- "node_modules/whatwg-url": { +- "version": "7.1.0", +- "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz", +- "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==", +- "license": "MIT", +- "dependencies": { +- "lodash.sortby": "^4.7.0", +- "tr46": "^1.0.1", +- "webidl-conversions": "^4.0.2" +- } +- }, +- "node_modules/whatwg-url/node_modules/webidl-conversions": { +- "version": "4.0.2", +- "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", +- "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==", +- "license": "BSD-2-Clause" +- }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", +@@ -37575,16 +36015,17 @@ + } + }, + "node_modules/xml-crypto": { +- "version": "3.2.1", +- "resolved": "https://registry.npmjs.org/xml-crypto/-/xml-crypto-3.2.1.tgz", +- "integrity": "sha512-0GUNbPtQt+PLMsC5HoZRONX+K6NBJEqpXe/lsvrFj0EqfpGPpVfJKGE7a5jCg8s2+Wkrf/2U1G41kIH+zC9eyQ==", ++ "version": "6.1.2", ++ "resolved": "https://registry.npmjs.org/xml-crypto/-/xml-crypto-6.1.2.tgz", ++ "integrity": "sha512-leBOVQdVi8FvPJrMYoum7Ici9qyxfE4kVi+AkpUoYCSXaQF4IlBm1cneTK9oAxR61LpYxTx7lNcsnBIeRpGW2w==", + "license": "MIT", + "dependencies": { +- "@xmldom/xmldom": "^0.8.8", +- "xpath": "0.0.32" ++ "@xmldom/is-dom-node": "^1.0.1", ++ "@xmldom/xmldom": "^0.8.10", ++ "xpath": "^0.0.33" + }, + "engines": { +- "node": ">=4.0.0" ++ "node": ">=16" + } + }, + "node_modules/xml-crypto/node_modules/@xmldom/xmldom": { +@@ -37597,9 +36038,10 @@ + } + }, + "node_modules/xml-crypto/node_modules/xpath": { +- "version": "0.0.32", +- "resolved": "https://registry.npmjs.org/xpath/-/xpath-0.0.32.tgz", +- "integrity": "sha512-rxMJhSIoiO8vXcWvSifKqhvV96GjiD5wYb8/QHdoRyQvraTpp4IEv944nhGausZZ3u7dhQXteZuZbaqfpB7uYw==", ++ "version": "0.0.33", ++ "resolved": "https://registry.npmjs.org/xpath/-/xpath-0.0.33.tgz", ++ "integrity": "sha512-NNXnzrkDrAzalLhIUc01jO2mOzXGXh1JwPgkihcLLzw98c0WgYDmmjSh1Kl3wzaxSVWMuA+fe0WTWOBDWCBmNA==", ++ "license": "MIT", + "engines": { + "node": ">=0.6.0" + } +@@ -37660,6 +36102,7 @@ + "version": "15.1.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-15.1.1.tgz", + "integrity": "sha512-yMqGBqtXyeN1e3TGYvgNgDVZ3j84W4cwkOXQswghol6APgZWaff9lnbvN7MHYJOiXsvGPXtjTYJEiC9J2wv9Eg==", ++ "license": "MIT", + "engines": { + "node": ">=8.0" + } +@@ -38376,7 +36819,6 @@ + "version": "1.0.0", + "license": "Proprietary", + "dependencies": { +- "@google-cloud/secret-manager": "^5.6.0", + "@overleaf/logger": "*", + "@overleaf/metrics": "*", + "@overleaf/mongo-utils": "*", +@@ -39089,11 +37531,9 @@ + "services/web": { + "name": "@overleaf/web", + "dependencies": { +- "@contentful/rich-text-html-renderer": "^16.0.2", +- "@contentful/rich-text-types": "^16.0.2", + "@google-cloud/bigquery": "^6.0.1", + "@node-oauth/oauth2-server": "^5.1.0", +- "@node-saml/passport-saml": "^4.0.4", ++ "@node-saml/passport-saml": "^5.1.0", + "@overleaf/access-token-encryptor": "*", + "@overleaf/fetch-utils": "*", + "@overleaf/logger": "*", +@@ -39105,7 +37545,6 @@ + "@overleaf/redis-wrapper": "*", + "@overleaf/settings": "*", + "@phosphor-icons/react": "^2.1.7", +- "@slack/webhook": "^7.0.2", + "@stripe/react-stripe-js": "^3.1.1", + "@stripe/stripe-js": "^5.6.0", + "@xmldom/xmldom": "^0.7.13", +@@ -39124,7 +37563,6 @@ + "celebrate": "^15.0.3", + "connect-redis": "^6.1.3", + "content-disposition": "^0.5.0", +- "contentful": "^10.8.5", + "cookie": "^0.2.3", + "cookie-parser": "1.4.6", + "crc-32": "^1.2.2", +@@ -39159,7 +37597,7 @@ + "moment": "^2.29.4", + "mongodb-legacy": "6.1.3", + "mongoose": "8.9.5", +- "multer": "overleaf/multer#4dbceda355efc3fc8ac3cf5c66c3778c8a6fdb23", ++ "multer": "^2.0.2", + "nocache": "^2.1.0", + "node-fetch": "^2.7.0", + "nodemailer": "^6.7.0", +@@ -40077,24 +38515,6 @@ + "url": "https://github.com/sponsors/isaacs" + } + }, +- "services/web/node_modules/multer": { +- "version": "2.0.1", +- "resolved": "git+ssh://git@github.com/overleaf/multer.git#4dbceda355efc3fc8ac3cf5c66c3778c8a6fdb23", +- "integrity": "sha512-kkvPK48OQibR5vIoTQBbZp1uWVCvT9MrW3Y0mqdhFYJP/HVJujb4eSCEU0yj+hyf0Y+H/BKCmPdM4fJnzqAO4w==", +- "license": "MIT", +- "dependencies": { +- "append-field": "^1.0.0", +- "busboy": "^1.6.0", +- "concat-stream": "^2.0.0", +- "mkdirp": "^0.5.6", +- "object-assign": "^4.1.1", +- "type-is": "^1.6.18", +- "xtend": "^4.0.2" +- }, +- "engines": { +- "node": ">= 10.16.0" +- } +- }, + "services/web/node_modules/nise": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/nise/-/nise-1.5.3.tgz", +diff -u -x node_modules -r a/services/history-v1/package.json b/services/history-v1/package.json +--- a/services/history-v1/package.json 2025-06-11 11:06:33.000000000 +0100 ++++ b/services/history-v1/package.json 2025-07-28 12:36:17.000000000 +0100 +@@ -6,7 +6,6 @@ + "license": "Proprietary", + "private": true, + "dependencies": { +- "@google-cloud/secret-manager": "^5.6.0", + "@overleaf/logger": "*", + "@overleaf/metrics": "*", + "@overleaf/mongo-utils": "*", +diff -u -x node_modules -r a/services/web/package.json b/services/web/package.json +--- a/services/web/package.json 2025-06-06 09:16:39.000000000 +0100 ++++ b/services/web/package.json 2025-07-28 12:35:57.000000000 +0100 +@@ -70,11 +70,9 @@ + "safari > 14" + ], + "dependencies": { +- "@contentful/rich-text-html-renderer": "^16.0.2", +- "@contentful/rich-text-types": "^16.0.2", + "@google-cloud/bigquery": "^6.0.1", + "@node-oauth/oauth2-server": "^5.1.0", +- "@node-saml/passport-saml": "^4.0.4", ++ "@node-saml/passport-saml": "^5.1.0", + "@overleaf/access-token-encryptor": "*", + "@overleaf/fetch-utils": "*", + "@overleaf/logger": "*", +@@ -86,9 +84,8 @@ + "@overleaf/redis-wrapper": "*", + "@overleaf/settings": "*", + "@phosphor-icons/react": "^2.1.7", +- "@slack/webhook": "^7.0.2", +- "@stripe/stripe-js": "^5.6.0", + "@stripe/react-stripe-js": "^3.1.1", ++ "@stripe/stripe-js": "^5.6.0", + "@xmldom/xmldom": "^0.7.13", + "accepts": "^1.3.7", + "ajv": "^8.12.0", +@@ -105,7 +102,6 @@ + "celebrate": "^15.0.3", + "connect-redis": "^6.1.3", + "content-disposition": "^0.5.0", +- "contentful": "^10.8.5", + "cookie": "^0.2.3", + "cookie-parser": "1.4.6", + "crc-32": "^1.2.2", +@@ -140,7 +136,7 @@ + "moment": "^2.29.4", + "mongodb-legacy": "6.1.3", + "mongoose": "8.9.5", +- "multer": "overleaf/multer#4dbceda355efc3fc8ac3cf5c66c3778c8a6fdb23", ++ "multer": "^2.0.2", + "nocache": "^2.1.0", + "node-fetch": "^2.7.0", + "nodemailer": "^6.7.0",