diff --git a/services/history-v1/storage/scripts/persist_and_expire_queues.sh b/services/history-v1/storage/scripts/persist_and_expire_queues.sh index d9ff60ea31..cb96eff3e8 100644 --- a/services/history-v1/storage/scripts/persist_and_expire_queues.sh +++ b/services/history-v1/storage/scripts/persist_and_expire_queues.sh @@ -1,3 +1,3 @@ #!/bin/sh -node storage/scripts/persist_redis_chunks.js +node storage/scripts/persist_redis_chunks.mjs node storage/scripts/expire_redis_chunks.js diff --git a/services/history-v1/storage/scripts/persist_redis_chunks.js b/services/history-v1/storage/scripts/persist_redis_chunks.js deleted file mode 100644 index 20963ba90f..0000000000 --- a/services/history-v1/storage/scripts/persist_redis_chunks.js +++ /dev/null @@ -1,70 +0,0 @@ -const logger = require('@overleaf/logger') -const commandLineArgs = require('command-line-args') -const redis = require('../lib/redis') -const knex = require('../lib/knex.js') -const knexReadOnly = require('../lib/knex_read_only.js') -const { client } = require('../lib/mongodb.js') -const { scanAndProcessDueItems } = require('../lib/scan') -const persistBuffer = require('../lib/persist_buffer') -const { claimPersistJob } = require('../lib/chunk_store/redis') -const { loadGlobalBlobs } = require('../lib/blob_store/index.js') - -// Something is registering 11 listeners, over the limit of 10, which generates -// a lot of warning noise. -require('node:events').EventEmitter.defaultMaxListeners = 11 - -const rclient = redis.rclientHistory - -const optionDefinitions = [{ name: 'dry-run', alias: 'd', type: Boolean }] -const options = commandLineArgs(optionDefinitions) -const DRY_RUN = options['dry-run'] || false - -logger.initialize('persist-redis-chunks') - -async function persistProjectAction(projectId) { - const job = await claimPersistJob(projectId) - // Set limits to force us to persist all of the changes. - const farFuture = new Date() - farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000) - const limits = { - maxChanges: 0, - minChangeTimestamp: farFuture, - maxChangeTimestamp: farFuture, - } - await persistBuffer(projectId, limits) - if (job && job.close) { - await job.close() - } -} - -async function runPersistChunks() { - await loadGlobalBlobs() - await scanAndProcessDueItems( - rclient, - 'persistChunks', - 'persist-time', - persistProjectAction, - DRY_RUN - ) -} - -if (require.main === module) { - runPersistChunks() - .catch(err => { - logger.fatal( - { err, taskName: 'persistChunks' }, - 'Unhandled error in runPersistChunks' - ) - process.exit(1) - }) - .finally(async () => { - await redis.disconnect() - await client.close() - await knex.destroy() - await knexReadOnly.destroy() - }) -} else { - module.exports = { - runPersistChunks, - } -} diff --git a/services/history-v1/storage/scripts/persist_redis_chunks.mjs b/services/history-v1/storage/scripts/persist_redis_chunks.mjs new file mode 100644 index 0000000000..41eea6fbd2 --- /dev/null +++ b/services/history-v1/storage/scripts/persist_redis_chunks.mjs @@ -0,0 +1,76 @@ +import logger from '@overleaf/logger' +import commandLineArgs from 'command-line-args' +import * as redis from '../lib/redis.js' +import knex from '../lib/knex.js' +import knexReadOnly from '../lib/knex_read_only.js' +import { client } from '../lib/mongodb.js' +import { scanAndProcessDueItems } from '../lib/scan.js' +import persistBuffer from '../lib/persist_buffer.js' +import { claimPersistJob } from '../lib/chunk_store/redis.js' +import { loadGlobalBlobs } from '../lib/blob_store/index.js' +import { EventEmitter } from 'node:events' +import { fileURLToPath } from 'node:url' + +// Something is registering 11 listeners, over the limit of 10, which generates +// a lot of warning noise. +EventEmitter.defaultMaxListeners = 11 + +const rclient = redis.rclientHistory + +const optionDefinitions = [{ name: 'dry-run', alias: 'd', type: Boolean }] +const options = commandLineArgs(optionDefinitions) +const DRY_RUN = options['dry-run'] || false + +logger.initialize('persist-redis-chunks') + +async function persistProjectAction(projectId) { + const job = await claimPersistJob(projectId) + // Set limits to force us to persist all of the changes. + const farFuture = new Date() + farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000) + const limits = { + maxChanges: 0, + minChangeTimestamp: farFuture, + maxChangeTimestamp: farFuture, + } + await persistBuffer(projectId, limits) + if (job && job.close) { + await job.close() + } +} + +async function runPersistChunks() { + await loadGlobalBlobs() + await scanAndProcessDueItems( + rclient, + 'persistChunks', + 'persist-time', + persistProjectAction, + DRY_RUN + ) +} + +async function main() { + try { + await runPersistChunks() + } catch (err) { + logger.fatal( + { err, taskName: 'persistChunks' }, + 'Unhandled error in runPersistChunks' + ) + process.exit(1) + } finally { + await redis.disconnect() + await client.close() + await knex.destroy() + await knexReadOnly.destroy() + } +} + +// Check if the module is being run directly +const currentScriptPath = fileURLToPath(import.meta.url) +if (process.argv[1] === currentScriptPath) { + main() +} + +export { runPersistChunks } diff --git a/services/history-v1/test/acceptance/js/storage/persist_redis_chunks.test.js b/services/history-v1/test/acceptance/js/storage/persist_redis_chunks.test.js index 3f2a4a390f..58261703bb 100644 --- a/services/history-v1/test/acceptance/js/storage/persist_redis_chunks.test.js +++ b/services/history-v1/test/acceptance/js/storage/persist_redis_chunks.test.js @@ -16,7 +16,7 @@ const { setupProjectState } = require('./support/redis') const { runScript } = require('./support/runscript') const persistChanges = require('../../../../storage/lib/persist_changes') -const SCRIPT_PATH = 'storage/scripts/persist_redis_chunks.js' +const SCRIPT_PATH = 'storage/scripts/persist_redis_chunks.mjs' describe('persist_redis_chunks script', function () { before(cleanup.everything)