From 385523bcdc336f3a6b91d01531113c25ae047389 Mon Sep 17 00:00:00 2001 From: Liangjun Song <146005915+adai26@users.noreply.github.com> Date: Thu, 22 May 2025 11:51:53 +0100 Subject: [PATCH] Merge pull request #24790 from overleaf/ls-use-script-runner Update some scripts to use Script Runner GitOrigin-RevId: aaa11f94dcfd328c158bb02d1b9fb2adfb1bb146 --- libraries/mongo-utils/batchedUpdate.js | 12 ++++-- .../add_salesforce_data_to_subscriptions.mjs | 37 ++++++++++--------- .../web/scripts/add_user_count_to_csv.mjs | 3 +- ...ckfill_recurly_to_subscription_mapping.mjs | 6 ++- .../sync_group_subscription_memberships.mjs | 3 +- .../back_fill_doc_name_for_deleted_docs.mjs | 11 ++++-- .../web/scripts/back_fill_dummy_doc_meta.mjs | 3 +- .../web/scripts/back_fill_staff_access.mjs | 3 +- ...g_user_personal_and_group_subscription.mjs | 20 +++++++--- services/web/scripts/check_docs.mjs | 3 +- .../web/scripts/check_institution_users.mjs | 3 +- services/web/scripts/clear_admin_sessions.mjs | 3 +- .../clear_institution_notifications.mjs | 3 +- services/web/scripts/clear_project_tokens.mjs | 3 +- services/web/scripts/clear_sessions_2fa.mjs | 3 +- services/web/scripts/convert_doc_to_file.mjs | 3 +- .../scripts/count_encrypted_access_tokens.mjs | 3 +- services/web/scripts/count_image_files.mjs | 3 +- .../web/scripts/delete_dangling_file_refs.mjs | 3 +- .../delete_orphaned_doc_comment_ranges.mjs | 3 +- .../delete_orphaned_docs_online_check.mjs | 3 +- services/web/scripts/disconnect_all_users.mjs | 3 +- services/web/scripts/e2e_test_setup.mjs | 3 +- .../web/scripts/find_malformed_filetrees.mjs | 14 +++++-- .../fix_group_invite_emails_to_lowercase.mjs | 14 +++++-- .../web/scripts/fix_malformed_filetree.mjs | 3 +- services/web/scripts/fix_oversized_docs.mjs | 3 +- services/web/scripts/force_doc_flush.mjs | 3 +- .../scripts/history/clean_sl_history_data.mjs | 3 +- .../history/migrate_ranges_support.mjs | 3 +- .../web/scripts/learn/checkSanitize/index.mjs | 3 +- services/web/scripts/lib/ScriptRunner.mjs | 25 ++++++++----- .../lowercase_institution_user_ids.mjs | 3 +- .../merge_group_subscription_members.mjs | 3 +- services/web/scripts/migrate_audit_logs.mjs | 15 ++++++-- .../scripts/oauth/backfill_hashed_secrets.mjs | 3 +- services/web/scripts/oauth/create_token.mjs | 3 +- .../web/scripts/oauth/register_client.mjs | 3 +- services/web/scripts/oauth/remove_client.mjs | 3 +- .../web/scripts/recover_docs_from_redis.mjs | 3 +- .../recurly/change_prices_at_renewal.mjs | 3 +- .../collect_paypal_past_due_invoice.mjs | 3 +- .../scripts/recurly/generate_addon_prices.mjs | 3 +- .../get_manually_billed_users_details.mjs | 3 +- .../recurly/get_recurly_group_prices.mjs | 3 +- ...d_conditions_for_manually_billed_users.mjs | 3 +- .../web/scripts/refresh_institution_users.mjs | 3 +- .../scripts/remove_feature_from_all_users.mjs | 3 +- .../web/scripts/remove_oauth_application.mjs | 3 +- .../web/scripts/remove_unconfirmed_emails.mjs | 13 ++++--- .../web/scripts/restore_orphaned_docs.mjs | 3 +- .../web/scripts/restore_soft_deleted_docs.mjs | 3 +- services/web/scripts/set_tex_live_image.mjs | 3 +- services/web/scripts/soft_delete_project.mjs | 3 +- .../web/scripts/sso_id_migration_check.mjs | 3 +- .../scripts/sso_id_remove_not_migrated.mjs | 3 +- .../web/scripts/undelete_project_to_user.mjs | 3 +- .../web/scripts/unlink_third_party_id.mjs | 3 +- .../back_fill_hiding_ai_features.mjs | 10 +++-- .../enable_wf_autoCreatedAccount.mjs | 3 +- ...e_writefull_without_autoCreatedAccount.mjs | 3 +- .../split_writefull_disabled_from_unset.mjs | 10 +++-- 62 files changed, 225 insertions(+), 112 deletions(-) diff --git a/libraries/mongo-utils/batchedUpdate.js b/libraries/mongo-utils/batchedUpdate.js index 7e3ad677db..41af41f0d4 100644 --- a/libraries/mongo-utils/batchedUpdate.js +++ b/libraries/mongo-utils/batchedUpdate.js @@ -35,6 +35,7 @@ let BATCHED_UPDATE_RUNNING = false * @property {string} [BATCH_RANGE_START] * @property {string} [BATCH_SIZE] * @property {string} [VERBOSE_LOGGING] + * @property {(progress: string) => Promise} [trackProgress] */ /** @@ -210,7 +211,7 @@ async function batchedUpdate( update, projection, findOptions, - batchedUpdateOptions + batchedUpdateOptions = {} ) { // only a single batchedUpdate can run at a time due to global variables if (BATCHED_UPDATE_RUNNING) { @@ -226,6 +227,8 @@ async function batchedUpdate( return 0 } refreshGlobalOptionsForBatchedUpdate(batchedUpdateOptions) + const { trackProgress = async progress => console.warn(progress) } = + batchedUpdateOptions findOptions = findOptions || {} findOptions.readPreference = READ_PREFERENCE_SECONDARY @@ -255,9 +258,10 @@ async function batchedUpdate( nextBatch.map(entry => entry._id) )}` ) - } else { - console.error(`Running update on batch ending ${renderObjectId(end)}`) } + await trackProgress( + `Running update on batch ending ${renderObjectId(end)}` + ) if (typeof update === 'function') { await update(nextBatch) @@ -265,7 +269,7 @@ async function batchedUpdate( await performUpdate(collection, nextBatch, update) } } - console.error(`Completed batch ending ${renderObjectId(end)}`) + await trackProgress(`Completed batch ending ${renderObjectId(end)}`) start = end } return updated diff --git a/services/web/scripts/add_salesforce_data_to_subscriptions.mjs b/services/web/scripts/add_salesforce_data_to_subscriptions.mjs index 02ef02689f..264640f587 100755 --- a/services/web/scripts/add_salesforce_data_to_subscriptions.mjs +++ b/services/web/scripts/add_salesforce_data_to_subscriptions.mjs @@ -4,6 +4,7 @@ import { parse } from 'csv' import Stream from 'node:stream/promises' import { ObjectId } from '../app/src/infrastructure/mongodb.js' import { Subscription } from '../app/src/models/Subscription.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' function usage() { console.log( @@ -76,20 +77,22 @@ const stats = { }, } -function showStats() { - console.log('Stats:') - console.log(` Total rows: ${stats.totalRows}`) - console.log(` Processed rows: ${stats.processedRows}`) - console.log(` Skipped (no subscription ID): ${stats.subscriptionIDMissing}`) - console.log(` Used V1 ID: ${stats.usedV1ID}`) - console.log(` Used Salesforce ID: ${stats.usedSalesforceID}`) - if (commit) { - console.log('Database operations:') - console.log(` Errors: ${stats.db.errors}`) - console.log(` Matched: ${stats.db.matched}`) - console.log(` Updated: ${stats.db.updated}`) - console.log(` Update attempted: ${stats.db.updateAttempted}`) - } +function generateStats() { + return `Stats: + Total rows: ${stats.totalRows} + Processed rows: ${stats.processedRows} + Skipped (no subscription ID): ${stats.subscriptionIDMissing} + Used V1 ID: ${stats.usedV1ID} + Used Salesforce ID: ${stats.usedSalesforceID}${ + commit + ? ` +Database operations: + Errors: ${stats.db.errors} + Matched: ${stats.db.matched} + Updated: ${stats.db.updated} + Update attempted: ${stats.db.updateAttempted}` + : '' + }` } function pickRelevantColumns(row) { @@ -161,7 +164,7 @@ async function processRows(rows) { } } -async function main() { +async function main(trackProgress) { await Stream.pipeline( fs.createReadStream(filename), parse({ @@ -191,6 +194,7 @@ async function main() { }), processRows ) + await trackProgress(generateStats()) } if (!commit) { @@ -199,6 +203,5 @@ if (!commit) { console.log('Committing changes to the database') } -await main() -showStats() +await scriptRunner(main) process.exit() diff --git a/services/web/scripts/add_user_count_to_csv.mjs b/services/web/scripts/add_user_count_to_csv.mjs index 04709df5b2..5447ec9c73 100644 --- a/services/web/scripts/add_user_count_to_csv.mjs +++ b/services/web/scripts/add_user_count_to_csv.mjs @@ -9,6 +9,7 @@ import minimist from 'minimist' import UserGetter from '../app/src/Features/User/UserGetter.js' import { db } from '../app/src/infrastructure/mongodb.js' import _ from 'lodash' +import { scriptRunner } from './lib/ScriptRunner.mjs' const argv = minimist(process.argv.slice(2), { string: ['domain', 'output'], @@ -86,7 +87,7 @@ async function getUsersByHostnameWithSubdomain(domain, projection) { } try { - await main() + await scriptRunner(main) console.log('Done') process.exit(0) } catch (error) { diff --git a/services/web/scripts/analytics/backfill_recurly_to_subscription_mapping.mjs b/services/web/scripts/analytics/backfill_recurly_to_subscription_mapping.mjs index 16f5abe0ad..9ad583844c 100644 --- a/services/web/scripts/analytics/backfill_recurly_to_subscription_mapping.mjs +++ b/services/web/scripts/analytics/backfill_recurly_to_subscription_mapping.mjs @@ -19,6 +19,7 @@ import AccountMappingHelper from '../../app/src/Features/Analytics/AccountMappin import { registerAccountMapping } from '../../app/src/Features/Analytics/AnalyticsManager.js' import { triggerGracefulShutdown } from '../../app/src/infrastructure/GracefulShutdown.js' import Validation from '../../app/src/infrastructure/Validation.js' +import { scriptRunner } from '../lib/ScriptRunner.mjs' const paramsSchema = Validation.Joi.object({ endDate: Validation.Joi.string().isoDate(), @@ -59,7 +60,7 @@ function registerMapping(subscription) { } } -async function main() { +async function main(trackProgress) { const additionalBatchedUpdateOptions = {} if (endDate) { @@ -83,6 +84,7 @@ async function main() { { verboseLogging: verbose, ...additionalBatchedUpdateOptions, + trackProgress, } ) @@ -109,7 +111,7 @@ if (error) { triggerGracefulShutdown(done => done(1)) } else { logger.info({ verbose, commit, endDate }, commit ? 'COMMITTING' : 'DRY RUN') - await main() + await scriptRunner(main) triggerGracefulShutdown({ close(done) { diff --git a/services/web/scripts/analytics/sync_group_subscription_memberships.mjs b/services/web/scripts/analytics/sync_group_subscription_memberships.mjs index 5bca95eef4..f687f43460 100644 --- a/services/web/scripts/analytics/sync_group_subscription_memberships.mjs +++ b/services/web/scripts/analytics/sync_group_subscription_memberships.mjs @@ -5,6 +5,7 @@ import { DeletedSubscription } from '../../app/src/models/DeletedSubscription.js import minimist from 'minimist' import _ from 'lodash' import mongodb from 'mongodb-legacy' +import { scriptRunner } from '../lib/ScriptRunner.mjs' const { ObjectId } = mongodb @@ -272,7 +273,7 @@ const setup = () => { setup() try { - await main() + await scriptRunner(main) console.error('Done.') process.exit(0) } catch (error) { diff --git a/services/web/scripts/back_fill_doc_name_for_deleted_docs.mjs b/services/web/scripts/back_fill_doc_name_for_deleted_docs.mjs index d22bb3ea7e..b311176184 100644 --- a/services/web/scripts/back_fill_doc_name_for_deleted_docs.mjs +++ b/services/web/scripts/back_fill_doc_name_for_deleted_docs.mjs @@ -3,10 +3,11 @@ import { promiseMapWithLimit, promisify } from '@overleaf/promise-utils' import { db } from '../app/src/infrastructure/mongodb.js' import { fileURLToPath } from 'node:url' import _ from 'lodash' +import { scriptRunner } from './lib/ScriptRunner.mjs' const sleep = promisify(setTimeout) -async function main(options) { +async function main(options, trackProgress) { if (!options) { options = {} } @@ -28,7 +29,9 @@ async function main(options) { async projects => { await processBatch(projects, options) }, - { _id: 1, deletedDocs: 1 } + { _id: 1, deletedDocs: 1 }, + undefined, + { trackProgress } ) } @@ -83,7 +86,9 @@ export default main if (fileURLToPath(import.meta.url) === process.argv[1]) { try { - await main() + await scriptRunner( + async trackProgress => await main(undefined, trackProgress) + ) process.exit(0) } catch (error) { console.error({ error }) diff --git a/services/web/scripts/back_fill_dummy_doc_meta.mjs b/services/web/scripts/back_fill_dummy_doc_meta.mjs index 0fccdc46a7..e0684ea0be 100644 --- a/services/web/scripts/back_fill_dummy_doc_meta.mjs +++ b/services/web/scripts/back_fill_dummy_doc_meta.mjs @@ -7,6 +7,7 @@ import { import _ from 'lodash' import LRUCache from 'lru-cache' import { fileURLToPath } from 'node:url' +import { scriptRunner } from './lib/ScriptRunner.mjs' const { ObjectId } = mongodb const sleep = promisify(setTimeout) @@ -151,7 +152,7 @@ export default main if (fileURLToPath(import.meta.url) === process.argv[1]) { try { - await main() + await scriptRunner(async () => await main()) console.error('Done.') process.exit(0) } catch (error) { diff --git a/services/web/scripts/back_fill_staff_access.mjs b/services/web/scripts/back_fill_staff_access.mjs index c26958e8d9..76d17fba83 100644 --- a/services/web/scripts/back_fill_staff_access.mjs +++ b/services/web/scripts/back_fill_staff_access.mjs @@ -3,6 +3,7 @@ import { READ_PREFERENCE_SECONDARY, } from '../app/src/infrastructure/mongodb.js' import UserSessionsManager from '../app/src/Features/User/UserSessionsManager.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' const COMMIT = process.argv.includes('--commit') const KEEP_SESSIONS = process.argv.includes('--keep-sessions') @@ -84,7 +85,7 @@ async function main() { } try { - await main() + await scriptRunner(main) console.error('Done.') process.exit(0) } catch (error) { diff --git a/services/web/scripts/back_fill_warning_user_personal_and_group_subscription.mjs b/services/web/scripts/back_fill_warning_user_personal_and_group_subscription.mjs index efa5a79197..f9b5388e45 100644 --- a/services/web/scripts/back_fill_warning_user_personal_and_group_subscription.mjs +++ b/services/web/scripts/back_fill_warning_user_personal_and_group_subscription.mjs @@ -1,6 +1,7 @@ import NotificationsBuilder from '../app/src/Features/Notifications/NotificationsBuilder.js' import { db } from '../app/src/infrastructure/mongodb.js' import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' const DRY_RUN = !process.argv.includes('--dry-run=false') @@ -55,14 +56,23 @@ async function processBatch(groupSubscriptionsBatch) { } } -async function main() { - await batchedUpdate(db.subscriptions, { groupPlan: true }, processBatch, { - member_ids: 1, - }) +async function main(trackProgress) { + await batchedUpdate( + db.subscriptions, + { groupPlan: true }, + processBatch, + { + member_ids: 1, + }, + undefined, + { + trackProgress, + } + ) } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/check_docs.mjs b/services/web/scripts/check_docs.mjs index d6fccd5db3..699738f75c 100644 --- a/services/web/scripts/check_docs.mjs +++ b/services/web/scripts/check_docs.mjs @@ -9,6 +9,7 @@ import { } from '../app/src/infrastructure/mongodb.js' import DocstoreManager from '../app/src/Features/Docstore/DocstoreManager.js' import { NotFoundError } from '../app/src/Features/Errors/Errors.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' const OPTS = parseArgs() @@ -213,7 +214,7 @@ function docsHaveTrackedChanges(docs) { } try { - await main() + await scriptRunner(main) process.exit(0) } catch (err) { console.error(err) diff --git a/services/web/scripts/check_institution_users.mjs b/services/web/scripts/check_institution_users.mjs index e89c0d114b..5cb4fb7973 100644 --- a/services/web/scripts/check_institution_users.mjs +++ b/services/web/scripts/check_institution_users.mjs @@ -1,5 +1,6 @@ import InstitutionsManager from '../app/src/Features/Institutions/InstitutionsManager.js' import { ensureRunningOnMongoSecondaryWithTimeout } from './helpers/env_variable_helper.mjs' +import { scriptRunner } from './lib/ScriptRunner.mjs' ensureRunningOnMongoSecondaryWithTimeout(300000) @@ -18,7 +19,7 @@ async function main() { } try { - await main() + await scriptRunner(main) } catch (error) { console.error(error) process.exit(1) diff --git a/services/web/scripts/clear_admin_sessions.mjs b/services/web/scripts/clear_admin_sessions.mjs index 3d5623c864..c3f5b28c4b 100644 --- a/services/web/scripts/clear_admin_sessions.mjs +++ b/services/web/scripts/clear_admin_sessions.mjs @@ -3,6 +3,7 @@ import { READ_PREFERENCE_SECONDARY, } from '../app/src/infrastructure/mongodb.js' import UserSessionsManager from '../app/src/Features/User/UserSessionsManager.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' const COMMIT = process.argv.includes('--commit') const LOG_SESSIONS = !process.argv.includes('--log-sessions=false') @@ -58,7 +59,7 @@ async function main() { } try { - await main() + await scriptRunner(main) console.error('Done.') process.exit(0) } catch (error) { diff --git a/services/web/scripts/clear_institution_notifications.mjs b/services/web/scripts/clear_institution_notifications.mjs index b97c20d5ff..29a77de5db 100644 --- a/services/web/scripts/clear_institution_notifications.mjs +++ b/services/web/scripts/clear_institution_notifications.mjs @@ -1,6 +1,7 @@ import { promisify } from 'node:util' import InstitutionsManager from '../app/src/Features/Institutions/InstitutionsManager.js' import { fileURLToPath } from 'node:url' +import { scriptRunner } from './lib/ScriptRunner.mjs' const sleep = promisify(setTimeout) async function main() { @@ -39,7 +40,7 @@ async function main() { if (fileURLToPath(import.meta.url) === process.argv[1]) { try { - await main() + await scriptRunner(main) console.log('Done.') process.exit(0) } catch (error) { diff --git a/services/web/scripts/clear_project_tokens.mjs b/services/web/scripts/clear_project_tokens.mjs index ab3d3c5a84..798e67aded 100644 --- a/services/web/scripts/clear_project_tokens.mjs +++ b/services/web/scripts/clear_project_tokens.mjs @@ -1,4 +1,5 @@ import ProjectDetailsHandler from '../app/src/Features/Project/ProjectDetailsHandler.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' const projectId = process.argv[2] if (!/^(?=[a-f\d]{24}$)(\d+[a-f]|[a-f]+\d)/.test(projectId)) { @@ -21,7 +22,7 @@ function main() { } try { - await main() + await scriptRunner(main) } catch (error) { console.error(error) process.exit(1) diff --git a/services/web/scripts/clear_sessions_2fa.mjs b/services/web/scripts/clear_sessions_2fa.mjs index e6058d357c..d6557faa62 100644 --- a/services/web/scripts/clear_sessions_2fa.mjs +++ b/services/web/scripts/clear_sessions_2fa.mjs @@ -1,6 +1,7 @@ import { promisify, promiseMapWithLimit } from '@overleaf/promise-utils' import UserSessionsRedis from '../app/src/Features/User/UserSessionsRedis.js' import minimist from 'minimist' +import { scriptRunner } from './lib/ScriptRunner.mjs' const rClient = UserSessionsRedis.client() @@ -76,7 +77,7 @@ async function main() { } try { - await main() + await scriptRunner(main) } catch (error) { console.error(error) process.exit(1) diff --git a/services/web/scripts/convert_doc_to_file.mjs b/services/web/scripts/convert_doc_to_file.mjs index db4cb8b309..a1d401dc23 100644 --- a/services/web/scripts/convert_doc_to_file.mjs +++ b/services/web/scripts/convert_doc_to_file.mjs @@ -2,6 +2,7 @@ import minimist from 'minimist' import { ObjectId } from '../app/src/infrastructure/mongodb.js' import ProjectEntityUpdateHandler from '../app/src/Features/Project/ProjectEntityUpdateHandler.js' import Errors from '../app/src/Features/Errors/Errors.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' async function main() { const argv = minimist(process.argv.slice(2)) @@ -35,7 +36,7 @@ async function main() { } try { - await main() + await scriptRunner(main) console.log('Done.') process.exit(0) } catch (error) { diff --git a/services/web/scripts/count_encrypted_access_tokens.mjs b/services/web/scripts/count_encrypted_access_tokens.mjs index 003179f3eb..e989d49d15 100644 --- a/services/web/scripts/count_encrypted_access_tokens.mjs +++ b/services/web/scripts/count_encrypted_access_tokens.mjs @@ -5,6 +5,7 @@ import { import _ from 'lodash' import { formatTokenUsageStats } from '@overleaf/access-token-encryptor/scripts/helpers/format-usage-stats.js' import { ensureMongoTimeout } from './helpers/env_variable_helper.mjs' +import { scriptRunner } from './lib/ScriptRunner.mjs' if (!process.env.MONGO_SOCKET_TIMEOUT) { const TEN_MINUTES = 1000 * 60 * 10 @@ -65,7 +66,7 @@ async function main() { } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/count_image_files.mjs b/services/web/scripts/count_image_files.mjs index 17ad4b84ab..33a9bcc1fd 100644 --- a/services/web/scripts/count_image_files.mjs +++ b/services/web/scripts/count_image_files.mjs @@ -3,6 +3,7 @@ import { READ_PREFERENCE_SECONDARY, } from '../app/src/infrastructure/mongodb.js' import { extname } from 'node:path' +import { scriptRunner } from './lib/ScriptRunner.mjs' const FILE_TYPES = [ '.jpg', @@ -71,7 +72,7 @@ function countFiles(folder, result) { } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/delete_dangling_file_refs.mjs b/services/web/scripts/delete_dangling_file_refs.mjs index 39383bed79..50dde9e174 100644 --- a/services/web/scripts/delete_dangling_file_refs.mjs +++ b/services/web/scripts/delete_dangling_file_refs.mjs @@ -10,6 +10,7 @@ import Errors from '../app/src/Features/Errors/Errors.js' import FileStoreHandler from '../app/src/Features/FileStore/FileStoreHandler.js' import ProjectEntityMongoUpdateHandler from '../app/src/Features/Project/ProjectEntityMongoUpdateHandler.js' import { iterablePaths } from '../app/src/Features/Project/IterablePath.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' const { ObjectId } = mongodb @@ -123,7 +124,7 @@ async function deleteFile(projectId, fileId) { } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error({ error }) diff --git a/services/web/scripts/delete_orphaned_doc_comment_ranges.mjs b/services/web/scripts/delete_orphaned_doc_comment_ranges.mjs index 5b9a39714c..c99bfe5b9d 100644 --- a/services/web/scripts/delete_orphaned_doc_comment_ranges.mjs +++ b/services/web/scripts/delete_orphaned_doc_comment_ranges.mjs @@ -3,6 +3,7 @@ import ChatApiHandler from '../app/src/Features/Chat/ChatApiHandler.js' import DocstoreManager from '../app/src/Features/Docstore/DocstoreManager.js' import DocumentUpdaterHandler from '../app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js' import { promiseMapWithLimit } from '@overleaf/promise-utils' +import { scriptRunner } from './lib/ScriptRunner.mjs' const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10 @@ -43,7 +44,7 @@ async function main() { } try { - await main() + await scriptRunner(main) console.log('Done.') process.exit(0) } catch (error) { diff --git a/services/web/scripts/delete_orphaned_docs_online_check.mjs b/services/web/scripts/delete_orphaned_docs_online_check.mjs index 2affae4cf9..5ab8e17d45 100644 --- a/services/web/scripts/delete_orphaned_docs_online_check.mjs +++ b/services/web/scripts/delete_orphaned_docs_online_check.mjs @@ -8,6 +8,7 @@ import { } from '../app/src/infrastructure/mongodb.js' import { promiseMapWithLimit } from '@overleaf/promise-utils' import DeleteOrphanedDataHelper from './delete_orphaned_data_helper.mjs' +import { scriptRunner } from './lib/ScriptRunner.mjs' const { ObjectId } = mongodb const sleep = promisify(setTimeout) @@ -170,7 +171,7 @@ async function letUserDoubleCheckInputs() { } try { - await main() + await scriptRunner(main) console.error('Done.') process.exit(0) } catch (error) { diff --git a/services/web/scripts/disconnect_all_users.mjs b/services/web/scripts/disconnect_all_users.mjs index 41736ac69f..f8e43ad849 100644 --- a/services/web/scripts/disconnect_all_users.mjs +++ b/services/web/scripts/disconnect_all_users.mjs @@ -3,6 +3,7 @@ import Settings from '@overleaf/settings' import AdminController from '../app/src/Features/ServerAdmin/AdminController.js' import minimist from 'minimist' import { fileURLToPath } from 'node:url' +import { scriptRunner } from './lib/ScriptRunner.mjs' const args = minimist(process.argv.slice(2), { string: ['confirm-site-url', 'delay-in-seconds'], @@ -60,7 +61,7 @@ async function main() { if (fileURLToPath(import.meta.url) === process.argv[1]) { try { - await main() + await scriptRunner(main) console.error('Done.') process.exit(0) } catch (error) { diff --git a/services/web/scripts/e2e_test_setup.mjs b/services/web/scripts/e2e_test_setup.mjs index 7f0e8e3fef..4d529ccfe9 100644 --- a/services/web/scripts/e2e_test_setup.mjs +++ b/services/web/scripts/e2e_test_setup.mjs @@ -9,6 +9,7 @@ import ProjectDeleter from '../app/src/Features/Project/ProjectDeleter.js' import SplitTestManager from '../app/src/Features/SplitTests/SplitTestManager.js' import UserDeleter from '../app/src/Features/User/UserDeleter.js' import UserRegistrationHandler from '../app/src/Features/User/UserRegistrationHandler.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' const MONOREPO = Path.dirname( Path.dirname(Path.dirname(Path.dirname(fileURLToPath(import.meta.url)))) @@ -158,7 +159,7 @@ async function main() { await provisionSplitTests() } -await main() +await scriptRunner(main) await GracefulShutdown.gracefulShutdown( { close(cb) { diff --git a/services/web/scripts/find_malformed_filetrees.mjs b/services/web/scripts/find_malformed_filetrees.mjs index 25b4a77a59..2614c7d622 100644 --- a/services/web/scripts/find_malformed_filetrees.mjs +++ b/services/web/scripts/find_malformed_filetrees.mjs @@ -1,6 +1,7 @@ // @ts-check import { db, ObjectId } from '../app/src/infrastructure/mongodb.js' import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' /** * @typedef {Object} Doc @@ -30,7 +31,12 @@ import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' * @property {Array} rootFolder */ -async function main() { +/** + * @param {(progress: string) => Promise} trackProgress + * @returns {Promise} + * @async + */ +async function main(trackProgress) { let projectsProcessed = 0 await batchedUpdate( db.projects, @@ -59,7 +65,9 @@ async function main() { } } }, - { _id: 1, rootFolder: 1 } + { _id: 1, rootFolder: 1 }, + undefined, + { trackProgress } ) } @@ -161,7 +169,7 @@ function* findBadPaths(folder) { } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/fix_group_invite_emails_to_lowercase.mjs b/services/web/scripts/fix_group_invite_emails_to_lowercase.mjs index 83024be6e5..a9bc0636d7 100644 --- a/services/web/scripts/fix_group_invite_emails_to_lowercase.mjs +++ b/services/web/scripts/fix_group_invite_emails_to_lowercase.mjs @@ -1,5 +1,6 @@ import { db } from '../app/src/infrastructure/mongodb.js' import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' const DRY_RUN = process.env.DRY_RUN !== 'false' @@ -44,7 +45,7 @@ async function processBatch(subscriptions) { } } -async function main() { +async function main(trackProgress) { const projection = { _id: 1, teamInvites: 1, @@ -54,11 +55,18 @@ async function main() { $exists: true, }, } - await batchedUpdate(db.subscriptions, query, processBatch, projection) + await batchedUpdate( + db.subscriptions, + query, + processBatch, + projection, + undefined, + { trackProgress } + ) } try { - await main() + await scriptRunner(main) console.error('Done.') process.exit(0) } catch (error) { diff --git a/services/web/scripts/fix_malformed_filetree.mjs b/services/web/scripts/fix_malformed_filetree.mjs index 2358bed850..ff838d15c6 100644 --- a/services/web/scripts/fix_malformed_filetree.mjs +++ b/services/web/scripts/fix_malformed_filetree.mjs @@ -15,6 +15,7 @@ import minimist from 'minimist' import readline from 'node:readline' import fs from 'node:fs' import logger from '@overleaf/logger' +import { scriptRunner } from './lib/ScriptRunner.mjs' const { ObjectId } = mongodb const lastUpdated = new Date() @@ -287,7 +288,7 @@ function findUniqueName(existingFilenames) { try { try { - await main() + await scriptRunner(main) } finally { logStats() } diff --git a/services/web/scripts/fix_oversized_docs.mjs b/services/web/scripts/fix_oversized_docs.mjs index 9f2e250b92..1fe7b8337a 100644 --- a/services/web/scripts/fix_oversized_docs.mjs +++ b/services/web/scripts/fix_oversized_docs.mjs @@ -8,6 +8,7 @@ import ProjectEntityMongoUpdateHandler from '../app/src/Features/Project/Project import ProjectLocator from '../app/src/Features/Project/ProjectLocator.js' import RedisWrapper from '@overleaf/redis-wrapper' import Settings from '@overleaf/settings' +import { scriptRunner } from './lib/ScriptRunner.mjs' const opts = parseArgs() const redis = RedisWrapper.createClient(Settings.redis.web) @@ -155,7 +156,7 @@ async function deleteDocFromRedis(projectId, docId) { } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/force_doc_flush.mjs b/services/web/scripts/force_doc_flush.mjs index 399816e859..b6791710b0 100644 --- a/services/web/scripts/force_doc_flush.mjs +++ b/services/web/scripts/force_doc_flush.mjs @@ -1,6 +1,7 @@ import mongodb from 'mongodb-legacy' import { db } from '../app/src/infrastructure/mongodb.js' import DocumentUpdaterHandler from '../app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' const { ObjectId } = mongodb const PROJECT_ID = process.env.PROJECT_ID @@ -67,7 +68,7 @@ function getDocument() { } try { - await main() + await scriptRunner(main) console.error('Done.') process.exit(0) } catch (error) { diff --git a/services/web/scripts/history/clean_sl_history_data.mjs b/services/web/scripts/history/clean_sl_history_data.mjs index 0f8e93661a..8eb541e078 100644 --- a/services/web/scripts/history/clean_sl_history_data.mjs +++ b/services/web/scripts/history/clean_sl_history_data.mjs @@ -1,5 +1,6 @@ import { db } from '../../app/src/infrastructure/mongodb.js' import { ensureMongoTimeout } from '../helpers/env_variable_helper.mjs' +import { scriptRunner } from '../lib/ScriptRunner.mjs' // Ensure default mongo query timeout has been increased 1h if (!process.env.MONGO_SOCKET_TIMEOUT) { ensureMongoTimeout(360000) @@ -66,7 +67,7 @@ async function gracefullyDropCollection(collection) { } try { - await main() + await scriptRunner(main) } catch (err) { console.error(err) process.exit(1) diff --git a/services/web/scripts/history/migrate_ranges_support.mjs b/services/web/scripts/history/migrate_ranges_support.mjs index f62628f9ad..d0bde55a0b 100644 --- a/services/web/scripts/history/migrate_ranges_support.mjs +++ b/services/web/scripts/history/migrate_ranges_support.mjs @@ -1,5 +1,6 @@ import HistoryRangesSupportMigration from '../../app/src/Features/History/HistoryRangesSupportMigration.mjs' import minimist from 'minimist' +import { scriptRunner } from '../lib/ScriptRunner.mjs' async function main() { const { @@ -111,7 +112,7 @@ function arrayOpt(value) { } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/learn/checkSanitize/index.mjs b/services/web/scripts/learn/checkSanitize/index.mjs index 1c608cb2fe..55ad2caa38 100644 --- a/services/web/scripts/learn/checkSanitize/index.mjs +++ b/services/web/scripts/learn/checkSanitize/index.mjs @@ -1,6 +1,7 @@ import checkSanitizeOptions from './checkSanitizeOptions.mjs' import Scrape from './scrape.mjs' import { fileURLToPath } from 'node:url' +import { scriptRunner } from '../../lib/ScriptRunner.mjs' const { getAllPagesAndCache, scrapeAndCachePage } = Scrape @@ -32,7 +33,7 @@ async function main() { if (fileURLToPath(import.meta.url) === process.argv[1]) { try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/lib/ScriptRunner.mjs b/services/web/scripts/lib/ScriptRunner.mjs index 1708fa9310..478a71815b 100644 --- a/services/web/scripts/lib/ScriptRunner.mjs +++ b/services/web/scripts/lib/ScriptRunner.mjs @@ -1,23 +1,28 @@ import { ScriptLog } from '../../app/src/models/ScriptLog.mjs' import Settings from '@overleaf/settings' +const UNKNOWN = 'unknown' + async function beforeScriptExecution(canonicalName, vars, scriptPath) { let log = new ScriptLog({ canonicalName, filePathAtVersion: scriptPath, - podName: process.env.OL_POD_NAME, - username: process.env.OL_USERNAME, - imageVersion: process.env.OL_IMAGE_VERSION, + podName: process.env.OL_POD_NAME ?? UNKNOWN, + username: process.env.OL_USERNAME ?? UNKNOWN, + imageVersion: process.env.OL_IMAGE_VERSION ?? UNKNOWN, vars, }) log = await log.save() - console.log( - '\n==================================' + - '\n✨ Your script is running!' + - '\n📊 Track progress at:' + - `\n${Settings.adminUrl}/admin/script-log/${log._id}` + - '\n==================================\n' - ) + // Print Script Log link if ran by a user + if (process.env.OL_USERNAME) { + console.log( + '\n==================================' + + '\n✨ Your script is running!' + + '\n📊 Track progress at:' + + `\n${Settings.adminUrl}/admin/script-log/${log._id}` + + '\n==================================\n' + ) + } return log._id } diff --git a/services/web/scripts/lowercase_institution_user_ids.mjs b/services/web/scripts/lowercase_institution_user_ids.mjs index e670d10615..e72bff87ef 100644 --- a/services/web/scripts/lowercase_institution_user_ids.mjs +++ b/services/web/scripts/lowercase_institution_user_ids.mjs @@ -2,6 +2,7 @@ import { db } from '../app/src/infrastructure/mongodb.js' import minimist from 'minimist' import UserGetter from '../app/src/Features/User/UserGetter.js' import fs from 'node:fs' +import { scriptRunner } from './lib/ScriptRunner.mjs' function usage() { console.log( @@ -112,7 +113,7 @@ async function main() { } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/merge_group_subscription_members.mjs b/services/web/scripts/merge_group_subscription_members.mjs index 0d4d6acb6c..d8201c95ea 100644 --- a/services/web/scripts/merge_group_subscription_members.mjs +++ b/services/web/scripts/merge_group_subscription_members.mjs @@ -10,6 +10,7 @@ import { db, ObjectId } from '../app/src/infrastructure/mongodb.js' import SubscriptionUpdater from '../app/src/Features/Subscription/SubscriptionUpdater.js' import minimist from 'minimist' +import { scriptRunner } from './lib/ScriptRunner.mjs' const argv = minimist(process.argv.slice(2), { string: ['target', 'source'], boolean: ['commit'], @@ -93,7 +94,7 @@ async function main() { } try { - await main() + await scriptRunner(main) console.error('Done.') process.exit(0) } catch (error) { diff --git a/services/web/scripts/migrate_audit_logs.mjs b/services/web/scripts/migrate_audit_logs.mjs index 558c20cd06..9591275fca 100644 --- a/services/web/scripts/migrate_audit_logs.mjs +++ b/services/web/scripts/migrate_audit_logs.mjs @@ -3,10 +3,11 @@ import { promiseMapWithLimit, promisify } from '@overleaf/promise-utils' import { db, ObjectId } from '../app/src/infrastructure/mongodb.js' import _ from 'lodash' import { fileURLToPath } from 'node:url' +import { scriptRunner } from './lib/ScriptRunner.mjs' const sleep = promisify(setTimeout) -async function main(options) { +async function main(options, trackProgress) { if (!options) { options = {} } @@ -54,7 +55,9 @@ async function main(options) { async users => { await processUsersBatch(users, options) }, - { _id: 1, auditLog: 1 } + { _id: 1, auditLog: 1 }, + undefined, + { trackProgress } ) } @@ -67,7 +70,9 @@ async function main(options) { async projects => { await processProjectsBatch(projects, options) }, - { _id: 1, auditLog: 1 } + { _id: 1, auditLog: 1 }, + undefined, + { trackProgress } ) } } @@ -152,7 +157,9 @@ export default main if (fileURLToPath(import.meta.url) === process.argv[1]) { try { - await main() + await scriptRunner( + async trackProgress => await main(undefined, trackProgress) + ) console.log('Done.') process.exit(0) } catch (error) { diff --git a/services/web/scripts/oauth/backfill_hashed_secrets.mjs b/services/web/scripts/oauth/backfill_hashed_secrets.mjs index e3352ea932..6e0f8c66e9 100644 --- a/services/web/scripts/oauth/backfill_hashed_secrets.mjs +++ b/services/web/scripts/oauth/backfill_hashed_secrets.mjs @@ -3,6 +3,7 @@ import { READ_PREFERENCE_SECONDARY, } from '../../app/src/infrastructure/mongodb.js' import { hashSecret } from '../../modules/oauth2-server/app/src/SecretsHelper.js' +import { scriptRunner } from '../lib/ScriptRunner.mjs' async function main() { console.log('Hashing client secrets...') @@ -35,7 +36,7 @@ async function hashSecrets(collection, field) { } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/oauth/create_token.mjs b/services/web/scripts/oauth/create_token.mjs index cf833e1be2..4d43981a19 100644 --- a/services/web/scripts/oauth/create_token.mjs +++ b/services/web/scripts/oauth/create_token.mjs @@ -1,6 +1,7 @@ import minimist from 'minimist' import { db } from '../../app/src/infrastructure/mongodb.js' import { hashSecret } from '../../modules/oauth2-server/app/src/SecretsHelper.js' +import { scriptRunner } from '../lib/ScriptRunner.mjs' async function main() { const opts = parseArgs() @@ -88,7 +89,7 @@ Options: } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/oauth/register_client.mjs b/services/web/scripts/oauth/register_client.mjs index 8ca97f7321..65248f3464 100644 --- a/services/web/scripts/oauth/register_client.mjs +++ b/services/web/scripts/oauth/register_client.mjs @@ -2,6 +2,7 @@ import minimist from 'minimist' import mongodb from 'mongodb-legacy' import { db } from '../../app/src/infrastructure/mongodb.js' import { hashSecret } from '../../modules/oauth2-server/app/src/SecretsHelper.js' +import { scriptRunner } from '../lib/ScriptRunner.mjs' const { ObjectId } = mongodb @@ -142,7 +143,7 @@ function toArray(value) { } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/oauth/remove_client.mjs b/services/web/scripts/oauth/remove_client.mjs index 700c19ac70..24080491b2 100644 --- a/services/web/scripts/oauth/remove_client.mjs +++ b/services/web/scripts/oauth/remove_client.mjs @@ -3,6 +3,7 @@ import { db, READ_PREFERENCE_SECONDARY, } from '../../app/src/infrastructure/mongodb.js' +import { scriptRunner } from '../lib/ScriptRunner.mjs' async function main() { const opts = parseArgs() @@ -113,7 +114,7 @@ Options: } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/recover_docs_from_redis.mjs b/services/web/scripts/recover_docs_from_redis.mjs index ad69b5469f..85709b471b 100644 --- a/services/web/scripts/recover_docs_from_redis.mjs +++ b/services/web/scripts/recover_docs_from_redis.mjs @@ -8,6 +8,7 @@ import ProjectEntityRestoreHandler from '../app/src/Features/Project/ProjectEnti import RedisWrapper from '@overleaf/redis-wrapper' import Settings from '@overleaf/settings' import logger from '@overleaf/logger' +import { scriptRunner } from './lib/ScriptRunner.mjs' const opts = parseArgs() const redis = RedisWrapper.createClient(Settings.redis.web) @@ -173,7 +174,7 @@ async function deleteDocFromRedis(projectId, docId) { } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/recurly/change_prices_at_renewal.mjs b/services/web/scripts/recurly/change_prices_at_renewal.mjs index ad891f1b9d..53016acc7d 100644 --- a/services/web/scripts/recurly/change_prices_at_renewal.mjs +++ b/services/web/scripts/recurly/change_prices_at_renewal.mjs @@ -4,6 +4,7 @@ import * as csv from 'csv' import minimist from 'minimist' import recurly from 'recurly' import Settings from '@overleaf/settings' +import { scriptRunner } from '../lib/ScriptRunner.mjs' const recurlyClient = new recurly.Client(Settings.apis.recurly.apiKey) @@ -223,7 +224,7 @@ class ReportError extends Error { } try { - await main() + await scriptRunner(main) } catch (error) { console.error(error) process.exit(1) diff --git a/services/web/scripts/recurly/collect_paypal_past_due_invoice.mjs b/services/web/scripts/recurly/collect_paypal_past_due_invoice.mjs index b735ea17ed..2bf827bae8 100644 --- a/services/web/scripts/recurly/collect_paypal_past_due_invoice.mjs +++ b/services/web/scripts/recurly/collect_paypal_past_due_invoice.mjs @@ -2,6 +2,7 @@ import RecurlyWrapper from '../../app/src/Features/Subscription/RecurlyWrapper.j import minimist from 'minimist' import logger from '@overleaf/logger' import { fileURLToPath } from 'node:url' +import { scriptRunner } from '../lib/ScriptRunner.mjs' const waitMs = fileURLToPath(import.meta.url) === process.argv[1] @@ -119,7 +120,7 @@ const main = async () => { if (fileURLToPath(import.meta.url) === process.argv[1]) { try { - await main() + await scriptRunner(main) logger.info('Done.') process.exit(0) } catch (error) { diff --git a/services/web/scripts/recurly/generate_addon_prices.mjs b/services/web/scripts/recurly/generate_addon_prices.mjs index 37378e6baf..6ae9d1846d 100644 --- a/services/web/scripts/recurly/generate_addon_prices.mjs +++ b/services/web/scripts/recurly/generate_addon_prices.mjs @@ -1,6 +1,7 @@ // @ts-check import settings from '@overleaf/settings' import recurly from 'recurly' +import { scriptRunner } from '../lib/ScriptRunner.mjs' const ADD_ON_CODE = process.argv[2] @@ -54,4 +55,4 @@ async function getPlan(planCode) { return await recurlyClient.getPlan(`code-${planCode}`) } -await main() +await scriptRunner(main) diff --git a/services/web/scripts/recurly/get_manually_billed_users_details.mjs b/services/web/scripts/recurly/get_manually_billed_users_details.mjs index dafec9f8d6..0f7711652b 100644 --- a/services/web/scripts/recurly/get_manually_billed_users_details.mjs +++ b/services/web/scripts/recurly/get_manually_billed_users_details.mjs @@ -5,6 +5,7 @@ import { setTimeout } from 'node:timers/promises' import minimist from 'minimist' import * as csv from 'csv' import Stream from 'node:stream/promises' +import { scriptRunner } from '../lib/ScriptRunner.mjs' const recurlyApiKey = Settings.apis.recurly.apiKey if (!recurlyApiKey) { @@ -95,7 +96,7 @@ async function main() { } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/recurly/get_recurly_group_prices.mjs b/services/web/scripts/recurly/get_recurly_group_prices.mjs index e00a1e4c66..db734ead31 100644 --- a/services/web/scripts/recurly/get_recurly_group_prices.mjs +++ b/services/web/scripts/recurly/get_recurly_group_prices.mjs @@ -7,6 +7,7 @@ import recurly from 'recurly' import Settings from '@overleaf/settings' +import { scriptRunner } from '../lib/ScriptRunner.mjs' const recurlySettings = Settings.apis.recurly const recurlyApiKey = recurlySettings ? recurlySettings.apiKey : undefined @@ -39,7 +40,7 @@ async function main() { } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error({ error }) diff --git a/services/web/scripts/recurly/update_terms_and_conditions_for_manually_billed_users.mjs b/services/web/scripts/recurly/update_terms_and_conditions_for_manually_billed_users.mjs index f47ee6a2f0..ae8f19423b 100644 --- a/services/web/scripts/recurly/update_terms_and_conditions_for_manually_billed_users.mjs +++ b/services/web/scripts/recurly/update_terms_and_conditions_for_manually_billed_users.mjs @@ -4,6 +4,7 @@ import fs from 'node:fs' import minimist from 'minimist' import * as csv from 'csv' import { setTimeout } from 'node:timers/promises' +import { scriptRunner } from '../lib/ScriptRunner.mjs' const recurlyApiKey = Settings.apis.recurly.apiKey if (!recurlyApiKey) { @@ -95,7 +96,7 @@ async function main() { } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/refresh_institution_users.mjs b/services/web/scripts/refresh_institution_users.mjs index 6edb523108..e67bd3c346 100644 --- a/services/web/scripts/refresh_institution_users.mjs +++ b/services/web/scripts/refresh_institution_users.mjs @@ -1,5 +1,6 @@ import minimist from 'minimist' import InstitutionsManager from '../app/src/Features/Institutions/InstitutionsManager.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' const institutionId = parseInt(process.argv[2]) if (isNaN(institutionId)) throw new Error('No institution id') @@ -31,7 +32,7 @@ function main() { } try { - await main() + await scriptRunner(main) } catch (error) { console.error(error) process.exit(1) diff --git a/services/web/scripts/remove_feature_from_all_users.mjs b/services/web/scripts/remove_feature_from_all_users.mjs index fe53bda39a..f138501f0e 100644 --- a/services/web/scripts/remove_feature_from_all_users.mjs +++ b/services/web/scripts/remove_feature_from_all_users.mjs @@ -3,6 +3,7 @@ import { READ_PREFERENCE_SECONDARY, } from '../app/src/infrastructure/mongodb.js' import parseArgs from 'minimist' +import { scriptRunner } from './lib/ScriptRunner.mjs' async function _removeFeatureFromAllUsers(feature, commit) { let removals = 0 @@ -44,7 +45,7 @@ async function main() { } try { - await main() + await scriptRunner(main) console.log('Done') process.exit(0) } catch (error) { diff --git a/services/web/scripts/remove_oauth_application.mjs b/services/web/scripts/remove_oauth_application.mjs index 601a68a171..fd37ca374c 100644 --- a/services/web/scripts/remove_oauth_application.mjs +++ b/services/web/scripts/remove_oauth_application.mjs @@ -1,6 +1,7 @@ import { OauthApplication } from '../app/src/models/OauthApplication.js' import parseArgs from 'minimist' import OError from '@overleaf/o-error' +import { scriptRunner } from './lib/ScriptRunner.mjs' async function _removeOauthApplication(appId) { if (!appId) { @@ -24,7 +25,7 @@ async function main() { } try { - await main() + await scriptRunner(main) console.log('Done') process.exit(0) } catch (error) { diff --git a/services/web/scripts/remove_unconfirmed_emails.mjs b/services/web/scripts/remove_unconfirmed_emails.mjs index e86540dba6..246334315a 100644 --- a/services/web/scripts/remove_unconfirmed_emails.mjs +++ b/services/web/scripts/remove_unconfirmed_emails.mjs @@ -8,6 +8,7 @@ import fs from 'node:fs/promises' import * as csv from 'csv' import { promisify } from 'node:util' import _ from 'lodash' +import { scriptRunner } from './lib/ScriptRunner.mjs' const CSV_FILENAME = '/tmp/remove_unconfirmed_emails.csv' /** @@ -38,7 +39,7 @@ const { generate, consume, commit, help } = minimist(process.argv.slice(2), { default: { generate: false, consume: false, commit: false }, }) -async function generateCsvFile() { +async function generateCsvFile(trackProgress) { console.time('generate_csv') let processedUsersCount = 0 @@ -92,7 +93,9 @@ async function generateCsvFile() { totalEmailsToRemove += unconfirmedSecondaries.length } }, - { _id: 1, signUpDate: 1, emails: 1, email: 1 } + { _id: 1, signUpDate: 1, emails: 1, email: 1 }, + undefined, + { trackProgress } ) const csvContent = await stringifyAsync(records) @@ -226,7 +229,7 @@ async function consumeCsvFile() { console.log() } -async function main() { +async function main(trackProgress) { if (help) { return usage() } @@ -247,14 +250,14 @@ async function main() { } if (generate) { - await generateCsvFile() + await generateCsvFile(trackProgress) } else if (consume) { await consumeCsvFile() } } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/restore_orphaned_docs.mjs b/services/web/scripts/restore_orphaned_docs.mjs index ebbe0d5e4b..7b4df055a9 100644 --- a/services/web/scripts/restore_orphaned_docs.mjs +++ b/services/web/scripts/restore_orphaned_docs.mjs @@ -1,6 +1,7 @@ import ProjectEntityRestoreHandler from '../app/src/Features/Project/ProjectEntityRestoreHandler.js' import ProjectEntityHandler from '../app/src/Features/Project/ProjectEntityHandler.js' import DocstoreManager from '../app/src/Features/Docstore/DocstoreManager.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' const ARGV = process.argv.slice(2) const DEVELOPER_USER_ID = ARGV.shift() @@ -35,7 +36,7 @@ async function main() { } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/restore_soft_deleted_docs.mjs b/services/web/scripts/restore_soft_deleted_docs.mjs index 3e4575fa2b..5faf85559e 100644 --- a/services/web/scripts/restore_soft_deleted_docs.mjs +++ b/services/web/scripts/restore_soft_deleted_docs.mjs @@ -1,5 +1,6 @@ import ProjectEntityRestoreHandler from '../app/src/Features/Project/ProjectEntityRestoreHandler.js' import DocstoreManager from '../app/src/Features/Docstore/DocstoreManager.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' const ARGV = process.argv.slice(2) const DEVELOPER_USER_ID = ARGV.shift() @@ -24,7 +25,7 @@ async function main() { } try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/set_tex_live_image.mjs b/services/web/scripts/set_tex_live_image.mjs index 63b67fb4ca..a0d48dd9f3 100644 --- a/services/web/scripts/set_tex_live_image.mjs +++ b/services/web/scripts/set_tex_live_image.mjs @@ -1,6 +1,7 @@ import Settings from '@overleaf/settings' import mongodb from 'mongodb-legacy' import { Project } from '../app/src/models/Project.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' const { ObjectId } = mongodb @@ -61,7 +62,7 @@ async function updateImage(image, projectIds) { } try { - await main() + await scriptRunner(main) process.exit() } catch (error) { console.error(error) diff --git a/services/web/scripts/soft_delete_project.mjs b/services/web/scripts/soft_delete_project.mjs index 801bcc9a70..f530040980 100644 --- a/services/web/scripts/soft_delete_project.mjs +++ b/services/web/scripts/soft_delete_project.mjs @@ -1,5 +1,6 @@ import minimist from 'minimist' import ProjectDeleter from '../app/src/Features/Project/ProjectDeleter.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' async function main() { const argv = minimist(process.argv.slice(2)) @@ -14,7 +15,7 @@ async function main() { } try { - await main() + await scriptRunner(main) console.log('Done.') process.exit(0) } catch (error) { diff --git a/services/web/scripts/sso_id_migration_check.mjs b/services/web/scripts/sso_id_migration_check.mjs index 8f8f037b49..9638f6b960 100644 --- a/services/web/scripts/sso_id_migration_check.mjs +++ b/services/web/scripts/sso_id_migration_check.mjs @@ -1,5 +1,6 @@ import SAMLUserIdMigrationHandler from '../modules/saas-authentication/app/src/SAML/SAMLUserIdMigrationHandler.mjs' import { ensureRunningOnMongoSecondaryWithTimeout } from './helpers/env_variable_helper.mjs' +import { scriptRunner } from './lib/ScriptRunner.mjs' ensureRunningOnMongoSecondaryWithTimeout(300000) @@ -10,7 +11,7 @@ const emitUsers = process.argv.includes('--emit-users') console.log('Checking SSO user ID migration for institution:', institutionId) try { - await main() + await scriptRunner(main) } catch (error) { console.error(error) process.exit(1) diff --git a/services/web/scripts/sso_id_remove_not_migrated.mjs b/services/web/scripts/sso_id_remove_not_migrated.mjs index cf836f4796..cc7a5a874e 100644 --- a/services/web/scripts/sso_id_remove_not_migrated.mjs +++ b/services/web/scripts/sso_id_remove_not_migrated.mjs @@ -1,5 +1,6 @@ import SAMLUserIdMigrationHandler from '../modules/saas-authentication/app/src/SAML/SAMLUserIdMigrationHandler.mjs' import { ensureMongoTimeout } from './helpers/env_variable_helper.mjs' +import { scriptRunner } from './lib/ScriptRunner.mjs' ensureMongoTimeout(300000) @@ -30,7 +31,7 @@ async function main() { } try { - await main() + await scriptRunner(main) } catch (error) { console.error(error) process.exit(1) diff --git a/services/web/scripts/undelete_project_to_user.mjs b/services/web/scripts/undelete_project_to_user.mjs index ee837a4190..0863ad5f23 100644 --- a/services/web/scripts/undelete_project_to_user.mjs +++ b/services/web/scripts/undelete_project_to_user.mjs @@ -1,5 +1,6 @@ import minimist from 'minimist' import ProjectDeleter from '../app/src/Features/Project/ProjectDeleter.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' async function main() { const argv = minimist(process.argv.slice(2)) @@ -14,7 +15,7 @@ async function main() { } try { - await main() + await scriptRunner(main) console.log('Done.') process.exit(0) } catch (error) { diff --git a/services/web/scripts/unlink_third_party_id.mjs b/services/web/scripts/unlink_third_party_id.mjs index 8511deb403..eb76995bc2 100644 --- a/services/web/scripts/unlink_third_party_id.mjs +++ b/services/web/scripts/unlink_third_party_id.mjs @@ -1,6 +1,7 @@ import minimist from 'minimist' import ThirdPartyIdentityManager from '../app/src/Features/User/ThirdPartyIdentityManager.js' import UserGetter from '../app/src/Features/User/UserGetter.js' +import { scriptRunner } from './lib/ScriptRunner.mjs' /** * This script is used to remove a linked third party identity from a user account. @@ -79,7 +80,7 @@ async function main() { setup() try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error(error) diff --git a/services/web/scripts/writefull/back_fill_hiding_ai_features.mjs b/services/web/scripts/writefull/back_fill_hiding_ai_features.mjs index 2b188a812b..e40e065d51 100644 --- a/services/web/scripts/writefull/back_fill_hiding_ai_features.mjs +++ b/services/web/scripts/writefull/back_fill_hiding_ai_features.mjs @@ -1,7 +1,8 @@ import { db } from '../../app/src/infrastructure/mongodb.js' import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' +import { scriptRunner } from '../lib/ScriptRunner.mjs' -async function main() { +async function main(trackProgress) { // update all applicable user models await batchedUpdate( db.users, @@ -12,7 +13,10 @@ async function main() { $set: { 'aiErrorAssistant.enabled': false, }, - } + }, + undefined, + undefined, + { trackProgress } ) console.log('completed syncing writefull state with error assist') } @@ -20,7 +24,7 @@ async function main() { export default main try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error({ error }) diff --git a/services/web/scripts/writefull/enable_wf_autoCreatedAccount.mjs b/services/web/scripts/writefull/enable_wf_autoCreatedAccount.mjs index 190af04ef0..bdae9d8f26 100644 --- a/services/web/scripts/writefull/enable_wf_autoCreatedAccount.mjs +++ b/services/web/scripts/writefull/enable_wf_autoCreatedAccount.mjs @@ -3,6 +3,7 @@ import mongodb from 'mongodb-legacy' import fs from 'node:fs' import { fileURLToPath } from 'node:url' import { chunkArray } from '../helpers/chunkArray.mjs' +import { scriptRunner } from '../lib/ScriptRunner.mjs' const { ObjectId } = mongodb @@ -37,7 +38,7 @@ export default main if (fileURLToPath(import.meta.url) === process.argv[1]) { try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error({ error }) diff --git a/services/web/scripts/writefull/enable_writefull_without_autoCreatedAccount.mjs b/services/web/scripts/writefull/enable_writefull_without_autoCreatedAccount.mjs index 60a20626da..8f76ad666d 100644 --- a/services/web/scripts/writefull/enable_writefull_without_autoCreatedAccount.mjs +++ b/services/web/scripts/writefull/enable_writefull_without_autoCreatedAccount.mjs @@ -2,6 +2,7 @@ import { db } from '../../app/src/infrastructure/mongodb.js' import mongodb from 'mongodb-legacy' import fs from 'node:fs' import { fileURLToPath } from 'node:url' +import { scriptRunner } from '../lib/ScriptRunner.mjs' const { ObjectId } = mongodb @@ -36,7 +37,7 @@ export default main if (fileURLToPath(import.meta.url) === process.argv[1]) { try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error({ error }) diff --git a/services/web/scripts/writefull/split_writefull_disabled_from_unset.mjs b/services/web/scripts/writefull/split_writefull_disabled_from_unset.mjs index 85b6cf438c..dc86f8897b 100644 --- a/services/web/scripts/writefull/split_writefull_disabled_from_unset.mjs +++ b/services/web/scripts/writefull/split_writefull_disabled_from_unset.mjs @@ -4,10 +4,11 @@ import mongodb from 'mongodb-legacy' import fs from 'node:fs' import { fileURLToPath } from 'node:url' import { chunkArray } from '../helpers/chunkArray.mjs' +import { scriptRunner } from '../lib/ScriptRunner.mjs' const { ObjectId } = mongodb -async function main() { +async function main(trackProgress) { // search for file of users who already explicitly opted out first const optOutPath = process.argv[2] const optedOutFile = fs.readFileSync(optOutPath, 'utf8') @@ -20,7 +21,10 @@ async function main() { await batchedUpdate( db.users, { 'writefull.enabled': false }, // and is false - { $set: { 'writefull.enabled': null } } + { $set: { 'writefull.enabled': null } }, + undefined, + undefined, + { trackProgress } ) const chunks = chunkArray(optedOutList) @@ -41,7 +45,7 @@ export default main if (fileURLToPath(import.meta.url) === process.argv[1]) { try { - await main() + await scriptRunner(main) process.exit(0) } catch (error) { console.error({ error })