diff --git a/services/web/scripts/migration_compile_timeout_60s_to_20s.js b/services/web/scripts/migration_compile_timeout_60s_to_20s.js deleted file mode 100644 index d58b58b60f..0000000000 --- a/services/web/scripts/migration_compile_timeout_60s_to_20s.js +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env node - -const minimist = require('minimist') -const { - db, - READ_PREFERENCE_SECONDARY, - waitForDb, -} = require('../app/src/infrastructure/mongodb') -const { batchedUpdate } = require('./helpers/batchedUpdate') - -async function logCount() { - const count60s = await db.users.countDocuments( - { 'features.compileTimeout': { $lte: 60, $ne: 20 } }, - { readPreference: READ_PREFERENCE_SECONDARY } - ) - console.log(`Found ${count60s} users with compileTimeout <= 60s && != 20s`) - const count20s = await db.users.countDocuments( - { 'features.compileTimeout': 20 }, - { readPreference: READ_PREFERENCE_SECONDARY } - ) - console.log(`Found ${count20s} users with compileTimeout == 20s`) -} - -const main = async ({ COMMIT, SKIP_COUNT }) => { - console.time('Script Duration') - - await waitForDb() - - if (!SKIP_COUNT) { - await logCount() - } - - if (COMMIT) { - const nModified = await batchedUpdate( - 'users', - { 'features.compileTimeout': { $lte: 60, $ne: 20 } }, - // NOTE: Always update featuresUpdatedAt to ensure the user's features synced with BigQuery - { $set: { 'features.compileTimeout': 20, featuresUpdatedAt: new Date() } } - ) - console.log(`Updated ${nModified} records`) - } - - console.timeEnd('Script Duration') -} - -const setup = () => { - const argv = minimist(process.argv.slice(2)) - const COMMIT = argv.commit !== undefined - const SKIP_COUNT = argv['skip-count'] !== undefined - if (!COMMIT) { - console.warn('Doing dry run. Add --commit to commit changes') - } - return { COMMIT, SKIP_COUNT } -} - -main(setup()) - .catch(err => { - console.error(err) - process.exit(1) - }) - .then(() => process.exit(0)) diff --git a/services/web/scripts/migration_compile_timeout_60s_to_20s_fixup_features_updated_at.js b/services/web/scripts/migration_compile_timeout_60s_to_20s_fixup_features_updated_at.js deleted file mode 100644 index bc417477c1..0000000000 --- a/services/web/scripts/migration_compile_timeout_60s_to_20s_fixup_features_updated_at.js +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env node - -const minimist = require('minimist') -const { - db, - READ_PREFERENCE_SECONDARY, - waitForDb, -} = require('../app/src/infrastructure/mongodb') -const { batchedUpdate } = require('./helpers/batchedUpdate') - -// A few seconds after the previous migration script was run -const FEATURES_UPDATED_AT = new Date('2024-04-16T12:41:00Z') - -const query = { - 'features.compileTimeout': 20, - $or: [ - { featuresUpdatedAt: { $exists: false } }, - { featuresUpdatedAt: { $lt: FEATURES_UPDATED_AT } }, - ], -} - -async function logCount() { - const usersToUpdate = await db.users.countDocuments(query, { - readPreference: READ_PREFERENCE_SECONDARY, - }) - console.log( - `Found ${usersToUpdate} users needing their featuresUpdatedAt updated` - ) -} - -const main = async ({ COMMIT, SKIP_COUNT }) => { - console.time('Script Duration') - - await waitForDb() - - if (!SKIP_COUNT) { - await logCount() - } - - if (COMMIT) { - const nModified = await batchedUpdate('users', query, { - $set: { featuresUpdatedAt: FEATURES_UPDATED_AT }, - }) - console.log(`Updated ${nModified} records`) - } - - console.timeEnd('Script Duration') -} - -const setup = () => { - const argv = minimist(process.argv.slice(2)) - const COMMIT = argv.commit !== undefined - const SKIP_COUNT = argv['skip-count'] !== undefined - if (!COMMIT) { - console.warn('Doing dry run. Add --commit to commit changes') - } - return { COMMIT, SKIP_COUNT } -} - -main(setup()) - .catch(err => { - console.error(err) - process.exit(1) - }) - .then(() => process.exit(0)) diff --git a/services/web/scripts/migration_compile_timeout_60s_to_20s_fixup_new_users.js b/services/web/scripts/migration_compile_timeout_60s_to_20s_fixup_new_users.js deleted file mode 100644 index ce51092c04..0000000000 --- a/services/web/scripts/migration_compile_timeout_60s_to_20s_fixup_new_users.js +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env node - -const minimist = require('minimist') -const { - db, - READ_PREFERENCE_SECONDARY, - waitForDb, -} = require('../app/src/infrastructure/mongodb') -const { batchedUpdate } = require('./helpers/batchedUpdate') - -// A few seconds after the previous migration script was run -const FEATURES_UPDATED_AT = new Date('2024-04-16T12:41:00Z') - -const query = { - 'features.compileTimeout': 20, - featuresUpdatedAt: FEATURES_UPDATED_AT, - signUpDate: { $gt: FEATURES_UPDATED_AT }, -} - -async function logCount() { - const usersToUpdate = await db.users.countDocuments(query, { - readPreference: READ_PREFERENCE_SECONDARY, - }) - console.log( - `Found ${usersToUpdate} users needing their featuresUpdatedAt removed` - ) -} - -const main = async ({ COMMIT, SKIP_COUNT }) => { - console.time('Script Duration') - - await waitForDb() - - if (!SKIP_COUNT) { - await logCount() - } - - if (COMMIT) { - const nModified = await batchedUpdate('users', query, { - $unset: { featuresUpdatedAt: 1 }, - }) - console.log(`Updated ${nModified} records`) - } - - console.timeEnd('Script Duration') -} - -const setup = () => { - const argv = minimist(process.argv.slice(2)) - const COMMIT = argv.commit !== undefined - const SKIP_COUNT = argv['skip-count'] !== undefined - if (!COMMIT) { - console.warn('Doing dry run. Add --commit to commit changes') - } - return { COMMIT, SKIP_COUNT } -} - -main(setup()) - .catch(err => { - console.error(err) - process.exit(1) - }) - .then(() => process.exit(0)) diff --git a/services/web/test/acceptance/src/MigrateUserFeatureTimeoutTests.js b/services/web/test/acceptance/src/MigrateUserFeatureTimeoutTests.js deleted file mode 100644 index 8dd221b2f3..0000000000 --- a/services/web/test/acceptance/src/MigrateUserFeatureTimeoutTests.js +++ /dev/null @@ -1,447 +0,0 @@ -const { expect } = require('chai') -const { promisify } = require('node:util') -const { exec } = require('node:child_process') -const logger = require('@overleaf/logger') -const { db } = require('../../../app/src/infrastructure/mongodb') - -async function runScript(args = []) { - try { - return await promisify(exec)( - ['node', 'scripts/migration_compile_timeout_60s_to_20s.js', ...args].join( - ' ' - ) - ) - } catch (error) { - logger.error({ error }, 'script failed') - throw error - } -} - -async function runFixupScript(args = []) { - try { - return await promisify(exec)( - [ - 'node', - 'scripts/migration_compile_timeout_60s_to_20s_fixup_features_updated_at.js', - ...args, - ].join(' ') - ) - } catch (error) { - logger.error({ error }, 'script failed') - throw error - } -} - -async function runSecondFixupScript(args = []) { - try { - return await promisify(exec)( - [ - 'node', - 'scripts/migration_compile_timeout_60s_to_20s_fixup_new_users.js', - ...args, - ].join(' ') - ) - } catch (error) { - logger.error({ error }, 'script failed') - throw error - } -} - -describe('MigrateUserFeatureTimeoutTests', function () { - describe('initial script', function () { - const usersInput = { - noFeatures: {}, - noFeatureTimeout: { features: {} }, - timeout10s: { - features: { compileTimeout: 10, other: 'val1' }, - bar: '1', - featuresUpdatedAt: new Date('2020-01-01'), - }, - timeout20s: { features: { compileTimeout: 20, other: 'val2' }, bar: '2' }, - timeout30s: { - features: { compileTimeout: 30, other: 'val3' }, - bar: '3', - featuresUpdatedAt: new Date('2025-01-01'), - }, - timeout60s: { - features: { compileTimeout: 60, other: 'val4' }, - bar: '4', - featuresUpdatedAt: new Date(), - }, - timeout120s: { - features: { compileTimeout: 120, other: 'val5' }, - bar: '5', - }, - timeout180s: { - features: { compileTimeout: 180, other: 'val6' }, - bar: '6', - featuresUpdatedAt: new Date('2020-01-01'), - }, - } - - const usersKeys = Object.keys(usersInput) - const userIds = {} - - beforeEach('insert users', async function () { - const usersInsertedValues = await db.users.insertMany( - usersKeys.map(key => ({ - ...usersInput[key], - email: `${key}@example.com`, - })) - ) - usersKeys.forEach( - (key, index) => (userIds[key] = usersInsertedValues.insertedIds[index]) - ) - }) - - afterEach('clear users', async function () { - await db.users.deleteMany({}) - }) - - it('gives correct counts in dry mode', async function () { - const users = await db.users.find().toArray() - expect(users).to.have.lengthOf(usersKeys.length) - - const result = await runScript([]) - - expect(result.stderr).to.contain( - 'Doing dry run. Add --commit to commit changes' - ) - expect(result.stdout).to.contain( - 'Found 3 users with compileTimeout <= 60s && != 20s' - ) - expect(result.stdout).to.contain( - 'Found 1 users with compileTimeout == 20s' - ) - expect(result.stdout).not.to.contain('Updated') - - const usersAfter = await db.users.find().toArray() - - expect(usersAfter).to.deep.equal(users) - }) - - it("updates users compileTimeout when '--commit' is set", async function () { - const users = await db.users.find().toArray() - expect(users).to.have.lengthOf(usersKeys.length) - const result = await runScript(['--commit']) - - expect(result.stdout).to.contain( - 'Found 3 users with compileTimeout <= 60s && != 20s' - ) - expect(result.stdout).to.contain( - 'Found 1 users with compileTimeout == 20s' - ) - expect(result.stdout).to.contain('Updated 3 records') - - const usersAfter = await db.users.find().toArray() - - expect( - usersAfter.map(({ featuresUpdatedAt, ...rest }) => rest) - ).to.deep.equal([ - { _id: userIds.noFeatures, email: 'noFeatures@example.com' }, - { - _id: userIds.noFeatureTimeout, - email: 'noFeatureTimeout@example.com', - features: {}, - }, - { - _id: userIds.timeout10s, - email: 'timeout10s@example.com', - features: { compileTimeout: 20, other: 'val1' }, - bar: '1', - }, - { - _id: userIds.timeout20s, - email: 'timeout20s@example.com', - features: { compileTimeout: 20, other: 'val2' }, - bar: '2', - }, - { - _id: userIds.timeout30s, - email: 'timeout30s@example.com', - features: { compileTimeout: 20, other: 'val3' }, - bar: '3', - }, - { - _id: userIds.timeout60s, - email: 'timeout60s@example.com', - features: { compileTimeout: 20, other: 'val4' }, - bar: '4', - }, - { - _id: userIds.timeout120s, - email: 'timeout120s@example.com', - features: { compileTimeout: 120, other: 'val5' }, - bar: '5', - }, - { - _id: userIds.timeout180s, - email: 'timeout180s@example.com', - features: { compileTimeout: 180, other: 'val6' }, - bar: '6', - }, - ]) - - expect(usersAfter[0].featuresUpdatedAt).to.be.undefined - expect(usersAfter[1].featuresUpdatedAt).to.be.undefined - expect(usersAfter[2].featuresUpdatedAt).to.be.instanceOf(Date) - expect(usersAfter[3].featuresUpdatedAt).to.be.undefined // was already 20s - expect(usersAfter[4].featuresUpdatedAt).to.be.instanceOf(Date) - expect(usersAfter[5].featuresUpdatedAt).to.be.instanceOf(Date) - expect(usersAfter[6].featuresUpdatedAt).to.be.undefined - - const result2 = await runScript([]) - - expect(result2.stdout).to.contain( - 'Found 0 users with compileTimeout <= 60s && != 20s' - ) - expect(result2.stdout).to.contain( - 'Found 4 users with compileTimeout == 20s' - ) - }) - }) - - const FEATURES_UPDATED_AT = new Date('2024-04-16T12:41:00Z') - - describe('fixup script', function () { - const usersInput = { - timeout20s1: { - features: { compileTimeout: 20 }, - }, - timeout20s2: { - features: { compileTimeout: 20 }, - featuresUpdatedAt: new Date('2023-01-01'), - }, - timeout20s3: { - features: { compileTimeout: 20 }, - featuresUpdatedAt: new Date('2025-01-01'), - }, - timeout240s1: { - features: { compileTimeout: 240 }, - }, - timeout240s2: { - features: { compileTimeout: 240 }, - featuresUpdatedAt: new Date('2023-01-01'), - }, - timeout240s3: { - features: { compileTimeout: 240 }, - featuresUpdatedAt: new Date('2025-01-01'), - }, - } - - const usersKeys = Object.keys(usersInput) - const userIds = {} - - beforeEach('insert users', async function () { - const usersInsertedValues = await db.users.insertMany( - usersKeys.map(key => ({ - ...usersInput[key], - email: `${key}@example.com`, - })) - ) - usersKeys.forEach( - (key, index) => (userIds[key] = usersInsertedValues.insertedIds[index]) - ) - }) - afterEach('clear users', async function () { - await db.users.deleteMany({}) - }) - - it('gives correct counts in dry mode', async function () { - const users = await db.users.find().toArray() - expect(users).to.have.lengthOf(usersKeys.length) - const result = await runFixupScript([]) - expect(result.stderr).to.contain( - 'Doing dry run. Add --commit to commit changes' - ) - expect(result.stdout).to.contain( - 'Found 2 users needing their featuresUpdatedAt updated' - ) - expect(result.stdout).not.to.contain('Updated 2 records') - const usersAfter = await db.users.find().toArray() - expect(usersAfter).to.deep.equal(users) - }) - - it("updates users featuresUpdatedAt when '--commit' is set", async function () { - const users = await db.users.find().toArray() - expect(users).to.have.lengthOf(usersKeys.length) - const result = await runFixupScript(['--commit']) - expect(result.stdout).to.contain( - 'Found 2 users needing their featuresUpdatedAt updated' - ) - expect(result.stdout).to.contain('Updated 2 records') - const usersAfter = await db.users.find().toArray() - expect(usersAfter).to.deep.equal([ - { - _id: userIds.timeout20s1, - email: 'timeout20s1@example.com', - features: { compileTimeout: 20 }, - featuresUpdatedAt: FEATURES_UPDATED_AT, - }, - { - _id: userIds.timeout20s2, - email: 'timeout20s2@example.com', - features: { compileTimeout: 20 }, - featuresUpdatedAt: FEATURES_UPDATED_AT, - }, - { - _id: userIds.timeout20s3, - email: 'timeout20s3@example.com', - features: { compileTimeout: 20 }, - featuresUpdatedAt: new Date('2025-01-01'), - }, - { - _id: userIds.timeout240s1, - email: 'timeout240s1@example.com', - features: { compileTimeout: 240 }, - }, - { - _id: userIds.timeout240s2, - email: 'timeout240s2@example.com', - features: { compileTimeout: 240 }, - featuresUpdatedAt: new Date('2023-01-01'), - }, - { - _id: userIds.timeout240s3, - email: 'timeout240s3@example.com', - features: { compileTimeout: 240 }, - featuresUpdatedAt: new Date('2025-01-01'), - }, - ]) - - const result2 = await runFixupScript([]) - - expect(result2.stdout).to.contain( - 'Found 0 users needing their featuresUpdatedAt updated' - ) - }) - }) - - describe('fixup recent users', function () { - const usersInput = { - timeout20sNewerUser: { - features: { compileTimeout: 20 }, - signUpDate: new Date('2026-01-01'), - }, - // only this user should get updated - timeout20sNewUser: { - features: { compileTimeout: 20 }, - signUpDate: new Date('2025-01-01'), - featuresUpdatedAt: FEATURES_UPDATED_AT, - }, - timeout20sOldUser: { - features: { compileTimeout: 20 }, - signUpDate: new Date('2023-01-01'), - featuresUpdatedAt: FEATURES_UPDATED_AT, - }, - - timeout240sNewerUser: { - features: { compileTimeout: 240 }, - signUpDate: new Date('2026-01-01'), - }, - // We didn't produce such mismatch (featuresUpdatedAt < signUpDate) on premium users. - // But we should still test that the script doesn't update them. - timeout240sNewUser: { - features: { compileTimeout: 240 }, - signUpDate: new Date('2025-01-01'), - featuresUpdatedAt: FEATURES_UPDATED_AT, - }, - timeout240sOldUser: { - features: { compileTimeout: 240 }, - signUpDate: new Date('2023-01-01'), - featuresUpdatedAt: FEATURES_UPDATED_AT, - }, - } - - const usersKeys = Object.keys(usersInput) - const userIds = {} - - beforeEach('insert users', async function () { - const usersInsertedValues = await db.users.insertMany( - usersKeys.map(key => ({ - ...usersInput[key], - email: `${key}@example.com`, - })) - ) - usersKeys.forEach( - (key, index) => (userIds[key] = usersInsertedValues.insertedIds[index]) - ) - }) - afterEach('clear users', async function () { - await db.users.deleteMany({}) - }) - - it('gives correct counts in dry mode', async function () { - const users = await db.users.find().toArray() - expect(users).to.have.lengthOf(usersKeys.length) - const result = await runSecondFixupScript([]) - expect(result.stderr).to.contain( - 'Doing dry run. Add --commit to commit changes' - ) - expect(result.stdout).to.contain( - 'Found 1 users needing their featuresUpdatedAt removed' - ) - expect(result.stdout).not.to.contain('Updated 1 records') - const usersAfter = await db.users.find().toArray() - expect(usersAfter).to.deep.equal(users) - }) - - it("removes users featuresUpdatedAt when '--commit' is set", async function () { - const users = await db.users.find().toArray() - expect(users).to.have.lengthOf(usersKeys.length) - const result = await runSecondFixupScript(['--commit']) - expect(result.stdout).to.contain( - 'Found 1 users needing their featuresUpdatedAt removed' - ) - expect(result.stdout).to.contain('Updated 1 records') - const usersAfter = await db.users.find().toArray() - expect(usersAfter).to.deep.equal([ - { - _id: userIds.timeout20sNewerUser, - email: 'timeout20sNewerUser@example.com', - features: { compileTimeout: 20 }, - signUpDate: new Date('2026-01-01'), - }, - { - _id: userIds.timeout20sNewUser, - email: 'timeout20sNewUser@example.com', - features: { compileTimeout: 20 }, - signUpDate: new Date('2025-01-01'), - }, - { - _id: userIds.timeout20sOldUser, - email: 'timeout20sOldUser@example.com', - features: { compileTimeout: 20 }, - featuresUpdatedAt: FEATURES_UPDATED_AT, - signUpDate: new Date('2023-01-01'), - }, - { - _id: userIds.timeout240sNewerUser, - email: 'timeout240sNewerUser@example.com', - features: { compileTimeout: 240 }, - signUpDate: new Date('2026-01-01'), - }, - { - _id: userIds.timeout240sNewUser, - email: 'timeout240sNewUser@example.com', - features: { compileTimeout: 240 }, - featuresUpdatedAt: FEATURES_UPDATED_AT, - signUpDate: new Date('2025-01-01'), - }, - { - _id: userIds.timeout240sOldUser, - email: 'timeout240sOldUser@example.com', - features: { compileTimeout: 240 }, - featuresUpdatedAt: FEATURES_UPDATED_AT, - signUpDate: new Date('2023-01-01'), - }, - ]) - - const result2 = await runSecondFixupScript([]) - - expect(result2.stdout).to.contain( - 'Found 0 users needing their featuresUpdatedAt removed' - ) - }) - }) -})