From 1447842fbdc36ad3015d21d38aa2c77cf1049256 Mon Sep 17 00:00:00 2001 From: Antoine Clausse Date: Mon, 17 Nov 2025 12:49:46 +0100 Subject: [PATCH] Remove scripts about unconfirmed emails removal (#29683) * Remove unconfirmed email deletion scripts * Remove tests GitOrigin-RevId: a0ef84207fced135a13074265fe5d3b38400d76f --- services/web/scripts/check_removed_emails.mjs | 214 ------------- .../web/scripts/re_add_deleted_emails.mjs | 288 ----------------- .../web/scripts/remove_unconfirmed_emails.mjs | 265 ---------------- .../src/ReAddDeletedEmailsTests.mjs | 275 ----------------- .../RemoveUnconfirmedEmailsScriptTests.mjs | 290 ------------------ 5 files changed, 1332 deletions(-) delete mode 100644 services/web/scripts/check_removed_emails.mjs delete mode 100644 services/web/scripts/re_add_deleted_emails.mjs delete mode 100644 services/web/scripts/remove_unconfirmed_emails.mjs delete mode 100644 services/web/test/acceptance/src/ReAddDeletedEmailsTests.mjs delete mode 100644 services/web/test/acceptance/src/RemoveUnconfirmedEmailsScriptTests.mjs diff --git a/services/web/scripts/check_removed_emails.mjs b/services/web/scripts/check_removed_emails.mjs deleted file mode 100644 index d9eb483e60..0000000000 --- a/services/web/scripts/check_removed_emails.mjs +++ /dev/null @@ -1,214 +0,0 @@ -// @ts-check - -import { db, ObjectId } from '../app/src/infrastructure/mongodb.js' -import fs from 'node:fs/promises' -import * as csv from 'csv' -import { promisify } from 'node:util' -import { scriptRunner } from './lib/ScriptRunner.mjs' -import { READ_PREFERENCE_SECONDARY } from '@overleaf/mongo-utils/batchedUpdate.js' - -const CSV_FILENAME = '/tmp/unconfirmed_emails.csv' - -/** - * @type {(csvString: string) => Promise} - */ -const parseAsync = promisify(csv.parse) - -/** - * Checks the fallout of services/web/scripts/remove_unconfirmed_emails.mjs - * which wrongly removed some emails that have been confirmed by users - */ -async function main(trackProgress) { - console.time('check_removed_emails') - - const csvContent = await fs.readFile(CSV_FILENAME, 'utf8') - const rows = await parseAsync(csvContent) - rows.shift() // Remove header row - const emailsByUserId = {} - - for (const [userId, email] of rows) { - if (!emailsByUserId[userId]) { - emailsByUserId[userId] = [] - } - emailsByUserId[userId].push(email) - } - - const userIds = Object.keys(emailsByUserId) - let processedUsersCount = 0 - - const counts = { - /** @type {string[]} */ - userNotFound: [], - /** @type {string[]} */ - notDeleted: [], - deleted: 0, - /** @type {string[]} */ - wasConfirmed: [], - /** @type {string[]} */ - wasConfirmedLegacy: [], - /** @type {string[]} */ - madePrimary: [], - /** @type {string[]} */ - madeSecondary: [], - /** @type {string[]} */ - isPrimary: [], - /** @type {string[]} */ - isAddedAgain: [], - } - - console.log('Total emails in the CSV:', rows.length) - console.log('Total users in the CSV:', userIds.length) - - for (const userId of userIds) { - const userEmails = emailsByUserId[userId] - - const user = await db.users.findOne( - { _id: new ObjectId(userId) }, - { readPreference: READ_PREFERENCE_SECONDARY } - ) - - if (!user) { - counts.userNotFound.push(userId) - continue - } - - for (const email of userEmails) { - const deletionLog = await db.userAuditLogEntries.findOne( - { - userId: new ObjectId(userId), - operation: 'remove-email', - 'info.removedEmail': email, - 'info.note': 'remove unconfirmed secondary emails', - }, - { readPreference: READ_PREFERENCE_SECONDARY } - ) - if (!deletionLog) { - counts.notDeleted.push(email) - continue - } - counts.deleted++ - - if (user.email === email) { - counts.isPrimary.push(email) - } - - const confirmationLog = await db.userAuditLogEntries.findOne( - { - userId: new ObjectId(userId), - operation: 'confirm-email-via-code', - 'info.email': email, - timestamp: { $gt: new Date('2025-02-25') }, - }, - { readPreference: READ_PREFERENCE_SECONDARY } - ) - if (confirmationLog) { - counts.wasConfirmed.push(email) - } - - const confirmationLegacyLog = await db.userAuditLogEntries.findOne( - { - userId: new ObjectId(userId), - operation: 'confirm-email', - 'info.email': email, - timestamp: { $gt: new Date('2025-02-25') }, - }, - { readPreference: READ_PREFERENCE_SECONDARY } - ) - if (confirmationLegacyLog) { - counts.wasConfirmedLegacy.push(email) - } - - const madePrimaryLog = await db.userAuditLogEntries.findOne( - { - userId: new ObjectId(userId), - operation: 'change-primary-email', - 'info.newPrimaryEmail': email, - timestamp: { $gt: new Date('2025-02-25') }, - }, - { readPreference: READ_PREFERENCE_SECONDARY } - ) - if (madePrimaryLog) { - counts.madePrimary.push(email) - } - - const madeSecondaryLog = await db.userAuditLogEntries.findOne( - { - userId: new ObjectId(userId), - operation: 'change-primary-email', - 'info.oldPrimaryEmail': email, - timestamp: { $gt: new Date('2025-02-25') }, - }, - { readPreference: READ_PREFERENCE_SECONDARY } - ) - if (madeSecondaryLog) { - counts.madeSecondary.push(email) - } - - if (user.emails.some(item => item.email === email)) { - counts.isAddedAgain.push(email) - } - } - - processedUsersCount++ - if (processedUsersCount % 100 === 0) { - trackProgress(`Processed ${processedUsersCount} users`) - } - } - - console.log() - console.log('Total emails in the CSV:', rows.length) - console.log('Total users in the CSV:', userIds.length) - console.log('Total users processed:', processedUsersCount) - console.log() - console.log('Users not found:', JSON.stringify(counts.userNotFound)) - console.log() - console.log('Emails not deleted:', counts.notDeleted.length) - console.log('Emails deleted:', counts.deleted) - console.log() - console.log('Emails that were confirmed:', counts.wasConfirmed.length) - console.log( - 'Emails that were confirmed:', - JSON.stringify(counts.wasConfirmed) - ) - console.log() - console.log( - 'Emails that were confirmed (legacy):', - counts.wasConfirmedLegacy.length - ) - console.log( - 'Emails that were confirmed (legacy):', - JSON.stringify(counts.wasConfirmedLegacy) - ) - console.log() - console.log('Emails that are primary:', counts.isPrimary.length) - console.log('Emails that are primary:', JSON.stringify(counts.isPrimary)) - console.log() - console.log('Emails that were made primary:', counts.madePrimary.length) - console.log( - 'Emails that were made primary:', - JSON.stringify(counts.madePrimary) - ) - console.log() - console.log('Emails that were made secondary:', counts.madeSecondary.length) - console.log( - 'Emails that were made secondary:', - JSON.stringify(counts.madeSecondary) - ) - console.log() - console.log('Emails that were added again:', counts.isAddedAgain.length) - console.log( - 'Emails that were added again:', - JSON.stringify(counts.isAddedAgain) - ) - console.log() - console.timeEnd('check_removed_emails') - console.log() -} - -try { - await scriptRunner(main) - process.exit(0) -} catch (error) { - console.error(error) - process.exit(1) -} diff --git a/services/web/scripts/re_add_deleted_emails.mjs b/services/web/scripts/re_add_deleted_emails.mjs deleted file mode 100644 index 8d8a620505..0000000000 --- a/services/web/scripts/re_add_deleted_emails.mjs +++ /dev/null @@ -1,288 +0,0 @@ -// @ts-check - -import minimist from 'minimist' -import { db, ObjectId } from '../app/src/infrastructure/mongodb.js' -import fs from 'node:fs/promises' -import * as csv from 'csv' -import { promisify } from 'node:util' -import { scriptRunner } from './lib/ScriptRunner.mjs' -import Errors from '../app/src/Features/Errors/Errors.js' -import UserGetter from '../app/src/Features/User/UserGetter.mjs' -import { READ_PREFERENCE_SECONDARY } from '@overleaf/mongo-utils/batchedUpdate.js' -import UserUpdater from '../app/src/Features/User/UserUpdater.mjs' -import EmailHelper from '../app/src/Features/Helpers/EmailHelper.mjs' -import AsyncLocalStorage from '../app/src/infrastructure/AsyncLocalStorage.js' -import AnalyticsManager from '../app/src/Features/Analytics/AnalyticsManager.mjs' -import UserAuditLogHandler from '../app/src/Features/User/UserAuditLogHandler.mjs' -import InstitutionsAPI from '../app/src/Features/Institutions/InstitutionsAPI.mjs' -import OError from '@overleaf/o-error' -import EmailChangeHelper from '../app/src/Features/Analytics/EmailChangeHelper.mjs' -import logger from '@overleaf/logger' - -const CSV_FILENAME = '/tmp/re_add_deleted_emails.csv' - -/** - * @type {(csvString: string) => Promise} - */ -const parseAsync = promisify(csv.parse) - -function usage() { - console.log('Usage: node re_add_deleted_emails.mjs [options]') - console.log( - 'fix wrongly removed emails by remove_unconfirmed_emails (see 2025-11-05 email removal)' - ) - console.log( - 'run this script with a CSV containing user IDs and emails to re-add save in ', - CSV_FILENAME - ) - console.log('Options:') - console.log(' --commit apply the changes') - process.exit(0) -} - -const { commit, help } = minimist(process.argv.slice(2), { - boolean: ['commit', 'help'], - alias: { help: 'h' }, - default: { commit: false }, -}) - -/** - * @param {string} email - */ -async function isEmailUsed(email) { - try { - await UserGetter.promises.ensureUniqueEmailAddress(email) - return false - } catch (err) { - if (err instanceof Errors.EmailExistsError) { - return true - } - throw err - } -} - -async function consumeCsvFile(trackProgress) { - console.time('re_add_deleted_emails') - - const csvContent = await fs.readFile(CSV_FILENAME, 'utf8') - const rows = await parseAsync(csvContent) - rows.shift() // Remove header row - - const emailsByUserId = {} - for (const [userId, email] of rows) { - if (!EmailHelper.parseEmail(email)) { - throw new Error(`invalid email ${email}`) - } - - if (!emailsByUserId[userId]) { - emailsByUserId[userId] = [] - } - emailsByUserId[userId].push(email) - } - - const userIds = Object.keys(emailsByUserId) - - const counts = { - /** @type {string[]} */ - processedUsers: [], - /** @type {string[]} */ - userNotFound: [], - /** @type {string[]} */ - emailsInUse: [], - /** @type {string[]} */ - alreadyOk: [], - /** @type {string[]} */ - primary: [], - /** @type {string[]} */ - secondary: [], - /** @type {string[]} */ - addedEmails: [], - } - - console.log('Total emails in the CSV:', rows.length) - console.log('Total users in the CSV:', userIds.length) - - for (const userId of userIds) { - const candidateEmails = emailsByUserId[userId] - - const user = await db.users.findOne( - { _id: new ObjectId(userId) }, - { readPreference: READ_PREFERENCE_SECONDARY } - ) - if (!user) { - counts.userNotFound.push(userId) - continue - } - - for (const email of candidateEmails) { - if (user.emails.some(item => item.email === email)) { - counts.alreadyOk.push(email) - continue - } - - const isUsed = await isEmailUsed(email) - const isOwnPrimary = user.email === email - - if (isUsed && !isOwnPrimary) { - counts.emailsInUse.push(email) - continue - } - - if (user.email === email) counts.primary.push(email) - else counts.secondary.push(email) - - if (commit) { - const auditLog = { - initiatorId: null, - ipAddress: null, - info: { - script: true, - note: 'fix wrongly removed unconfirmed secondary email', - }, - } - if (isOwnPrimary) { - // can't use addEmailAddress for primary email because ensureUniqueEmailAddress will throw - // using an override instead - await addEmailAddressOverride(user._id, email, {}, auditLog) - } else { - await UserUpdater.promises.addEmailAddress( - user._id, - email, - {}, - auditLog - ) - } - await UserUpdater.promises.confirmEmail(user._id, email) - } - counts.addedEmails.push(email) - } - - counts.processedUsers.push(userId) - trackProgress( - `Processed users: ${counts.processedUsers.length}/${userIds.length}` - ) - } - - console.log() - if (!commit) { - console.log('Dry-run, use --commit to apply changes') - console.log('This would be the result:') - console.log() - } - - console.log('Total emails in the CSV:', rows.length) - console.log('Total users in the CSV:', userIds.length) - console.log() - console.log('Users not found:', counts.userNotFound.length) - console.log('Users not found:', JSON.stringify(counts.userNotFound)) - console.log() - console.log('Already OK:', counts.alreadyOk.length) - console.log('Already OK:', JSON.stringify(counts.alreadyOk)) - console.log() - console.log('Already in use:', counts.emailsInUse.length) - console.log('Already in use:', JSON.stringify(counts.emailsInUse)) - console.log() - console.log('Primary:', counts.primary.length) - console.log('Primary:', JSON.stringify(counts.primary)) - console.log() - console.log('Secondary:', counts.secondary.length) - console.log('Secondary:', JSON.stringify(counts.secondary)) - console.log() - console.log('Added emails:', counts.addedEmails.length) - console.log('Added emails:', JSON.stringify(counts.addedEmails)) - console.log() - console.log() - console.timeEnd('re_add_deleted_emails') - console.log() -} - -async function main(trackProgress) { - if (help) { - return usage() - } - await consumeCsvFile(trackProgress) -} - -try { - await scriptRunner(main) - process.exit(0) -} catch (error) { - console.error(error) - process.exit(1) -} - -async function addEmailAddressOverride( - userId, - newEmail, - affiliationOptions, - auditLog -) { - AsyncLocalStorage.removeItem('userFullEmails') - newEmail = EmailHelper.parseEmail(newEmail) - if (!newEmail) { - throw new Error('invalid email') - } - - // Bypass ensureUniqueEmailAddress when re-adding primary emails - // await UserGetter.promises.ensureUniqueEmailAddress(newEmail) - - AnalyticsManager.recordEventForUserInBackground( - userId, - 'secondary-email-added' - ) - - await UserAuditLogHandler.promises.addEntry( - userId, - 'add-email', - auditLog.initiatorId, - auditLog.ipAddress, - { - ...auditLog.info, - newSecondaryEmail: newEmail, - } - ) - - try { - await InstitutionsAPI.promises.addAffiliation( - userId, - newEmail, - affiliationOptions - ) - } catch (error) { - throw OError.tag(error, 'problem adding affiliation while adding email') - } - - const createdAt = new Date() - let res - try { - const reversedHostname = newEmail.split('@')[1].split('').reverse().join('') - const update = { - $push: { - emails: { email: newEmail, createdAt, reversedHostname }, - }, - } - res = await UserUpdater.promises.updateUser( - { _id: userId, 'emails.email': { $ne: newEmail } }, - update - ) - } catch (error) { - throw OError.tag(error, 'problem updating users emails') - } - - if (res.matchedCount !== 1) { - return - } - - try { - await EmailChangeHelper.registerEmailCreation(userId, newEmail, { - // @ts-expect-error - This is copied from UserUpdater.mjs - createdAt: new Date(), - emailCreatedAt: createdAt, - }) - } catch (error) { - logger.warn( - { error, userId, newEmail }, - 'Error registering email creation with analytics' - ) - } -} diff --git a/services/web/scripts/remove_unconfirmed_emails.mjs b/services/web/scripts/remove_unconfirmed_emails.mjs deleted file mode 100644 index 279f09fedc..0000000000 --- a/services/web/scripts/remove_unconfirmed_emails.mjs +++ /dev/null @@ -1,265 +0,0 @@ -// @ts-check - -import minimist from 'minimist' -import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js' -import { db, ObjectId } from '../app/src/infrastructure/mongodb.js' -import UserAuditLogHandler from '../app/src/Features/User/UserAuditLogHandler.mjs' -import fs from 'node:fs/promises' -import * as csv from 'csv' -import { promisify } from 'node:util' -import _ from 'lodash' -import { scriptRunner } from './lib/ScriptRunner.mjs' - -const CSV_FILENAME = '/tmp/remove_unconfirmed_emails.csv' -/** - * @type {(records: string[][]) => Promise} - */ -const stringifyAsync = promisify(csv.stringify) -/** - * @type {(csvString: string) => Promise} - */ -const parseAsync = promisify(csv.parse) - -function usage() { - console.log('Usage: node remove_unconfirmed_emails.mjs') - console.log('Removes unconfirmed emails from users') - console.log('Options:') - console.log( - '' + - ' --generate generate the CSV file (remove_unconfirmed_emails.csv) containing the emails to remove\n' + - ' --consume consume the CSV file (remove_unconfirmed_emails.csv) and remove the emails (by default it is a dry-run)\n' + - ' --commit apply the changes (to be used with --consume)\n' - ) - process.exit(0) -} - -const { generate, consume, commit, help } = minimist(process.argv.slice(2), { - boolean: ['generate', 'consume', 'commit', 'help'], - alias: { help: 'h' }, - default: { generate: false, consume: false, commit: false }, -}) - -async function generateCsvFile(trackProgress) { - console.time('generate_csv') - - let processedUsersCount = 0 - let skippedUnconfirmedPrimaries = 0 - let totalEmailsToRemove = 0 - let totalUsersInCsv = 0 - - const records = [['User ID', 'Email', 'Sign Up Date']] - - await batchedUpdate( - db.users, - { - $and: [ - { emails: { $exists: true } }, - { emails: { $not: { $size: 0 } } }, - // Warning: this also matches unconfirmed primary emails - { - emails: { - $elemMatch: { - $or: [{ confirmedAt: { $exists: false } }, { confirmedAt: null }], - }, - }, - }, - ], - }, - async users => { - console.log('Process', users.length, 'users') - processedUsersCount += users.length - - for (const user of users) { - const unconfirmedSecondaries = user.emails.filter( - email => !email.confirmedAt && email.email !== user.email - ) - - if (unconfirmedSecondaries.length === 0) { - // Users can have been selected because of their unconfirmed primary email - // we don't want to remove those - skippedUnconfirmedPrimaries++ - continue - } - - for (const email of unconfirmedSecondaries) { - records.push([ - user._id.toString(), - email.email, - user.signUpDate.toISOString(), - ]) - } - - totalUsersInCsv++ - totalEmailsToRemove += unconfirmedSecondaries.length - } - }, - { _id: 1, signUpDate: 1, emails: 1, email: 1 }, - undefined, - { trackProgress } - ) - - const csvContent = await stringifyAsync(records) - await fs.writeFile(CSV_FILENAME, csvContent) - - console.log() - console.log('Processed users:', processedUsersCount) - console.log() - console.log('Generated CSV file:', CSV_FILENAME) - console.log('Total emails in the CSV:', totalEmailsToRemove) - console.log('Total users in the CSV:', totalUsersInCsv) - console.log( - 'Unconfirmed primary emails (skipped):', - skippedUnconfirmedPrimaries - ) - console.log() - console.timeEnd('generate_csv') - console.log() -} - -async function consumeCsvFile() { - console.time('consume_csv') - - const csvContent = await fs.readFile(CSV_FILENAME, 'utf8') - const rows = await parseAsync(csvContent) - rows.shift() // Remove header row - const emailsByUserId = {} - - for (const [userId, email] of rows) { - if (!emailsByUserId[userId]) { - emailsByUserId[userId] = [] - } - emailsByUserId[userId].push(email) - } - - const userIds = Object.keys(emailsByUserId) - let processedUsersCount = 0 - let removedEmailsCount = 0 - let totalModifiedUsersCount = 0 - const skippedEmail = { - userNotFound: 0, - nowConfirmed: 0, - nowPrimary: 0, - nowRemoved: 0, - } - - console.log('Total emails in the CSV:', rows.length) - console.log('Total users in the CSV:', userIds.length) - - for (const userId of userIds) { - const emailsToRemoveCandidates = emailsByUserId[userId] - - const user = await db.users.findOne({ _id: new ObjectId(userId) }) - if (!user) { - skippedEmail.userNotFound += emailsToRemoveCandidates.length - continue - } - - const emailsToRemove = emailsToRemoveCandidates.filter(email => { - const currentEmail = user.emails.find(e => e.email === email) - if (!currentEmail) { - skippedEmail.nowRemoved++ - return false - } - if (currentEmail.confirmedAt) { - skippedEmail.nowConfirmed++ - return false - } - if (currentEmail.email === user.email) { - skippedEmail.nowPrimary++ - return false - } - return true - }) - - removedEmailsCount += emailsToRemove.length - - if (commit && emailsToRemove.length > 0) { - for (const email of emailsToRemove) { - await UserAuditLogHandler.promises.addEntry( - userId, - 'remove-email', - undefined, - undefined, - { - removedEmail: email, - script: true, - note: 'remove unconfirmed secondary emails', - } - ) - } - - const updated = await db.users.updateOne( - { _id: new ObjectId(userId) }, - { $pull: { emails: { email: { $in: emailsToRemove } } } } - ) - totalModifiedUsersCount += updated.modifiedCount - } - - processedUsersCount++ - if (processedUsersCount % 100 === 0) { - console.log('Processed', processedUsersCount, 'users') - } - } - - console.log() - if (!commit) { - console.log('Dry-run, use --commit to apply changes') - console.log('This would be the result:') - console.log() - } - - console.log('Total emails in the CSV:', rows.length) - console.log('Total users in the CSV:', userIds.length) - console.log('Total users processed:', processedUsersCount) - console.log('Total emails removed:', removedEmailsCount) - console.log('Skipped emails:', _.sum(Object.values(skippedEmail))) - console.log(' - User not found:', skippedEmail.userNotFound) - console.log(' - Email now confirmed:', skippedEmail.nowConfirmed) - console.log(' - Email now primary:', skippedEmail.nowPrimary) - console.log(' - Email now removed:', skippedEmail.nowRemoved) - console.log() - - if (commit) { - console.log('Total users modified:', totalModifiedUsersCount) - } else { - console.log('Note: this was a dry-run. No changes were made.') - } - console.log() - console.timeEnd('consume_csv') - console.log() -} - -async function main(trackProgress) { - if (help) { - return usage() - } - - if (!generate && !consume) { - console.error('Error: Either --generate or --consume must be specified') - return usage() - } - - if (generate && consume) { - console.error('Error: Cannot use both --generate and --consume together') - return usage() - } - - if (commit && !consume) { - console.error('Error: --commit can only be used with --consume') - return usage() - } - - if (generate) { - await generateCsvFile(trackProgress) - } else if (consume) { - await consumeCsvFile() - } -} - -try { - await scriptRunner(main) - process.exit(0) -} catch (error) { - console.error(error) - process.exit(1) -} diff --git a/services/web/test/acceptance/src/ReAddDeletedEmailsTests.mjs b/services/web/test/acceptance/src/ReAddDeletedEmailsTests.mjs deleted file mode 100644 index 928d5b573d..0000000000 --- a/services/web/test/acceptance/src/ReAddDeletedEmailsTests.mjs +++ /dev/null @@ -1,275 +0,0 @@ -import { promisify } from 'node:util' -import { exec } from 'node:child_process' -import { expect } from 'chai' -import { filterOutput } from './helpers/settings.mjs' -import { db, ObjectId } from '../../../app/src/infrastructure/mongodb.js' -import fs from 'node:fs/promises' - -const CSV_FILENAME = '/tmp/re_add_deleted_emails.csv' - -async function runScript(commit) { - const result = await promisify(exec)( - ['node', 'scripts/re_add_deleted_emails.mjs', commit && '--commit'] - .filter(Boolean) - .join(' ') - ) - - return { - ...result, - stdout: result.stdout.split('\n').filter(filterOutput), - } -} - -/** - * @param {[string, string[]][]} userEmails - */ -const createUsers = async userEmails => - Promise.all( - userEmails.map(async ([email, emails]) => { - const _id = new ObjectId() - await db.users.insertOne({ - _id, - email, - emails: emails.map(email => ({ email })), - features: {}, - }) - return _id - }) - ) - -async function generateCsv(users) { - const text = 'User ID,Email' - const userRows = users.map(user => { - return `${user._id.toString()},${user.email}` - }) - await fs.writeFile(CSV_FILENAME, [text, ...userRows].join('\n')) -} - -describe('scripts/re_add_deleted_emails', function () { - let userIds - - afterEach(async function () { - try { - await fs.unlink(CSV_FILENAME) - } catch (err) { - // Ignore errors if file doesn't exist - } - }) - - describe('when user IDs dont match', function () { - beforeEach(async function () { - userIds = await createUsers([['mismatch1@xmpl.com', []]]) - await generateCsv([{ _id: new ObjectId(), email: 'mismatch2@xmpl.com' }]) - }) - - it('doesnt add new emails', async function () { - const { stdout } = await runScript(true) - expect(stdout).to.include('Total emails in the CSV: 1') - expect(stdout).to.include('Total users in the CSV: 1') - expect(stdout).to.include('Users not found: 1') - expect(stdout).to.include('Added emails: 0') - const updatedUser = await db.users.findOne({ _id: userIds[0] }) - expect(updatedUser.emails).to.have.length(0) - }) - }) - - describe('when email address is invalid', function () { - beforeEach(async function () { - userIds = await createUsers([['user@xmpl.com', []]]) - await generateCsv([{ _id: userIds[0], email: 'inv@lid@xmpl.com' }]) - }) - - it('throws', async function () { - await expect(runScript(true)).to.eventually.be.rejectedWith( - 'invalid email' - ) - const updatedUser = await db.users.findOne({ _id: userIds[0] }) - expect(updatedUser.emails).to.have.length(0) - }) - it('throws even without --commit', async function () { - await expect(runScript(false)).to.eventually.be.rejectedWith( - 'invalid email' - ) - const updatedUser = await db.users.findOne({ _id: userIds[0] }) - expect(updatedUser.emails).to.have.length(0) - }) - }) - - describe('when new email is used by another user', function () { - beforeEach(async function () { - userIds = await createUsers([ - ['user1@xmpl.com', []], - ['user2@xmpl.com', ['new-email@xmpl.com']], - ]) - await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }]) - }) - - it('doesnt add new emails', async function () { - const { stdout } = await runScript(true) - expect(stdout).to.include('Total emails in the CSV: 1') - expect(stdout).to.include('Total users in the CSV: 1') - expect(stdout).to.include('Users not found: 0') - expect(stdout).to.include('Already in use: ["new-email@xmpl.com"]') - expect(stdout).to.include('Added emails: 0') - const updatedUser = await db.users.findOne({ _id: userIds[0] }) - expect(updatedUser.emails).to.have.length(0) - }) - }) - - describe('when the user has 0 email in the array', function () { - beforeEach(async function () { - userIds = await createUsers([['user@xmpl.com', []]]) - }) - - it('adds the primary email to the user', async function () { - await generateCsv([{ _id: userIds[0], email: 'user@xmpl.com' }]) - const { stdout } = await runScript(true) - expect(stdout).to.include('Total emails in the CSV: 1') - expect(stdout).to.include('Total users in the CSV: 1') - expect(stdout).to.include('Users not found: 0') - expect(stdout).to.include('Added emails: 1') - const updatedUser = await db.users.findOne({ _id: userIds[0] }) - expect(updatedUser.emails).to.have.length(1) - expect(updatedUser.emails[0].email).to.equal('user@xmpl.com') - expect(updatedUser.emails[0].reversedHostname).to.equal('moc.lpmx') - expect(updatedUser.emails[0].confirmedAt).to.be.an.instanceof(Date) - expect(updatedUser.emails[0].createdAt).to.be.an.instanceof(Date) - expect(updatedUser.emails[0].reconfirmedAt).to.be.an.instanceof(Date) - const auditLogs = await db.userAuditLogEntries - .find({ userId: userIds[0] }) - .toArray() - expect(auditLogs).to.have.length(1) - expect(auditLogs[0].operation).to.equal('add-email') - expect(auditLogs[0].info).to.deep.include({ - script: true, - note: 'fix wrongly removed unconfirmed secondary email', - newSecondaryEmail: 'user@xmpl.com', - }) - }) - - it('adds the secondary email to the user', async function () { - await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }]) - const { stdout } = await runScript(true) - expect(stdout).to.include('Total emails in the CSV: 1') - expect(stdout).to.include('Total users in the CSV: 1') - expect(stdout).to.include('Users not found: 0') - expect(stdout).to.include('Added emails: 1') - const updatedUser = await db.users.findOne({ _id: userIds[0] }) - expect(updatedUser.emails).to.have.length(1) - expect(updatedUser.emails[0].email).to.equal('new-email@xmpl.com') - expect(updatedUser.emails[0].reversedHostname).to.equal('moc.lpmx') - expect(updatedUser.emails[0].confirmedAt).to.be.an.instanceof(Date) - expect(updatedUser.emails[0].createdAt).to.be.an.instanceof(Date) - expect(updatedUser.emails[0].reconfirmedAt).to.be.an.instanceof(Date) - const auditLogs = await db.userAuditLogEntries - .find({ userId: userIds[0] }) - .toArray() - expect(auditLogs).to.have.length(1) - expect(auditLogs[0].operation).to.equal('add-email') - expect(auditLogs[0].info).to.deep.include({ - script: true, - note: 'fix wrongly removed unconfirmed secondary email', - newSecondaryEmail: 'new-email@xmpl.com', - }) - }) - - it('doesnt add new emails without --commit', async function () { - await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }]) - const { stdout } = await runScript(false) - expect(stdout).to.include('Dry-run, use --commit to apply changes') - const updatedUser = await db.users.findOne({ _id: userIds[0] }) - expect(updatedUser.emails).to.have.length(0) - }) - }) - describe('when the user has several emails in the array', function () { - beforeEach(async function () { - userIds = await createUsers([ - [ - 'user@xmpl.com', - ['email1@xmpl.com', 'email2@xmpl.com', 'email3@xmpl.com'], - ], - ]) - await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }]) - }) - - it('adds the email to the user', async function () { - const { stdout } = await runScript(true) - - expect(stdout).to.include('Total emails in the CSV: 1') - expect(stdout).to.include('Total users in the CSV: 1') - expect(stdout).to.include('Users not found: 0') - expect(stdout).to.include('Added emails: 1') - const updatedUser = await db.users.findOne({ _id: userIds[0] }) - expect(updatedUser.emails).to.have.length(4) - - expect(updatedUser.emails[3].email).to.equal('new-email@xmpl.com') - expect(updatedUser.emails[3].reversedHostname).to.equal('moc.lpmx') - expect(updatedUser.emails[3].confirmedAt).to.be.an.instanceof(Date) - expect(updatedUser.emails[3].createdAt).to.be.an.instanceof(Date) - expect(updatedUser.emails[3].reconfirmedAt).to.be.an.instanceof(Date) - }) - - it('doesnt add new emails without --commit', async function () { - const { stdout } = await runScript(false) - expect(stdout).to.include('Dry-run, use --commit to apply changes') - const updatedUser = await db.users.findOne({ _id: userIds[0] }) - expect(updatedUser.emails).to.have.length(3) - }) - }) - - describe('all of the above', function () { - beforeEach(async function () { - userIds = await createUsers([ - ['user0@xmpl.com', []], - ['user1@xmpl.com', []], - ['user2@xmpl.com', ['a@xmpl.com', 'b@xmpl.com', 'c@xmpl.com']], - ['user3@xmpl.com', ['d@xmpl.com', 'e@xmpl.com', 'f@xmpl.com']], - ['user4@xmpl.com', ['x@xmpl.com', 'y@xmpl.com', 'z@xmpl.com']], - ['user5@xmpl.com', ['u@xmpl.com', 'v@xmpl.com', 'w@xmpl.com']], - ]) - await generateCsv([ - { _id: userIds[1], email: 'new1@xmpl.com' }, - { _id: userIds[1], email: 'user1@xmpl.com' }, - { _id: userIds[1], email: 'new2@xmpl.com' }, - { _id: userIds[2], email: 'new3@xmpl.com' }, - { _id: userIds[2], email: 'new4@xmpl.com' }, - { _id: userIds[2], email: 'user2@xmpl.com' }, - { _id: userIds[2], email: 'user3@xmpl.com' }, - { _id: userIds[3], email: 'd@xmpl.com' }, - { _id: userIds[5], email: 'a@xmpl.com' }, - { _id: new ObjectId(), email: 'a@xmpl.com' }, - ]) - }) - - it('updates users', async function () { - const { stdout } = await runScript(true) - expect(stdout).to.include('Total emails in the CSV: 10') - expect(stdout).to.include('Total users in the CSV: 5') - expect(stdout).to.include('Users not found: 1') - expect(stdout).to.include( - 'Already in use: ["user3@xmpl.com","a@xmpl.com"]' - ) - expect(stdout).to.include('Already OK: ["d@xmpl.com"]') - expect(stdout).to.include('Primary: ["user1@xmpl.com","user2@xmpl.com"]') - expect(stdout).to.include( - 'Secondary: ["new1@xmpl.com","new2@xmpl.com","new3@xmpl.com","new4@xmpl.com"]' - ) - expect(stdout).to.include( - 'Added emails: ["new1@xmpl.com","user1@xmpl.com","new2@xmpl.com","new3@xmpl.com","new4@xmpl.com","user2@xmpl.com"]' - ) - - const user0 = await db.users.findOne({ _id: userIds[0] }) - const user1 = await db.users.findOne({ _id: userIds[1] }) - const user2 = await db.users.findOne({ _id: userIds[2] }) - const user3 = await db.users.findOne({ _id: userIds[3] }) - const user4 = await db.users.findOne({ _id: userIds[4] }) - const user5 = await db.users.findOne({ _id: userIds[5] }) - expect(user0.emails).to.have.length(0) - expect(user1.emails).to.have.length(3) // new1, user1, new2 - expect(user2.emails).to.have.length(6) // a, b, c, new3, new4, user2 - expect(user3.emails).to.have.length(3) // d, e, f - expect(user4.emails).to.have.length(3) // x, y, z - expect(user5.emails).to.have.length(3) // u, v, w - }) - }) -}) diff --git a/services/web/test/acceptance/src/RemoveUnconfirmedEmailsScriptTests.mjs b/services/web/test/acceptance/src/RemoveUnconfirmedEmailsScriptTests.mjs deleted file mode 100644 index 6bb59965cc..0000000000 --- a/services/web/test/acceptance/src/RemoveUnconfirmedEmailsScriptTests.mjs +++ /dev/null @@ -1,290 +0,0 @@ -import { promisify } from 'node:util' -import { exec } from 'node:child_process' -import { expect } from 'chai' -import { filterOutput } from './helpers/settings.mjs' -import { db, ObjectId } from '../../../app/src/infrastructure/mongodb.js' -import fs from 'node:fs/promises' - -const CSV_FILENAME = '/tmp/remove_unconfirmed_emails.csv' - -async function runScript(mode, commit) { - const result = await promisify(exec)( - [ - 'node', - 'scripts/remove_unconfirmed_emails.mjs', - mode === 'generate' ? '--generate' : '--consume', - commit && '--commit', - ] - .filter(Boolean) - .join(' ') - ) - return { - ...result, - stdout: result.stdout.split('\n').filter(filterOutput), - } -} - -function createUser(signUpDate, emails, userIdx) { - const email = `primary${userIdx ?? ''}@overleaf.com` - return { - _id: new ObjectId(), - email, - emails, - signUpDate, - } -} - -describe('scripts/remove_unconfirmed_emails', function () { - let user - - afterEach(async function () { - try { - await fs.unlink(CSV_FILENAME) - } catch (err) { - // Ignore errors if file doesn't exist - } - }) - - describe('when removing unconfirmed secondary emails', function () { - beforeEach(async function () { - user = createUser(new Date('2000-01-01'), [ - { email: 'primary@overleaf.com', confirmedAt: new Date() }, - { email: 'unconfirmed1@overleaf.com' }, - { email: 'unconfirmed-special-,\'"@overleaf.com' }, - ]) - await db.users.insertOne(user) - }) - - it('should remove all unconfirmed secondary emails', async function () { - await runScript('generate') - const r = await runScript('consume', true) - - expect(r.stdout).to.include('Total emails in the CSV: 2') - expect(r.stdout).to.include('Total users processed: 1') - - const updatedUser = await db.users.findOne({ _id: user._id }) - expect(updatedUser.emails).to.have.length(1) - expect(updatedUser.emails[0].email).to.equal(user.email) - }) - - it('should not modify anything in dry run mode', async function () { - await runScript('generate') - const r = await runScript('consume', false) - - expect(r.stdout).to.include('Total emails in the CSV: 2') - expect(r.stdout).to.include('Total users processed: 1') - expect(r.stdout).to.include( - 'Note: this was a dry-run. No changes were made.' - ) - - const updatedUser = await db.users.findOne({ _id: user._id }) - expect(updatedUser.emails).to.have.length(3) - }) - }) - - describe('when handling confirmed secondary emails', function () { - beforeEach(async function () { - user = createUser(new Date('2000-01-01'), [ - { email: 'primary@overleaf.com', confirmedAt: new Date() }, - { email: 'confirmed@overleaf.com', confirmedAt: new Date() }, - ]) - await db.users.insertOne(user) - }) - - it('should preserve confirmed secondary emails', async function () { - await runScript('generate') - const r = await runScript('consume', true) - - expect(r.stdout).to.include('Total emails in the CSV: 0') - expect(r.stdout).to.include('Total users processed: 0') - - const updatedUser = await db.users.findOne({ _id: user._id }) - expect(updatedUser.emails).to.have.length(2) - expect(updatedUser.emails[1].confirmedAt).to.exist - }) - }) - - describe('when handling unconfirmed primary emails', function () { - beforeEach(async function () { - user = createUser(new Date('2000-01-01'), [ - { email: 'primary@overleaf.com' }, - ]) - await db.users.insertOne(user) - }) - - it('should not remove unconfirmed primary emails', async function () { - await runScript('generate') - const r = await runScript('consume', true) - - expect(r.stdout).to.include('Total emails in the CSV: 0') - expect(r.stdout).to.include('Total users processed: 0') - - const updatedUser = await db.users.findOne({ _id: user._id }) - expect(updatedUser.emails).to.have.length(1) - expect(updatedUser.emails[0].email).to.equal('primary@overleaf.com') - }) - }) - - describe('when users confirmed their email in between', function () { - beforeEach(async function () { - user = createUser(new Date('2000-01-01'), [ - { email: 'primary@overleaf.com' }, - { email: 'secondary@overleaf.com' }, - ]) - await db.users.insertOne(user) - }) - - it('should not remove emails from users who confirmed their email in between', async function () { - await runScript('generate') - - await db.users.updateOne( - { _id: user._id }, - { $set: { 'emails.1.confirmedAt': new Date() } } - ) - - const r = await runScript('consume', true) - - expect(r.stdout).to.include('Total emails in the CSV: 1') - expect(r.stdout).to.include('Skipped emails: 1') - expect(r.stdout).to.include(' - Email now confirmed: 1') - - const updatedUser = await db.users.findOne({ _id: user._id }) - expect(updatedUser.emails).to.have.length(2) - }) - }) - - describe('when users changed their primary email in between', function () { - beforeEach(async function () { - user = createUser(new Date('2000-01-01'), [ - { email: 'primary@overleaf.com' }, - { email: 'secondary@overleaf.com' }, - ]) - await db.users.insertOne(user) - }) - - it('should not remove emails from users who changed their primary email in between', async function () { - await runScript('generate') - - await db.users.updateOne( - { _id: user._id }, - { $set: { email: 'secondary@overleaf.com' } } - ) - const r = await runScript('consume', true) - - expect(r.stdout).to.include('Total emails in the CSV: 1') - expect(r.stdout).to.include('Skipped emails: 1') - expect(r.stdout).to.include(' - Email now primary: 1') - - const updatedUser = await db.users.findOne({ _id: user._id }) - expect(updatedUser.emails).to.have.length(2) - }) - }) - - describe('when users account was deleted in between', function () { - beforeEach(async function () { - user = createUser(new Date('2000-01-01'), [ - { email: 'primary@overleaf.com' }, - { email: 'secondary@overleaf.com' }, - ]) - await db.users.insertOne(user) - }) - - it('should skip emails from users whose account was deleted', async function () { - await runScript('generate') - - // Delete the user - await db.users.deleteOne({ _id: user._id }) - - const r = await runScript('consume', true) - - expect(r.stdout).to.include('Total emails in the CSV: 1') - expect(r.stdout).to.include('Skipped emails: 1') - expect(r.stdout).to.include(' - User not found: 1') - }) - }) - - describe('when users email was deleted in between', function () { - beforeEach(async function () { - user = createUser(new Date('2000-01-01'), [ - { email: 'primary@overleaf.com' }, - { email: 'secondary@overleaf.com' }, - ]) - await db.users.insertOne(user) - }) - - it('should skip emails that were already removed', async function () { - await runScript('generate') - - // Remove the secondary email - await db.users.updateOne( - { _id: user._id }, - { $pull: { emails: { email: 'secondary@overleaf.com' } } } - ) - - const r = await runScript('consume', true) - - expect(r.stdout).to.include('Total emails in the CSV: 1') - expect(r.stdout).to.include('Skipped emails: 1') - expect(r.stdout).to.include(' - Email now removed: 1') - - const updatedUser = await db.users.findOne({ _id: user._id }) - expect(updatedUser.emails).to.have.length(1) - expect(updatedUser.emails[0].email).to.equal('primary@overleaf.com') - }) - }) - - describe('when handling confirmation field edge cases', function () { - beforeEach(async function () { - user = createUser(new Date('2000-01-01'), [ - { email: 'primary@overleaf.com', confirmedAt: new Date() }, - { email: 'secondary1@overleaf.com', confirmedAt: null }, - { email: 'secondary2@overleaf.com' }, - ]) - await db.users.insertOne(user) - }) - - it('should remove emails with both missing and null confirmedAt', async function () { - await runScript('generate') - const r = await runScript('consume', true) - - expect(r.stdout).to.include('Total emails in the CSV: 2') - expect(r.stdout).to.include('Total users processed: 1') - - const updatedUser = await db.users.findOne({ _id: user._id }) - expect(updatedUser.emails).to.have.length(1) - expect(updatedUser.emails[0].email).to.equal(user.email) - }) - }) - - describe('CSV file generation', function () { - beforeEach(async function () { - user = createUser(new Date('2000-01-01'), [ - { email: 'primary@overleaf.com', confirmedAt: new Date() }, - { email: 'unconfirmed1@overleaf.com' }, - { email: 'confirmed1@overleaf.com', confirmedAt: new Date() }, - { email: 'unconfirmed2@overleaf.com' }, - { email: '!,@overleaf.com' }, - { email: "!'@overleaf.com" }, - { email: '!,\'"@overleaf.com' }, - ]) - await db.users.insertOne(user) - }) - - it('should generate a valid CSV file', async function () { - const r = await runScript('generate') - - expect(r.stdout).to.include( - 'Generated CSV file: /tmp/remove_unconfirmed_emails.csv' - ) - expect(r.stdout).to.include('Total emails in the CSV: 5') - const csvContent = await fs.readFile(CSV_FILENAME, 'utf8') - expect(csvContent).to.equal(`User ID,Email,Sign Up Date -${user._id},unconfirmed1@overleaf.com,2000-01-01T00:00:00.000Z -${user._id},unconfirmed2@overleaf.com,2000-01-01T00:00:00.000Z -${user._id},"!,@overleaf.com",2000-01-01T00:00:00.000Z -${user._id},!'@overleaf.com,2000-01-01T00:00:00.000Z -${user._id},"!,'""@overleaf.com",2000-01-01T00:00:00.000Z -`) - }) - }) -})