From 958e05a001703ffe77f7f4554a105e2db850df9e Mon Sep 17 00:00:00 2001 From: Miguel Serrano Date: Wed, 30 Apr 2025 15:12:56 +0200 Subject: [PATCH] [web] script to update group members via CSV (#24861) * [web] script to update group members via CSV GitOrigin-RevId: 973d1bdb1180af008608e14e1ff31af83e47f630 --- .../Subscription/SubscriptionGroupHandler.js | 127 +++++++ .../scripts/add_subscription_members_csv.mjs | 202 ++++++++++ .../SubscriptionGroupHandlerTests.js | 345 ++++++++++++++++++ 3 files changed, 674 insertions(+) create mode 100644 services/web/scripts/add_subscription_members_csv.mjs diff --git a/services/web/app/src/Features/Subscription/SubscriptionGroupHandler.js b/services/web/app/src/Features/Subscription/SubscriptionGroupHandler.js index 3d03ce0d2f..b92ce807f6 100644 --- a/services/web/app/src/Features/Subscription/SubscriptionGroupHandler.js +++ b/services/web/app/src/Features/Subscription/SubscriptionGroupHandler.js @@ -1,11 +1,15 @@ const { callbackify } = require('util') +const _ = require('lodash') +const OError = require('@overleaf/o-error') const SubscriptionUpdater = require('./SubscriptionUpdater') const SubscriptionLocator = require('./SubscriptionLocator') const SubscriptionController = require('./SubscriptionController') const { Subscription } = require('../../models/Subscription') +const { User } = require('../../models/User') const RecurlyClient = require('./RecurlyClient') const PlansLocator = require('./PlansLocator') const SubscriptionHandler = require('./SubscriptionHandler') +const TeamInvitesHandler = require('./TeamInvitesHandler') const GroupPlansData = require('./GroupPlansData') const Modules = require('../../infrastructure/Modules') const { MEMBERS_LIMIT_ADD_ON_CODE } = require('./PaymentProviderEntities') @@ -14,6 +18,8 @@ const { PendingChangeError, InactiveError, } = require('./Errors') +const EmailHelper = require('../Helpers/EmailHelper') +const { InvalidEmailError } = require('../Errors/Errors') async function removeUserFromGroup(subscriptionId, userIdToRemove) { await SubscriptionUpdater.promises.removeUserFromGroup( @@ -328,6 +334,125 @@ async function upgradeGroupPlan(ownerId) { ) } +async function updateGroupMembersBulk( + inviterId, + subscriptionId, + emailList, + options = {} +) { + const { removeMembersNotIncluded, commit } = options + + // remove duplications and empty values + emailList = _.uniq(_.compact(emailList)) + + const invalidEmails = emailList.filter( + email => !EmailHelper.parseEmail(email) + ) + + if (invalidEmails.length > 0) { + throw new InvalidEmailError('email not valid', { + invalidEmails, + }) + } + + const subscription = await Subscription.findOne({ + _id: subscriptionId, + }).exec() + + const existingUserData = await User.find( + { + _id: { $in: subscription.member_ids }, + }, + { _id: 1, email: 1, 'emails.email': 1 } + ).exec() + + const existingUsers = existingUserData.map(user => ({ + _id: user._id, + emails: user.emails?.map(user => user.email), + })) + + const currentMemberEmails = _.flatten( + existingUsers + .filter(userData => userData.emails?.length > 0) + .map(user => user.emails) + ) + + const currentInvites = + subscription.teamInvites?.map(invite => invite.email) || [] + if (subscription.invited_emails?.length > 0) { + currentInvites.push(...subscription.invited_emails) + } + + const invitesToSend = _.difference( + emailList, + currentMemberEmails.concat(currentInvites) + ) + + let membersToRemove + let invitesToRevoke + let newTotalCount + + if (!removeMembersNotIncluded) { + membersToRemove = [] + invitesToRevoke = [] + newTotalCount = + existingUsers.length + currentInvites.length + invitesToSend.length + } else { + membersToRemove = [] + for (const existingUser of existingUsers) { + if (_.intersection(existingUser.emails, emailList).length === 0) { + membersToRemove.push(existingUser._id) + } + } + const invitesToMaintain = _.intersection(emailList, currentInvites) + invitesToRevoke = _.difference(currentInvites, invitesToMaintain) + newTotalCount = + existingUsers.length - + membersToRemove.length + + invitesToMaintain.length + + invitesToSend.length + } + + const result = { + emailsToSendInvite: invitesToSend, + emailsToRevokeInvite: invitesToRevoke, + membersToRemove, + currentMemberCount: existingUsers.length, + newTotalCount, + membersLimit: subscription.membersLimit, + } + + if (commit) { + if (newTotalCount > subscription.membersLimit) { + const { currentMemberCount, newTotalCount, membersLimit } = result + throw new OError('limit reached', { + currentMemberCount, + newTotalCount, + membersLimit, + }) + } + for (const email of invitesToSend) { + await TeamInvitesHandler.promises.createInvite( + inviterId, + subscription, + email + ) + } + for (const email of invitesToRevoke) { + await TeamInvitesHandler.promises.revokeInvite( + inviterId, + subscription, + email + ) + } + for (const user of membersToRemove) { + await removeUserFromGroup(subscription._id, user._id) + } + } + + return result +} + module.exports = { removeUserFromGroup: callbackify(removeUserFromGroup), replaceUserReferencesInGroups: callbackify(replaceUserReferencesInGroups), @@ -344,6 +469,7 @@ module.exports = { getGroupPlanUpgradePreview: callbackify(getGroupPlanUpgradePreview), upgradeGroupPlan: callbackify(upgradeGroupPlan), checkBillingInfoExistence: callbackify(checkBillingInfoExistence), + updateGroupMembersBulk: callbackify(updateGroupMembersBulk), promises: { removeUserFromGroup, replaceUserReferencesInGroups, @@ -360,5 +486,6 @@ module.exports = { getGroupPlanUpgradePreview, upgradeGroupPlan, checkBillingInfoExistence, + updateGroupMembersBulk, }, } diff --git a/services/web/scripts/add_subscription_members_csv.mjs b/services/web/scripts/add_subscription_members_csv.mjs new file mode 100644 index 0000000000..4120e16fdb --- /dev/null +++ b/services/web/scripts/add_subscription_members_csv.mjs @@ -0,0 +1,202 @@ +import fs from 'node:fs' +import minimist from 'minimist' +import { parse } from 'csv' +import Stream from 'node:stream/promises' +import SubscriptionGroupHandler from '../app/src/Features/Subscription/SubscriptionGroupHandler.js' +import { Subscription } from '../app/src/models/Subscription.js' +import { InvalidEmailError } from '../app/src/Features/Errors/Errors.js' + +function usage() { + console.log( + 'Usage: node scripts/add_subscription_members_csv.mjs -f -i -s [options]' + ) + console.log('Required arguments:') + console.log( + ' -s, --subscriptionId The ID of the subscription to update' + ) + console.log( + ' -i, --inviterId The ID of the user sending the invites' + ) + console.log( + ' -f, --filename The path to the file to read data from' + ) + console.log('Options:') + console.log( + ' --commit, -c Whether changes should be committed to the DB invites should be sent/revoked' + ) + console.log( + ' --removeMembersNotIncluded -r Remove members that are not in the CSV. Disabled when managed users are enabled for the subscription' + ) + console.log( + ' --verbose, -v Prints detailed information about the affected group members' + ) + console.log(' -h, --help Show this help message') + process.exit(0) +} + +let { + commit, + removeMembersNotIncluded, + inviterId, + subscriptionId, + filename, + help, + verbose, +} = minimist(process.argv.slice(2), { + string: ['filename', 'subscriptionId', 'inviterId'], + boolean: ['commit', 'removeMembersNotIncluded', 'help', 'verbose'], + alias: { + commit: 'c', + removeMembersNotIncluded: 'r', + filename: 'f', + help: 'h', + inviterId: 'i', + subscriptionId: 's', + verbose: 'v', + }, + default: { + commit: false, + removeMembersNotIncluded: false, + help: false, + verbose: false, + }, +}) + +const EMAIL_FIELD = 'email' + +if (help) { + usage() + process.exit(0) +} + +if (!subscriptionId || !inviterId || !filename) { + usage() + process.exit(1) +} + +async function processRows(rows) { + const emailList = [] + for await (const row of rows) { + const email = row[EMAIL_FIELD] + if (email) { + emailList.push(email) + } + } + if (emailList.length === 0) { + console.error(`CSV error: 'email' column doesn't exist or it's empty'`) + process.exit(1) + } + + let previewResult + + try { + previewResult = + await SubscriptionGroupHandler.promises.updateGroupMembersBulk( + inviterId, + subscriptionId, + emailList, + { removeMembersNotIncluded } + ) + } catch (error) { + if (error instanceof InvalidEmailError) { + console.error(`${filename} contains invalid email addresses:`) + console.error(error.info?.invalidEmails.join(',')) + process.exit(1) + } else { + throw error + } + } + + console.log('Result Preview:') + logResult(previewResult) + + if (previewResult.newTotalCount > previewResult.membersLimit) { + console.warn( + 'WARNING: the invite list has reached the membership limit (newTotalCount > membersLimit)' + ) + if (commit) { + console.error(`Invites won't be sent and users won't be deleted`) + } + process.exit(1) + } + + if (!commit) { + console.log( + 'this is a dry-run, use the --commit option to send the invite and make any DB changes' + ) + return + } + + console.log( + `Sending invites to ${previewResult.emailsToSendInvite.length} email addresses` + ) + + if (previewResult.membersToRemove > 0) { + console.log( + `${previewResult.membersToRemove.length} members will be removed from the group` + ) + } + + const commitResult = + await SubscriptionGroupHandler.promises.updateGroupMembersBulk( + inviterId, + subscriptionId, + emailList, + { removeMembersNotIncluded, commit } + ) + + console.log('Result:') + logResult(commitResult) +} + +function logResult(result) { + console.log( + JSON.stringify( + { + ...result, + emailsToSendInvite: verbose + ? result.emailsToSendInvite + : result.emailsToSendInvite.length, + membersToRemove: verbose + ? result.membersToRemove + : result.membersToRemove.length, + emailsToRevokeInvite: verbose + ? result.emailsToRevokeInvite + : result.emailsToRevokeInvite.length, + }, + null, + 2 + ) + ) +} + +async function main() { + const subscription = await Subscription.findOne({ + _id: subscriptionId, + }).exec() + if (!subscription) { + console.error(`subscription with id=${subscriptionId} not found`) + process.exit(1) + } + if (subscription.managedUsersEnabled && removeMembersNotIncluded) { + console.warn( + `subscription with id=${subscriptionId} has 'managedUsersEnabled=true'` + + `'--removeMembersNotIncluded' has been disabled` + ) + removeMembersNotIncluded = false + } + await Stream.pipeline( + fs.createReadStream(filename), + parse({ + columns: true, + }), + processRows + ) +} + +main() + .then(() => process.exit(0)) + .catch(error => { + console.error(error) + process.exit(1) + }) diff --git a/services/web/test/unit/src/Subscription/SubscriptionGroupHandlerTests.js b/services/web/test/unit/src/Subscription/SubscriptionGroupHandlerTests.js index ce9a379d00..d9e42d645c 100644 --- a/services/web/test/unit/src/Subscription/SubscriptionGroupHandlerTests.js +++ b/services/web/test/unit/src/Subscription/SubscriptionGroupHandlerTests.js @@ -1,7 +1,11 @@ const SandboxedModule = require('sandboxed-module') +const { ObjectId } = require('mongodb-legacy') const sinon = require('sinon') const { expect } = require('chai') const MockRequest = require('../helpers/MockRequest') +const { + InvalidEmailError, +} = require('../../../../app/src/Features/Errors/Errors') const modulePath = '../../../../app/src/Features/Subscription/SubscriptionGroupHandler' @@ -27,6 +31,7 @@ describe('SubscriptionGroupHandler', function () { admin_id: this.adminUser_id, manager_ids: [this.adminUser_id], _id: this.subscription_id, + membersLimit: 100, } this.changeRequest = { @@ -109,6 +114,10 @@ describe('SubscriptionGroupHandler', function () { findOne: sinon.stub().returns({ exec: sinon.stub().resolves }), } + this.User = { + find: sinon.stub().returns({ exec: sinon.stub().resolves }), + } + this.SessionManager = { getLoggedInUserId: sinon.stub().returns(this.user._id), } @@ -152,6 +161,13 @@ describe('SubscriptionGroupHandler', function () { }, } + this.TeamInvitesHandler = { + promises: { + revokeInvite: sinon.stub().resolves(), + createInvite: sinon.stub().resolves(), + }, + } + this.GroupPlansData = { enterprise: { collaborator: { @@ -194,9 +210,13 @@ describe('SubscriptionGroupHandler', function () { './SubscriptionLocator': this.SubscriptionLocator, './SubscriptionController': this.SubscriptionController, './SubscriptionHandler': this.SubscriptionHandler, + './TeamInvitesHandler': this.TeamInvitesHandler, '../../models/Subscription': { Subscription: this.Subscription, }, + '../../models/User': { + User: this.User, + }, './RecurlyClient': this.RecurlyClient, './PlansLocator': this.PlansLocator, './PaymentProviderEntities': this.PaymentProviderEntities, @@ -861,4 +881,329 @@ describe('SubscriptionGroupHandler', function () { this.RecurlyClient.promises.getPaymentMethod.should.not.have.been.called }) }) + + describe('updateGroupMembersBulk', function () { + const inviterId = new ObjectId() + + let members + let emailList + let callUpdateGroupMembersBulk + + beforeEach(function () { + members = [ + { + _id: new ObjectId(), + email: 'user1@example.com', + emails: [{ email: 'user1@example.com' }], + }, + { + _id: new ObjectId(), + email: 'user2-alias@example.com', + emails: [ + { + email: 'user2-alias@example.com', + }, + { + email: 'user2@example.com', + }, + ], + }, + { + _id: new ObjectId(), + email: 'user3@example.com', + emails: [{ email: 'user3@example.com' }], + }, + ] + + emailList = [ + 'user1@example.com', + 'user2@example.com', + 'new-user@example.com', // primary email of existing user + 'new-user-2@example.com', // secondary email of existing user + ] + callUpdateGroupMembersBulk = async (options = {}) => { + this.Subscription.findOne = sinon + .stub() + .returns({ exec: sinon.stub().resolves(this.subscription) }) + + this.User.find = sinon + .stub() + .returns({ exec: sinon.stub().resolves(members) }) + + return await this.Handler.promises.updateGroupMembersBulk( + inviterId, + this.subscription._id, + emailList, + options + ) + } + }) + + it('throws an error when any of the emails is invalid', async function () { + emailList.push('invalid@email') + + await expect( + callUpdateGroupMembersBulk({ commit: true }) + ).to.be.rejectedWith(InvalidEmailError) + }) + + describe('with commit = false', function () { + describe('with removeMembersNotIncluded = false', function () { + it('should preview zero users to delete, and should not send invites', async function () { + const result = await callUpdateGroupMembersBulk() + + expect(result).to.deep.equal({ + emailsToSendInvite: [ + 'new-user@example.com', + 'new-user-2@example.com', + ], + emailsToRevokeInvite: [], + membersToRemove: [], + currentMemberCount: 3, + newTotalCount: 5, + membersLimit: this.subscription.membersLimit, + }) + + expect(this.TeamInvitesHandler.promises.createInvite).not.to.have.been + .called + + expect(this.SubscriptionUpdater.promises.removeUserFromGroup).not.to + .have.been.called + }) + }) + + describe('with removeMembersNotIncluded = true', function () { + it('should preview the users to be deleted, and should not send invites', async function () { + const result = await callUpdateGroupMembersBulk({ + removeMembersNotIncluded: true, + }) + + expect(result).to.deep.equal({ + emailsToSendInvite: [ + 'new-user@example.com', + 'new-user-2@example.com', + ], + emailsToRevokeInvite: [], + membersToRemove: [members[2]._id], + currentMemberCount: 3, + newTotalCount: 4, + membersLimit: this.subscription.membersLimit, + }) + + expect(this.TeamInvitesHandler.promises.createInvite).not.to.have.been + .called + + expect(this.SubscriptionUpdater.promises.removeUserFromGroup).not.to + .have.been.called + }) + + it('should preview but not revoke invites to emails that are no longer invited', async function () { + this.subscription.teamInvites = [ + { email: 'new-user@example.com' }, + { email: 'no-longer-invited@example.com' }, + ] + + const result = await callUpdateGroupMembersBulk({ + removeMembersNotIncluded: true, + }) + + expect(result.emailsToRevokeInvite).to.deep.equal([ + 'no-longer-invited@example.com', + ]) + + expect(this.TeamInvitesHandler.promises.revokeInvite).not.to.have.been + .called + }) + }) + + it('does not throw an error when the member limit is reached', async function () { + this.subscription.membersLimit = 3 + const result = await callUpdateGroupMembersBulk() + + expect(result.membersLimit).to.equal(3) + expect(result.newTotalCount).to.equal(5) + }) + }) + + describe('with commit = true', function () { + describe('with removeMembersNotIncluded = false', function () { + it('should preview zero users to delete, and should send invites', async function () { + const result = await callUpdateGroupMembersBulk({ commit: true }) + + expect(result).to.deep.equal({ + emailsToSendInvite: [ + 'new-user@example.com', + 'new-user-2@example.com', + ], + emailsToRevokeInvite: [], + membersToRemove: [], + currentMemberCount: 3, + newTotalCount: 5, + membersLimit: this.subscription.membersLimit, + }) + + expect(this.SubscriptionUpdater.promises.removeUserFromGroup).not.to + .have.been.called + + expect( + this.TeamInvitesHandler.promises.createInvite.callCount + ).to.equal(2) + + expect( + this.TeamInvitesHandler.promises.createInvite + ).to.have.been.calledWith( + inviterId, + this.subscription, + 'new-user@example.com' + ) + + expect( + this.TeamInvitesHandler.promises.createInvite + ).to.have.been.calledWith( + inviterId, + this.subscription, + 'new-user-2@example.com' + ) + }) + + it('should not send invites to emails already invited', async function () { + this.subscription.teamInvites = [{ email: 'new-user@example.com' }] + + const result = await callUpdateGroupMembersBulk({ commit: true }) + + expect(result.emailsToSendInvite).to.deep.equal([ + 'new-user-2@example.com', + ]) + + expect( + this.TeamInvitesHandler.promises.createInvite.callCount + ).to.equal(1) + + expect( + this.TeamInvitesHandler.promises.createInvite + ).to.have.been.calledWith( + inviterId, + this.subscription, + 'new-user-2@example.com' + ) + }) + + it('should preview and not revoke invites to emails that are no longer invited', async function () { + this.subscription.teamInvites = [ + { email: 'new-user@example.com' }, + { email: 'no-longer-invited@example.com' }, + ] + + const result = await callUpdateGroupMembersBulk({ + commit: true, + }) + + expect(result.emailsToRevokeInvite).to.deep.equal([]) + + expect(this.TeamInvitesHandler.promises.revokeInvite).not.to.have.been + .called + }) + }) + + describe('with removeMembersNotIncluded = true', function () { + it('should remove users from group, and should send invites', async function () { + const result = await callUpdateGroupMembersBulk({ + commit: true, + removeMembersNotIncluded: true, + }) + + expect(result).to.deep.equal({ + emailsToSendInvite: [ + 'new-user@example.com', + 'new-user-2@example.com', + ], + emailsToRevokeInvite: [], + membersToRemove: [members[2]._id], + currentMemberCount: 3, + newTotalCount: 4, + membersLimit: this.subscription.membersLimit, + }) + + expect( + this.SubscriptionUpdater.promises.removeUserFromGroup.callCount + ).to.equal(1) + + expect( + this.SubscriptionUpdater.promises.removeUserFromGroup + ).to.have.been.calledWith(this.subscription._id, members[2]._id) + + expect( + this.TeamInvitesHandler.promises.createInvite.callCount + ).to.equal(2) + + expect( + this.TeamInvitesHandler.promises.createInvite + ).to.have.been.calledWith( + inviterId, + this.subscription, + 'new-user@example.com' + ) + + expect( + this.TeamInvitesHandler.promises.createInvite + ).to.have.been.calledWith( + inviterId, + this.subscription, + 'new-user-2@example.com' + ) + }) + + it('should send invites and revoke invites to emails no longer invited', async function () { + this.subscription.teamInvites = [ + { email: 'new-user@example.com' }, + { email: 'no-longer-invited@example.com' }, + ] + + const result = await callUpdateGroupMembersBulk({ + commit: true, + removeMembersNotIncluded: true, + }) + + expect(result.emailsToSendInvite).to.deep.equal([ + 'new-user-2@example.com', + ]) + + expect(result.emailsToRevokeInvite).to.deep.equal([ + 'no-longer-invited@example.com', + ]) + + expect( + this.TeamInvitesHandler.promises.createInvite.callCount + ).to.equal(1) + + expect( + this.TeamInvitesHandler.promises.createInvite + ).to.have.been.calledWith( + inviterId, + this.subscription, + 'new-user-2@example.com' + ) + + expect( + this.TeamInvitesHandler.promises.revokeInvite.callCount + ).to.equal(1) + + expect( + this.TeamInvitesHandler.promises.revokeInvite + ).to.have.been.calledWith( + inviterId, + this.subscription, + 'no-longer-invited@example.com' + ) + }) + }) + + it('throws an error when the member limit is reached', async function () { + this.subscription.membersLimit = 3 + await expect( + callUpdateGroupMembersBulk({ commit: true }) + ).to.be.rejectedWith('limit reached') + }) + }) + }) })