diff --git a/libraries/eslint-plugin/index.js b/libraries/eslint-plugin/index.js index 10598abe45..5e6175a095 100644 --- a/libraries/eslint-plugin/index.js +++ b/libraries/eslint-plugin/index.js @@ -7,5 +7,6 @@ module.exports = { 'require-script-runner': require('./require-script-runner'), 'require-vi-doMock-valid-path': require('./require-vi-doMock-valid-path'), 'require-loading-label': require('./require-loading-label'), + 'require-cio-snake-case-properties': require('./require-cio-snake-case-properties'), }, } diff --git a/libraries/eslint-plugin/require-cio-snake-case-properties.js b/libraries/eslint-plugin/require-cio-snake-case-properties.js new file mode 100644 index 0000000000..a17d25e7b4 --- /dev/null +++ b/libraries/eslint-plugin/require-cio-snake-case-properties.js @@ -0,0 +1,111 @@ +'use strict' + +const SNAKE_CASE_RE = /^[a-z][a-z0-9]*(_[a-z0-9]+)*$/ + +function isSnakeCase(name) { + return SNAKE_CASE_RE.test(name) +} + +function getStaticKeyName(property) { + if (property.computed) return null + if (property.key.type === 'Identifier') return property.key.name + if (property.key.type === 'Literal' && typeof property.key.value === 'string') + return property.key.value + return null +} + +/** + * Check if a node is a call to CustomerIoHandler.updateUserAttributes() + * and return the attributes argument (2nd argument) + */ +function getUpdateUserAttributesArg(node) { + if ( + node.callee.type === 'MemberExpression' && + node.callee.object.type === 'Identifier' && + node.callee.object.name === 'CustomerIoHandler' && + node.callee.property.name === 'updateUserAttributes' && + node.arguments[1]?.type === 'ObjectExpression' + ) { + return node.arguments[1] + } + return null +} + +/** + * Check if a node is a call to Modules[.promises].hooks.fire('setUserProperties', ...) + * and return the attributes argument (3rd argument) + */ +function getSetUserPropertiesArg(node) { + const callee = node.callee + if (callee.type !== 'MemberExpression' || callee.property.name !== 'fire') { + return null + } + + // Check first argument is 'setUserProperties' + if ( + !node.arguments[0] || + node.arguments[0].type !== 'Literal' || + node.arguments[0].value !== 'setUserProperties' + ) { + return null + } + + // Match: Modules.hooks.fire or Modules.promises.hooks.fire + const obj = callee.object + if (obj.type === 'MemberExpression' && obj.property.name === 'hooks') { + const parent = obj.object + // Modules.hooks + if (parent.type === 'Identifier' && parent.name === 'Modules') { + if (node.arguments[2]?.type === 'ObjectExpression') { + return node.arguments[2] + } + } + // Modules.promises.hooks + if ( + parent.type === 'MemberExpression' && + parent.property.name === 'promises' && + parent.object.type === 'Identifier' && + parent.object.name === 'Modules' + ) { + if (node.arguments[2]?.type === 'ObjectExpression') { + return node.arguments[2] + } + } + } + + return null +} + +module.exports = { + meta: { + type: 'problem', + docs: { + description: + 'Enforce snake_case for Customer.io user property attribute names', + }, + }, + create(context) { + return { + CallExpression(node) { + const attrsNode = + getUpdateUserAttributesArg(node) || getSetUserPropertiesArg(node) + if (!attrsNode) return + + for (const property of attrsNode.properties) { + if (property.type === 'SpreadElement') continue + + const keyName = getStaticKeyName(property) + if (keyName === null) continue // skip computed/dynamic keys + + if (!isSnakeCase(keyName)) { + context.report({ + node: property.key, + message: `Customer.io attribute '{{name}}' must be in snake_case.`, + data: { name: keyName }, + }) + } + } + }, + } + }, +} diff --git a/libraries/eslint-plugin/rules.test.js b/libraries/eslint-plugin/rules.test.js index f7f85f9016..377ba0316c 100644 --- a/libraries/eslint-plugin/rules.test.js +++ b/libraries/eslint-plugin/rules.test.js @@ -4,6 +4,7 @@ const noUnnecessaryTrans = require('./no-unnecessary-trans') const shouldUnescapeTrans = require('./should-unescape-trans') const noGeneratedEditorThemes = require('./no-generated-editor-themes') const viDoMockValidPath = require('./require-vi-doMock-valid-path') +const requireCioSnakeCaseProperties = require('./require-cio-snake-case-properties') const ruleTester = new RuleTester({ parser: require.resolve('@typescript-eslint/parser'), @@ -166,3 +167,103 @@ ruleTester.run('domock-require-valid-path', viDoMockValidPath, { }, ], }) + +ruleTester.run( + 'require-cio-snake-case-properties', + requireCioSnakeCaseProperties, + { + valid: [ + // updateUserAttributes with snake_case keys + { + code: `CustomerIoHandler.updateUserAttributes(userId, { plan_type: 'free', group_size: 10 })`, + }, + // Modules.promises.hooks.fire with snake_case keys + { + code: `Modules.promises.hooks.fire('setUserProperties', userId, { plan_type: 'free', last_active: 123 })`, + }, + // Modules.hooks.fire with snake_case keys + { + code: `Modules.hooks.fire('setUserProperties', userId, { plan_type: 'free' })`, + }, + // Single-word keys are valid snake_case + { + code: `CustomerIoHandler.updateUserAttributes(userId, { email: 'a@b.com', role: 'admin' })`, + }, + // Computed/dynamic keys are skipped + { + code: `CustomerIoHandler.updateUserAttributes(userId, { [dynamicKey]: true })`, + }, + // Spread elements are skipped + { + code: `CustomerIoHandler.updateUserAttributes(userId, { ...existingAttrs })`, + }, + // Unrelated function calls are not checked + { + code: `SomeOtherHandler.updateUserAttributes(userId, { camelCase: true })`, + }, + // fire() with a different event name is not checked + { + code: `Modules.promises.hooks.fire('someOtherEvent', userId, { camelCase: true })`, + }, + ], + invalid: [ + // camelCase key in updateUserAttributes + { + code: `CustomerIoHandler.updateUserAttributes(userId, { planType: 'free' })`, + errors: [ + { + message: `Customer.io attribute 'planType' must be in snake_case.`, + }, + ], + }, + // kebab-case string key + { + code: `CustomerIoHandler.updateUserAttributes(userId, { 'plan-type': 'free' })`, + errors: [ + { + message: `Customer.io attribute 'plan-type' must be in snake_case.`, + }, + ], + }, + // PascalCase key + { + code: `CustomerIoHandler.updateUserAttributes(userId, { PlanType: 'free' })`, + errors: [ + { + message: `Customer.io attribute 'PlanType' must be in snake_case.`, + }, + ], + }, + // camelCase in Modules.promises.hooks.fire + { + code: `Modules.promises.hooks.fire('setUserProperties', userId, { planType: 'free' })`, + errors: [ + { + message: `Customer.io attribute 'planType' must be in snake_case.`, + }, + ], + }, + // camelCase in Modules.hooks.fire + { + code: `Modules.hooks.fire('setUserProperties', userId, { planType: 'free' })`, + errors: [ + { + message: `Customer.io attribute 'planType' must be in snake_case.`, + }, + ], + }, + // Multiple invalid keys report multiple errors + { + code: `CustomerIoHandler.updateUserAttributes(userId, { planType: 'free', groupSize: 10, plan_term: 'annual' })`, + errors: [ + { + message: `Customer.io attribute 'planType' must be in snake_case.`, + }, + { + message: `Customer.io attribute 'groupSize' must be in snake_case.`, + }, + ], + }, + ], + } +) diff --git a/services/web/.eslintrc.js b/services/web/.eslintrc.js index 0a45fde7a8..3a9b106394 100644 --- a/services/web/.eslintrc.js +++ b/services/web/.eslintrc.js @@ -29,6 +29,7 @@ module.exports = { 'import/no-extraneous-dependencies': 'error', '@overleaf/prefer-kebab-url': 'error', + '@overleaf/require-cio-snake-case-properties': 'error', // disable some TypeScript rules '@typescript-eslint/no-var-requires': 'off', diff --git a/services/web/app/src/Features/Project/ProjectListController.mjs b/services/web/app/src/Features/Project/ProjectListController.mjs index 4a38ecc4f2..e00059ff12 100644 --- a/services/web/app/src/Features/Project/ProjectListController.mjs +++ b/services/web/app/src/Features/Project/ProjectListController.mjs @@ -34,7 +34,7 @@ import { OnboardingDataCollection } from '../../models/OnboardingDataCollection. import UserSettingsHelper from './UserSettingsHelper.mjs' /** - * @import { GetProjectsRequest, GetProjectsResponse, AllUsersProjects, MongoProject, FormattedProject, MongoTag } from "./types" + * @import { GetProjectsRequest, GetProjectsResponse, AllUsersProjects, MongoProject, FormattedProject, MongoTag, SubscriptionRecord } from "./types" * @import { Project, ProjectApi, ProjectAccessLevel, Filters, Page, Sort, UserRef } from "../../../../types/project/dashboard/api" * @import { Affiliation } from "../../../../types/affiliation" * @import { Source } from "../Authorization/types" @@ -554,26 +554,26 @@ async function projectListPage(req, res, next) { Modules.promises.hooks .fire('setUserProperties', userId, { - overleafId: userId, - lastActive: user.lastActive + overleaf_id: userId, + last_active: user.lastActive ? Math.floor(user.lastActive.getTime() / 1000) : null, - signUpDate: user.signUpDate + sign_up_date: user.signUpDate ? Math.floor(user.signUpDate.getTime() / 1000) : null, ...(usersBestSubscription?.type && { - 'best-subscription-type': usersBestSubscription.type, + best_subscription_type: usersBestSubscription.type, }), - aiBlocked, - hasAiAssist, - ...(subjectArea && { subjectArea }), + ai_blocked: aiBlocked, + has_ai_assist: hasAiAssist, + ...(subjectArea && { subject_area: subjectArea }), ...(role && { role }), - ...(primaryOccupation && { primaryOccupation }), - ...(usedLatex && { usedLatex }), + ...(primaryOccupation && { primary_occupation: primaryOccupation }), + ...(usedLatex && { used_latex: usedLatex }), ...(countryCode && { country: countryCode }), - ...(commonsInstitution && { commonsInstitution }), - ...(groupRole && { groupRole }), - isManagedUser: Boolean(user.enrollment?.managedBy), + ...(commonsInstitution && { commons_institution: commonsInstitution }), + ...(groupRole && { group_role: groupRole }), + is_managed_user: Boolean(user.enrollment?.managedBy), ...(user.email && { email: user.email }), }) .catch(err => { diff --git a/services/web/app/src/Features/Subscription/CustomerIoPlanHelpers.mjs b/services/web/app/src/Features/Subscription/CustomerIoPlanHelpers.mjs new file mode 100644 index 0000000000..e1071bd9b2 --- /dev/null +++ b/services/web/app/src/Features/Subscription/CustomerIoPlanHelpers.mjs @@ -0,0 +1,408 @@ +import Settings from '@overleaf/settings' +import { AI_ADD_ON_CODE, isStandaloneAiAddOnPlanCode } from './AiHelper.mjs' +import FeaturesHelper from './FeaturesHelper.mjs' + +const INACTIVE_NEXT_RENEWAL_DATE_STATES = new Set([ + 'canceled', + 'cancelled', + 'expired', +]) +const PENDING_CANCELLATION_STATES = new Set(['canceled', 'cancelled']) + +function getSubscriptionState(subscription) { + return ( + subscription?.recurlyStatus?.state || subscription?.paymentProvider?.state + ) +} + +function toUnixTimestamp(dateValue) { + if (!dateValue) { + return null + } + + const date = new Date(dateValue) + if (Number.isNaN(date.getTime())) { + return null + } + + return Math.floor(date.getTime() / 1000) +} + +function normalizePlanType(bestSubscription) { + if (!bestSubscription) { + return null + } + + if (['standalone-ai-add-on', 'commons'].includes(bestSubscription.type)) { + return bestSubscription.type + } + + const planCode = bestSubscription.plan?.planCode + const isGroupPlan = bestSubscription.plan?.groupPlan === true + + if (!planCode) { + return bestSubscription.type || null + } + + if (planCode.startsWith('v1_')) { + return 'v1' + } + + if (planCode.includes('student')) { + return 'student' + } + + if (planCode.includes('professional')) { + return isGroupPlan ? 'group-professional' : 'professional' + } + + if (planCode.includes('collaborator')) { + return isGroupPlan ? 'group-standard' : 'standard' + } + + if (planCode.includes('personal')) { + return 'personal' + } + + if (isGroupPlan) { + return 'group-standard' + } + + return planCode +} + +function getFriendlyPlanName(planType) { + if (!planType) { + return null + } + + const friendlyPlanNames = { + free: 'Free', + personal: 'Personal', + standard: 'Standard', + professional: 'Pro', + student: 'Student', + commons: 'Commons', + 'group-standard': 'Group Standard', + 'group-professional': 'Group Pro', + 'standalone-ai-add-on': 'AI Assist add-on', + v1: 'Legacy', + } + + if (friendlyPlanNames[planType]) { + return friendlyPlanNames[planType] + } + + return planType +} + +function getPlanCadence(bestSubscription) { + if (!bestSubscription?.plan) { + return null + } + + return bestSubscription.plan.annual ? 'annual' : 'monthly' +} + +function getPlanCadenceFromPlanCode(planCode) { + if (!planCode) { + return null + } + + const plan = Settings.plans.find(candidate => candidate.planCode === planCode) + if (plan) { + return plan.annual ? 'annual' : 'monthly' + } + + if (planCode.includes('annual')) { + return 'annual' + } + + if (isStandaloneAiAddOnPlanCode(planCode)) { + return 'monthly' + } + + return null +} + +function getNextRenewalDateFromPaymentRecord(paymentRecord) { + const subscriptionState = paymentRecord?.subscription?.state + if (INACTIVE_NEXT_RENEWAL_DATE_STATES.has(subscriptionState)) { + return null + } + + return toUnixTimestamp(paymentRecord?.subscription?.periodEnd) +} + +function shouldClearNextRenewalDate(subscription) { + if (!subscription) { + return true + } + + return INACTIVE_NEXT_RENEWAL_DATE_STATES.has( + getSubscriptionState(subscription) + ) +} + +function getExpiryDateFromPaymentRecord(paymentRecord) { + const subscriptionState = paymentRecord?.subscription?.state + if (!PENDING_CANCELLATION_STATES.has(subscriptionState)) { + return null + } + + const expiryDate = toUnixTimestamp(paymentRecord?.subscription?.periodEnd) + if (expiryDate == null) { + return null + } + + return expiryDate > Math.floor(Date.now() / 1000) ? expiryDate : null +} + +function shouldClearExpiryDate(subscription) { + if (!subscription) { + return true + } + + return !PENDING_CANCELLATION_STATES.has(getSubscriptionState(subscription)) +} + +function hasIndividualAiAssistAddOn(individualSubscription, paymentRecord) { + if ( + !individualSubscription || + individualSubscription.groupPlan || + isStandaloneAiAddOnPlanCode(individualSubscription.planCode) + ) { + return false + } + + return Boolean( + paymentRecord?.subscription?.addOns?.some( + addOn => addOn.code === AI_ADD_ON_CODE + ) + ) +} + +function getAiPlanType( + bestSubscription, + individualSubscription, + paymentRecord, + writefullData +) { + if ( + bestSubscription?.type === 'standalone-ai-add-on' || + isStandaloneAiAddOnPlanCode(individualSubscription?.planCode) + ) { + return 'ai-assist-standalone' + } + + if (hasIndividualAiAssistAddOn(individualSubscription, paymentRecord)) { + return 'ai-assist-add-on' + } + + if (writefullData?.isPremium) { + return 'writefull-premium' + } + + return 'none' +} + +function getAiPlanCadence( + aiPlan, + bestSubscription, + individualSubscription, + paymentRecord +) { + if (aiPlan === 'ai-assist-standalone') { + return ( + getPlanCadenceFromPlanCode(individualSubscription?.planCode) || + getPlanCadenceFromPlanCode(paymentRecord?.subscription?.planCode) + ) + } + + if (aiPlan === 'ai-assist-add-on') { + return ( + getPlanCadence(bestSubscription) || + getPlanCadenceFromPlanCode(individualSubscription?.planCode) + ) + } + + return null +} + +function hasPlanAiEnabled(plan) { + if (!plan?.features) { + return false + } + + return ( + plan.features.aiUsageQuota === Settings.aiFeatures.unlimitedQuota || + plan.features.aiErrorAssistant === true + ) +} + +function getGroupAiEnabled( + memberGroupSubscriptions = [], + managedGroupSubscriptions = [], + userIsMemberOfGroupSubscription +) { + if (!userIsMemberOfGroupSubscription) { + return null + } + + const allGroupSubscriptions = [ + ...memberGroupSubscriptions, + ...managedGroupSubscriptions, + ] + + return allGroupSubscriptions.some(subscription => { + const plan = Settings.plans.find( + candidate => candidate.planCode === subscription.planCode + ) + return hasPlanAiEnabled(plan) + }) +} + +function getGroupSize( + bestSubscription, + memberGroupSubscriptions = [], + managedGroupSubscriptions = [] +) { + const allGroupSubscriptions = [ + ...memberGroupSubscriptions, + ...managedGroupSubscriptions, + ] + + if (allGroupSubscriptions.length === 0) { + return null + } + + const matchingBestGroupSubscription = + bestSubscription?.type === 'group' + ? allGroupSubscriptions.find( + subscription => + subscription.planCode === bestSubscription.plan?.planCode && + subscription.teamName === bestSubscription.subscription?.teamName + ) + : null + + if (matchingBestGroupSubscription?.membersLimit != null) { + return matchingBestGroupSubscription.membersLimit + } + + if (bestSubscription?.subscription?.membersLimit != null) { + return bestSubscription.subscription.membersLimit + } + + if ( + bestSubscription?.plan?.groupPlan && + bestSubscription.plan.membersLimit != null + ) { + return bestSubscription.plan.membersLimit + } + + return allGroupSubscriptions.reduce((largestGroupSize, subscription) => { + const plan = Settings.plans.find( + candidate => candidate.planCode === subscription.planCode + ) + const groupSize = subscription.membersLimit ?? plan?.membersLimit ?? 0 + + return Math.max(largestGroupSize, groupSize) + }, 0) +} + +function shouldUseCommonsBestSubscription( + hasCommons, + bestSubscription, + commonsPlan +) { + if (!hasCommons) { + return false + } + + if (bestSubscription == null) { + return true + } + + return FeaturesHelper.isFeatureSetBetter( + commonsPlan?.features || {}, + bestSubscription.plan?.features || {} + ) +} + +/** + * Compute plan-related user properties for sending to customer.io. + */ +function getPlanProperties({ + bestSubscription, + individualSubscription, + individualPaymentRecord, + memberGroupSubscriptions, + managedGroupSubscriptions, + userIsMemberOfGroupSubscription, + writefullData, +}) { + const planType = normalizePlanType(bestSubscription) + const displayPlanType = getFriendlyPlanName(planType) + const planTermLabel = getPlanCadence(bestSubscription) + const aiPlan = getAiPlanType( + bestSubscription, + individualSubscription, + individualPaymentRecord, + writefullData + ) + const aiPlanTermLabel = getAiPlanCadence( + aiPlan, + bestSubscription, + individualSubscription, + individualPaymentRecord + ) + const groupAiEnabled = getGroupAiEnabled( + memberGroupSubscriptions, + managedGroupSubscriptions, + userIsMemberOfGroupSubscription + ) + const nextRenewalDate = getNextRenewalDateFromPaymentRecord( + individualPaymentRecord + ) + const expiryDate = getExpiryDateFromPaymentRecord(individualPaymentRecord) + const groupSizeValue = getGroupSize( + bestSubscription, + memberGroupSubscriptions, + managedGroupSubscriptions + ) + + const nextRenewalDateTrait = + nextRenewalDate ?? + (shouldClearNextRenewalDate(individualSubscription) ? '' : undefined) + const expiryDateTrait = + expiryDate ?? + (shouldClearExpiryDate(individualSubscription) ? '' : undefined) + + const properties = { + ai_plan: aiPlan, + } + + if (planType) properties.plan_type = planType + if (displayPlanType) properties.display_plan_type = displayPlanType + if (planTermLabel) properties.plan_term_label = planTermLabel + if (aiPlanTermLabel) properties.ai_plan_term_label = aiPlanTermLabel + if (groupAiEnabled !== null) properties.group_ai_enabled = groupAiEnabled + if (nextRenewalDateTrait !== undefined) + properties.next_renewal_date = nextRenewalDateTrait + if (expiryDateTrait !== undefined) properties.expiry_date = expiryDateTrait + if (groupSizeValue !== null) properties.group_size = groupSizeValue + + return properties +} + +export default { + normalizePlanType, + getFriendlyPlanName, + getNextRenewalDateFromPaymentRecord, + getExpiryDateFromPaymentRecord, + getAiPlanType, + getAiPlanCadence, + hasPlanAiEnabled, + shouldUseCommonsBestSubscription, + getPlanProperties, +} diff --git a/services/web/app/src/Features/Subscription/FeaturesUpdater.mjs b/services/web/app/src/Features/Subscription/FeaturesUpdater.mjs index 61dbb23ce8..1e400b0f24 100644 --- a/services/web/app/src/Features/Subscription/FeaturesUpdater.mjs +++ b/services/web/app/src/Features/Subscription/FeaturesUpdater.mjs @@ -16,6 +16,7 @@ import AnalyticsManager from '../Analytics/AnalyticsManager.mjs' import Queues from '../../infrastructure/Queues.mjs' import Modules from '../../infrastructure/Modules.mjs' import SubscriptionViewModelBuilder from './SubscriptionViewModelBuilder.mjs' +import CustomerIoPlanHelpers from './CustomerIoPlanHelpers.mjs' import { AI_ADD_ON_CODE } from './AiHelper.mjs' import { fetchNothing } from '@overleaf/fetch-utils' import SplitTestHandler from '../SplitTests/SplitTestHandler.mjs' @@ -58,18 +59,12 @@ async function refreshFeatures(userId, reason) { const { features: newFeatures, featuresChanged } = await UserFeaturesUpdater.promises.updateFeatures(userId, features) - const bestSubscriptionType = await _getBestSubscriptionType(userId) - - Modules.promises.hooks - .fire('setUserProperties', userId, { - features, - 'best-subscription-type': bestSubscriptionType, - overleafId: userId, - ...(user.email && { email: user.email }), - }) - .catch(err => { - logger.error({ err, userId }, 'Failed to sync features to customer.io') - }) + _updateCustomerIoSubscriptionProperties(user, features).catch(err => { + logger.warn( + { err, userId }, + 'Failed to update subscription properties in customer.io' + ) + }) if (oldFeatures.dropbox === true && features.dropbox === false) { logger.debug({ userId }, '[FeaturesUpdater] must unlink dropbox') @@ -133,20 +128,61 @@ async function refreshFeatures(userId, reason) { return { features: newFeatures, featuresChanged } } -async function _getBestSubscriptionType(userId) { - try { - const { bestSubscription } = - await SubscriptionViewModelBuilder.promises.getUsersSubscriptionDetails({ - _id: userId, - }) - return bestSubscription?.type || 'free' - } catch (err) { - logger.warn( - { err, userId }, - 'Failed to calculate best-subscription-type for customer.io' - ) - return 'free' +async function _updateCustomerIoSubscriptionProperties(user, features) { + const userId = user._id + const { + bestSubscription, + individualSubscription, + memberGroupSubscriptions, + managedGroupSubscriptions, + } = await SubscriptionViewModelBuilder.promises.getUsersSubscriptionDetails({ + _id: userId, + }) + + const userIsMemberOfGroupSubscription = + memberGroupSubscriptions.length > 0 || managedGroupSubscriptions.length > 0 + + let individualPaymentRecord = null + if (individualSubscription && !individualSubscription.groupPlan) { + try { + ;[individualPaymentRecord] = await Modules.promises.hooks.fire( + 'getPaymentFromRecordPromise', + individualSubscription + ) + } catch (error) { + logger.warn( + { err: error, userId }, + 'Failed to load payment record for customer.io subscription properties' + ) + } } + + let writefullData = null + try { + writefullData = await UserGetter.promises.getWritefullData(userId) + } catch (error) { + logger.warn( + { err: error, userId }, + 'Failed to load writefull data for customer.io subscription properties' + ) + } + + const planProperties = CustomerIoPlanHelpers.getPlanProperties({ + bestSubscription, + individualSubscription, + individualPaymentRecord, + memberGroupSubscriptions, + managedGroupSubscriptions, + userIsMemberOfGroupSubscription, + writefullData, + }) + + await Modules.promises.hooks.fire('setUserProperties', userId, { + ...planProperties, + features, + overleaf_id: userId, + ...(user.email && { email: user.email }), + }) } /** diff --git a/services/web/scripts/batch_identify_to_cio.mjs b/services/web/scripts/batch_identify_to_cio.mjs new file mode 100644 index 0000000000..0de7971d8f --- /dev/null +++ b/services/web/scripts/batch_identify_to_cio.mjs @@ -0,0 +1,465 @@ +#!/usr/bin/env node + +/** + * This script reads a CSV file (output from migrate_mailchimp_to_cio.mjs or + * export_active_subscription_users_csv.mjs) and + * makes batch identify requests to Customer.io using the CDP Analytics node library. + * + * Usage: + * node scripts/batch_identify_to_cio.mjs --input INPUT-FILE [OPTIONS] + * + * Example: + * node scripts/batch_identify_to_cio.mjs --input /tmp/customerio_import.csv + * CUSTOMER_IO_API_KEY=xxx node scripts/batch_identify_to_cio.mjs --input /tmp/customerio_import.csv --commit + * + * Resuming after failure: + * CUSTOMER_IO_API_KEY=xxx node scripts/batch_identify_to_cio.mjs --input /tmp/customerio_import.csv --commit --skip 50000 + * + * Options: + * --input, -i PATH Input CSV file (from migrate_mailchimp_to_cio.mjs) (required) + * --commit Actually send to Customer.io (default is dry-run) + * --skip N Skip the first N rows (for resuming after failure) + * --batch-size N Maximum items per batch (default: 1000) + * --help, -h Show this help message + * + * Environment Variables: + * CUSTOMER_IO_API_KEY Customer.io CDP API key (required when using --commit) + * + * CSV Input Format (from migrate_mailchimp_to_cio.mjs): + * - email: Subscriber email address + * - overleafId: Overleaf user ID (from mongo_id) + * - created_at: Unix timestamp + * - cio_subscription_preferences.topics.: 'true' if subscribed + * - labsExperiments: JSON array of experiment names + * + * CSV Input Format (from export_active_subscription_users_csv.mjs): + * - user_id: Overleaf user ID + * - email: Subscriber email address + * - plan_type, display_plan_type, pre_migration_plan_type, + * pre_migration_display_plan_type, plan_term, ai_plan, ai_plan_term, + * next_renewal_date, expiry_date, group_ai_enabled, group_role + * + * Rate Limiting: + * - Max 3000 requests per 3 seconds (Customer.io limit) + * - Progress is logged every 10000 rows for recovery purposes + * + * Notes: + * - Users are identified by email address (used as userId) + * - The library handles batching internally via maxEventsInBatch + */ + +import fs, { createReadStream } from 'node:fs' +import * as csv from 'csv' +import minimist from 'minimist' +import { Analytics } from '@customerio/cdp-analytics-node' +import { scriptRunner } from './lib/ScriptRunner.mjs' + +const DEFAULT_BATCH_SIZE = 1000 +const RATE_LIMIT_REQUESTS = 3000 +const RATE_LIMIT_WINDOW_MS = 3000 +const PROGRESS_LOG_INTERVAL = 10000 + +function usage() { + console.error(`Usage: node scripts/batch_identify_to_cio.mjs --input INPUT-FILE [OPTIONS] + +Options: + --input, -i PATH Input CSV file (from migrate_mailchimp_to_cio.mjs) (required) + --commit Actually send to Customer.io (default is dry-run) + --skip N Skip the first N rows (for resuming after failure) + --batch-size N Maximum items per batch (default: ${DEFAULT_BATCH_SIZE}) + --help, -h Show this help message + +Environment Variables: + CUSTOMER_IO_API_KEY Customer.io CDP API key (required when using --commit) +`) + process.exit(1) +} + +/** + * Simple rate limiter that enforces max requests per time window + */ +class RateLimiter { + constructor(maxRequests, windowMs) { + this.maxRequests = maxRequests + this.windowMs = windowMs + this.requests = [] + } + + async waitIfNeeded() { + const now = Date.now() + // Remove requests outside the current window + this.requests = this.requests.filter(t => now - t < this.windowMs) + + if (this.requests.length >= this.maxRequests) { + // Wait until the oldest request falls outside the window + const oldestRequest = this.requests[0] + const waitTime = this.windowMs - (now - oldestRequest) + 10 // +10ms buffer + await new Promise(resolve => setTimeout(resolve, waitTime)) + // Clean up again after waiting + this.requests = this.requests.filter(t => Date.now() - t < this.windowMs) + } + + this.requests.push(Date.now()) + } +} + +/** + * Create a Customer.io Analytics client + */ +function createCioClient(batchSize) { + const apiKey = process.env.CUSTOMER_IO_API_KEY + + if (!apiKey) { + throw new Error( + 'CUSTOMER_IO_API_KEY environment variable is required. ' + + 'Set it to your Customer.io CDP API key.' + ) + } + + return new Analytics({ + writeKey: apiKey, + host: 'https://cdp.customer.io', + maxEventsInBatch: batchSize, + }) +} + +/** + * Convert a CSV row to a Customer.io identify payload + */ +function parseOptionalBoolean(value) { + if (value == null || value === '') { + return undefined + } + + const normalized = String(value).trim().toLowerCase() + if (normalized === 'true') { + return true + } + if (normalized === 'false') { + return false + } + + return undefined +} + +function parseOptionalInt(value) { + if (value == null || value === '') { + return undefined + } + + const parsed = parseInt(value, 10) + return Number.isNaN(parsed) ? undefined : parsed +} + +function getFirstDefinedValue(row, columnNames) { + for (const columnName of columnNames) { + if (Object.prototype.hasOwnProperty.call(row, columnName)) { + return row[columnName] + } + } + + return undefined +} + +function rowToIdentifyPayload(row) { + const email = getFirstDefinedValue(row, ['email']) + if (!email) { + return null + } + + const traits = {} + const overleafUserId = getFirstDefinedValue(row, [ + 'user_id', + 'userId', + 'overleafId', + ]) + + if (email) { + traits.email = email + } + + if (overleafUserId) { + traits.overleaf_id = overleafUserId + } + + // Add created_at if present (keep as unix timestamp) + const createdAtValue = getFirstDefinedValue(row, ['created_at']) + if (createdAtValue) { + const createdAt = parseOptionalInt(createdAtValue) + if (createdAt !== undefined) { + traits.created_at = createdAt + } + } + + // Add subscription status fields when present (from export_active_subscription_users_csv.mjs) + const stringTraitMappings = [ + { + columnNames: ['plan_type', 'planType'], + traitName: 'plan_type', + }, + { + columnNames: ['display_plan_type', 'displayPlanType'], + traitName: 'display_plan_type', + }, + { + columnNames: ['pre_migration_plan_type', 'preMigrationPlanType'], + traitName: 'pre_migration_plan_type', + }, + { + columnNames: [ + 'pre_migration_display_plan_type', + 'preMigrationDisplayPlanType', + ], + traitName: 'pre_migration_display_plan_type', + }, + { + columnNames: ['plan_term', 'planTerm', 'plan_term_label'], + traitName: 'plan_term', + }, + { + columnNames: ['ai_plan', 'aiPlan'], + traitName: 'ai_plan', + }, + { + columnNames: ['ai_plan_term', 'aiPlanTerm', 'ai_plan_term_label'], + traitName: 'ai_plan_term', + }, + { + columnNames: ['group_role', 'groupRole'], + traitName: 'group_role', + }, + ] + + for (const { columnNames, traitName } of stringTraitMappings) { + const value = getFirstDefinedValue(row, columnNames) + if (value) { + traits[traitName] = value + } + } + + const nextRenewalDateValue = getFirstDefinedValue(row, [ + 'next_renewal_date', + 'nextRenewalDate', + ]) + if (nextRenewalDateValue !== undefined) { + if (nextRenewalDateValue === '') { + traits.next_renewal_date = '' + } else { + const nextRenewalDate = parseOptionalInt(nextRenewalDateValue) + if (nextRenewalDate !== undefined) { + traits.next_renewal_date = nextRenewalDate + } + } + } + + const expiryDateValue = getFirstDefinedValue(row, [ + 'expiry_date', + 'expiryDate', + ]) + if (expiryDateValue !== undefined) { + if (expiryDateValue === '') { + traits.expiry_date = '' + } else { + const expiryDate = parseOptionalInt(expiryDateValue) + if (expiryDate !== undefined) { + traits.expiry_date = expiryDate + } + } + } + + const groupAiEnabledValue = getFirstDefinedValue(row, [ + 'group_ai_enabled', + 'groupAIEnabled', + ]) + if (groupAiEnabledValue !== undefined) { + const groupAiEnabled = parseOptionalBoolean(groupAiEnabledValue) + if (groupAiEnabled !== undefined) { + traits.group_ai_enabled = groupAiEnabled + } + } + + // Add subscription preferences + for (const key of Object.keys(row)) { + if (key.startsWith('cio_subscription_preferences.topics.')) { + if (row[key] === 'true') { + traits[key] = true + } + } + } + + // Add labsExperiments if present + if (row.labsExperiments) { + try { + traits.labsExperiments = JSON.parse(row.labsExperiments) + } catch { + // If it's not valid JSON, store as-is + traits.labsExperiments = row.labsExperiments + } + } + + return { + // Prefer stable Overleaf user id when available, otherwise fall back to email + userId: overleafUserId || email, + email, + traits, + } +} + +/** + * Create a CSV parser stream + */ +function createCsvParser(inputPath) { + return createReadStream(inputPath).pipe( + csv.parse({ + columns: true, + skip_empty_lines: true, + relax_column_count: true, + }) + ) +} + +/** + * Main script function + */ +function main() { + const argv = minimist(process.argv.slice(2), { + string: ['input', 'batch-size', 'skip'], + boolean: ['commit', 'help'], + alias: { + i: 'input', + h: 'help', + }, + }) + + if (argv.help) { + usage() + } + + const inputPath = argv.input + const commit = argv.commit + const dryRun = !commit + const batchSize = parseInt(argv['batch-size'], 10) || DEFAULT_BATCH_SIZE + const skipRows = parseInt(argv.skip, 10) || 0 + + if (!inputPath) { + console.error('Error: --input is required') + usage() + } + + if (!fs.existsSync(inputPath)) { + console.error(`Error: Input file not found: ${inputPath}`) + process.exit(1) + } + + if (commit && !process.env.CUSTOMER_IO_API_KEY) { + console.error( + 'Error: CUSTOMER_IO_API_KEY environment variable is required when using --commit' + ) + process.exit(1) + } + + scriptRunner( + async trackProgress => { + await trackProgress('Starting batch identify to Customer.io...') + await trackProgress(`Input: ${inputPath}`) + await trackProgress(`Batch size limit: ${batchSize}`) + if (skipRows > 0) { + await trackProgress(`Skipping first ${skipRows} rows`) + } + if (dryRun) { + await trackProgress('DRY RUN MODE - no requests will be sent') + } + + const rateLimiter = new RateLimiter( + RATE_LIMIT_REQUESTS, + RATE_LIMIT_WINDOW_MS + ) + const client = dryRun ? null : createCioClient(batchSize) + + // Listen to the 'error' event + if (!dryRun) { + client.on('error', err => { + console.error('cdp-analytics-node error occurred:') + console.error('Code:', err.code) + console.error('Reason:', err.reason) + if (err.ctx) { + console.error('Context:', err.ctx) + } + }) + } + + let rowNumber = 0 + let processedCount = 0 + let skippedCount = 0 + let lastProgressLog = 0 + let loggedFirstIdentifyPayload = false + + const parser = createCsvParser(inputPath) + + try { + for await (const row of parser) { + rowNumber++ + + // Skip rows if resuming + if (rowNumber <= skipRows) { + continue + } + + const payload = rowToIdentifyPayload(row) + if (!payload) { + skippedCount++ + continue + } + + if (dryRun && !loggedFirstIdentifyPayload) { + await trackProgress( + `First identify payload (dry run): ${JSON.stringify(payload)}` + ) + loggedFirstIdentifyPayload = true + } + + if (!dryRun) { + await rateLimiter.waitIfNeeded() + client.identify(payload) + } + + processedCount++ + + // Log progress periodically + if (processedCount - lastProgressLog >= PROGRESS_LOG_INTERVAL) { + await trackProgress( + `Progress: row ${rowNumber}, sent ${processedCount} requests (${skippedCount} skipped)` + ) + lastProgressLog = processedCount + } + } + } catch (error) { + await trackProgress( + `ERROR at row ${rowNumber}: ${error.message}. Resume with --skip ${rowNumber - 1}` + ) + throw error + } + + if (!dryRun && client) { + await trackProgress('Flushing remaining requests...') + await client.closeAndFlush() + } + + await trackProgress( + `Completed: processed ${processedCount} rows (${skippedCount} skipped due to missing email)` + ) + if (dryRun) { + await trackProgress( + `DRY RUN complete - would have sent ${processedCount} identify requests` + ) + } + process.exit(0) + }, + { inputPath, dryRun, batchSize, skipRows } + ).catch(err => { + console.error(err) + process.exit(1) + }) +} + +main() diff --git a/services/web/scripts/export_active_subscription_users_csv.mjs b/services/web/scripts/export_active_subscription_users_csv.mjs new file mode 100644 index 0000000000..3fa23eff98 --- /dev/null +++ b/services/web/scripts/export_active_subscription_users_csv.mjs @@ -0,0 +1,789 @@ +import fs from 'node:fs' +import { Parser as CSVParser } from 'json2csv' +import minimist from 'minimist' +import pLimit from 'p-limit' +import Settings from '@overleaf/settings' +import { scriptRunner } from './lib/ScriptRunner.mjs' +import { + db, + ObjectId, + READ_PREFERENCE_SECONDARY, +} from '../app/src/infrastructure/mongodb.mjs' +import PaymentService from '../modules/subscriptions/app/src/PaymentService.mjs' +import FeaturesHelper from '../app/src/Features/Subscription/FeaturesHelper.mjs' +import CustomerIoPlanHelpers from '../app/src/Features/Subscription/CustomerIoPlanHelpers.mjs' +import { isStandaloneAiAddOnPlanCode } from '../app/src/Features/Subscription/AiHelper.mjs' +import InstitutionsGetter from '../app/src/Features/Institutions/InstitutionsGetter.mjs' + +const CSV_FIELDS = [ + { label: 'user_id', value: 'userId' }, + { label: 'email', value: 'email' }, + { label: 'plan_type', value: 'planType' }, + { label: 'display_plan_type', value: 'displayPlanType' }, + { + label: 'pre_migration_plan_type', + value: 'preMigrationPlanType', + }, + { + label: 'pre_migration_display_plan_type', + value: 'preMigrationDisplayPlanType', + }, + { label: 'plan_term', value: 'planTerm' }, + { label: 'ai_plan', value: 'aiPlan' }, + { label: 'ai_plan_term', value: 'aiPlanTerm' }, + { label: 'next_renewal_date', value: 'nextRenewalDate' }, + { label: 'expiry_date', value: 'expiryDate' }, + { label: 'group_ai_enabled', value: 'groupAIEnabled' }, + { label: 'group_role', value: 'groupRole' }, +] +const CSV_FIELD_NAMES = CSV_FIELDS.map(field => field.value) + +const ACTIVE_SUBSCRIPTION_STATES = ['active', 'trialing'] +const COMMONS_PLAN = getPlan(Settings.institutionPlanCode) + +function usage() { + console.log(` +Usage: + node scripts/export_active_subscription_users_csv.mjs [options] + +Options: + --outputPath Output CSV path (default: /tmp/active_subscription_users.csv) + --concurrency Concurrent payment-provider lookups (default: 5) + --batchSize Number of users processed per batch (default: 500) + --resumeAfterUserId Resume processing strictly after this user id + --checkpointInterval Log resumable checkpoint every N processed users (default: 10000) + --append Append to existing output file (for resume runs) + --help Show this message +`) +} + +function parseArgs() { + const args = minimist(process.argv.slice(2), { + string: ['outputPath', 'resumeAfterUserId'], + boolean: ['help', 'append'], + default: { + outputPath: '/tmp/active_subscription_users.csv', + concurrency: 5, + batchSize: 500, + checkpointInterval: 10000, + append: false, + help: false, + }, + }) + + if (args.help) { + usage() + process.exit(0) + } + + return args +} + +function getPlan(planCode) { + return Settings.plans.find(plan => plan.planCode === planCode) || null +} + +function getPlanType(subscription) { + if (isStandaloneAiAddOnPlanCode(subscription.planCode)) { + return 'standalone-ai-add-on' + } + return subscription.groupPlan ? 'group' : 'individual' +} + +function getPlanCadence(subscription, plan) { + if (plan != null) { + return plan.annual ? 'annual' : 'monthly' + } + + if (isStandaloneAiAddOnPlanCode(subscription.planCode)) { + return subscription.planCode.includes('annual') ? 'annual' : 'monthly' + } + + return '' +} + +function userHasPremiumAiFeatures(user) { + return ( + user?.features?.aiErrorAssistant === true || + user?.features?.aiUsageQuota === Settings.aiFeatures.unlimitedQuota + ) +} + +async function userHasCurrentInstitutionLicence(userId, commonsCache) { + if (commonsCache.has(userId)) { + return commonsCache.get(userId) + } + + try { + const institutions = + await InstitutionsGetter.promises.getCurrentInstitutionsWithLicence( + userId + ) + const hasCommons = Boolean(institutions?.length) + commonsCache.set(userId, hasCommons) + return hasCommons + } catch (error) { + console.warn( + `Failed to evaluate commons licence for user ${userId}: ${error.message}` + ) + commonsCache.set(userId, false) + return false + } +} + +function isMemberOfGroupSubscription(candidates) { + return candidates.some(candidate => candidate.subscription?.groupPlan) +} + +function getAiPlanForUser({ + bestSubscription, + planType, + individualSubscription, + paymentRecord, + user, + userIsMemberOfGroupSubscription, + userHasActiveOverleafSubscription, +}) { + const baseAiPlan = CustomerIoPlanHelpers.getAiPlanType( + bestSubscription, + individualSubscription, + paymentRecord, + user?.writefull, + userIsMemberOfGroupSubscription + ) + + if (baseAiPlan !== 'none') { + return baseAiPlan + } + + if (!userHasActiveOverleafSubscription && userHasPremiumAiFeatures(user)) { + return 'ai-assist' + } + + return 'none' +} + +function getPaymentState(subscription) { + if (subscription?.recurlyStatus?.state) { + return subscription.recurlyStatus.state + } + + if (subscription?.paymentProvider?.state) { + return subscription.paymentProvider.state + } + + return null +} + +function isActivePaidSubscription(subscription) { + const hasRecurlySubscription = Boolean(subscription?.recurlySubscription_id) + const hasStripeSubscription = Boolean( + subscription?.paymentProvider?.subscriptionId + ) + + if (!hasRecurlySubscription && !hasStripeSubscription) { + return false + } + + return ACTIVE_SUBSCRIPTION_STATES.includes(getPaymentState(subscription)) +} + +function getGroupAiEnabled(candidates) { + const groupCandidates = candidates.filter( + candidate => getPlanType(candidate.subscription) === 'group' + ) + + if (groupCandidates.length === 0) { + return '' + } + + return groupCandidates.some(candidate => + CustomerIoPlanHelpers.hasPlanAiEnabled(candidate.plan) + ) +} + +function getGroupRole(candidates, userId) { + const groupCandidates = candidates.filter( + candidate => candidate.subscription?.groupPlan + ) + + if (groupCandidates.length === 0) { + return '' + } + + const isGroupAdminOrManager = groupCandidates.some(candidate => { + const subscription = candidate.subscription + const adminId = subscription?.admin_id?.toString() + const managerIds = (subscription?.manager_ids || []).map(id => + id?.toString() + ) + + return adminId === userId || managerIds.includes(userId) + }) + + return isGroupAdminOrManager ? 'admin' : 'member' +} + +function chooseBestCandidate(candidates) { + let best = null + + for (const candidate of candidates) { + if (best == null) { + best = candidate + continue + } + + const candidateType = getPlanType(candidate.subscription) + const bestType = getPlanType(best.subscription) + + if (candidateType === 'standalone-ai-add-on' && bestType !== 'free') { + continue + } + + if ( + candidateType !== 'standalone-ai-add-on' && + bestType === 'standalone-ai-add-on' + ) { + best = candidate + continue + } + + if ( + FeaturesHelper.isFeatureSetBetter( + candidate.plan?.features || {}, + best.plan?.features || {} + ) + ) { + best = candidate + } + } + + return best +} + +function getSubscriptionQuery() { + return { + $or: [ + { + recurlySubscription_id: { $exists: true, $nin: ['', null] }, + 'recurlyStatus.state': { $in: ACTIVE_SUBSCRIPTION_STATES }, + }, + { + 'paymentProvider.subscriptionId': { $exists: true, $nin: ['', null] }, + 'paymentProvider.state': { $in: ACTIVE_SUBSCRIPTION_STATES }, + }, + ], + } +} + +function getSubscriptionProjection() { + return { + _id: 1, + admin_id: 1, + manager_ids: 1, + member_ids: 1, + planCode: 1, + groupPlan: 1, + recurlySubscription_id: 1, + recurlyStatus: 1, + paymentProvider: 1, + addOns: 1, + } +} + +function getSupplementaryUserQuery() { + return { + $or: [ + { 'writefull.isPremium': true }, + { 'features.aiErrorAssistant': true }, + { 'features.aiUsageQuota': Settings.aiFeatures.unlimitedQuota }, + { + emails: { + $elemMatch: { + confirmedAt: { $exists: true }, + 'affiliation.institution.confirmed': true, + 'affiliation.licence': { $exists: true, $ne: 'free' }, + 'affiliation.pastReconfirmDate': { $ne: true }, + }, + }, + }, + ], + } +} + +function getTargetUserIdsPipeline(resumeAfterUserId) { + const pipeline = [ + { $match: getSubscriptionQuery() }, + { + $project: { + participantIds: { + $setUnion: [ + [{ $ifNull: ['$admin_id', null] }], + { $ifNull: ['$manager_ids', []] }, + { $ifNull: ['$member_ids', []] }, + ], + }, + }, + }, + { $unwind: '$participantIds' }, + { $match: { participantIds: { $ne: null } } }, + { $group: { _id: '$participantIds' } }, + { + $unionWith: { + coll: 'users', + pipeline: [ + { $match: getSupplementaryUserQuery() }, + { $project: { _id: 1 } }, + ], + }, + }, + { $group: { _id: '$_id' } }, + ] + + if (resumeAfterUserId) { + let resumeId = resumeAfterUserId + try { + resumeId = new ObjectId(resumeAfterUserId) + } catch { + // leave as-is for non-ObjectId identifiers + } + pipeline.push({ $match: { _id: { $gt: resumeId } } }) + } + + pipeline.push({ $sort: { _id: 1 } }) + + return pipeline +} + +function getTargetUserIdsCursor(resumeAfterUserId) { + return db.subscriptions.aggregate( + getTargetUserIdsPipeline(resumeAfterUserId), + { + allowDiskUse: true, + readPreference: READ_PREFERENCE_SECONDARY, + } + ) +} + +function isInvalidOrMissingSubscriptionError(error) { + const message = String(error?.message || '').toLowerCase() + + return ( + message.includes('no such subscription') || + message.includes('invalid subscription') || + message.includes('subscription not found') + ) +} + +async function getSubscriptionPaymentInfo( + userId, + subscription, + paymentInfoCache +) { + const subscriptionId = subscription._id.toString() + if (paymentInfoCache.has(subscriptionId)) { + return { + ...paymentInfoCache.get(subscriptionId), + skip: false, + } + } + + try { + const paymentRecord = + await PaymentService.promises.getPaymentFromRecord(subscription) + const nextRenewalDate = + CustomerIoPlanHelpers.getNextRenewalDateFromPaymentRecord( + paymentRecord + ) || '' + const paymentInfo = { nextRenewalDate, paymentRecord } + paymentInfoCache.set(subscriptionId, paymentInfo) + return { ...paymentInfo, skip: false } + } catch (error) { + if (isInvalidOrMissingSubscriptionError(error)) { + console.warn( + `Skipping user ${userId}: invalid/missing payment-provider subscription for subscription ${subscriptionId} (${error.message})` + ) + return { nextRenewalDate: '', paymentRecord: null, skip: true } + } + + console.error( + `Failed to get renewal date for subscription ${subscriptionId}:`, + error.message + ) + const paymentInfo = { nextRenewalDate: '', paymentRecord: null } + paymentInfoCache.set(subscriptionId, paymentInfo) + return { ...paymentInfo, skip: false } + } +} + +async function getUsersByIds(userIds) { + const objectIds = [] + for (const userId of userIds) { + try { + objectIds.push(new ObjectId(userId)) + } catch { + // ignore invalid ObjectId strings + } + } + + const idFilters = [] + if (objectIds.length > 0) { + idFilters.push({ _id: { $in: objectIds } }) + } + if (userIds.length > 0) { + idFilters.push({ _id: { $in: userIds } }) + } + + if (idFilters.length === 0) { + return new Map() + } + + const users = await db.users + .find(idFilters.length === 1 ? idFilters[0] : { $or: idFilters }, { + projection: { + _id: 1, + email: 1, + features: 1, + writefull: 1, + }, + readPreference: READ_PREFERENCE_SECONDARY, + }) + .toArray() + + return new Map(users.map(user => [user._id.toString(), user])) +} + +async function getActiveSubscriptionsForUserIds(userIds) { + const objectIds = [] + for (const userId of userIds) { + try { + objectIds.push(new ObjectId(userId)) + } catch { + // ignore invalid ObjectId strings + } + } + + const idValues = objectIds.length > 0 ? objectIds : userIds + + return await db.subscriptions + .find( + { + ...getSubscriptionQuery(), + $or: [ + { admin_id: { $in: idValues } }, + { manager_ids: { $in: idValues } }, + { member_ids: { $in: idValues } }, + ], + }, + { + projection: getSubscriptionProjection(), + readPreference: READ_PREFERENCE_SECONDARY, + } + ) + .toArray() +} + +function buildUserCandidatesMap(subscriptions, allowedUserIds) { + const userCandidates = new Map() + const allowedUserIdsSet = new Set(allowedUserIds) + + function addCandidate(userId, candidate) { + if (!userId || !allowedUserIdsSet.has(userId)) { + return + } + if (!userCandidates.has(userId)) { + userCandidates.set(userId, []) + } + userCandidates.get(userId).push(candidate) + } + + for (const subscription of subscriptions) { + const plan = getPlan(subscription.planCode) + const baseCandidate = { subscription, plan } + + const adminId = subscription.admin_id?.toString() + addCandidate(adminId, baseCandidate) + + if (Array.isArray(subscription.manager_ids)) { + for (const managerIdRaw of subscription.manager_ids) { + addCandidate(managerIdRaw?.toString(), baseCandidate) + } + } + + if (subscription.groupPlan && Array.isArray(subscription.member_ids)) { + for (const memberIdRaw of subscription.member_ids) { + addCandidate(memberIdRaw?.toString(), baseCandidate) + } + } + } + + return userCandidates +} + +function writeCsvRows(writeStream, rows, includeHeader) { + if (rows.length === 0) { + return + } + + const csvParser = new CSVParser({ + fields: includeHeader ? CSV_FIELDS : CSV_FIELD_NAMES, + header: includeHeader, + eol: '\n', + }) + + writeStream.write(`${csvParser.parse(rows)}\n`) +} + +function writeCsvHeader(writeStream) { + writeStream.write(`${CSV_FIELDS.map(field => field.label).join(',')}\n`) +} + +async function processUserBatch({ + userIds, + writeStream, + paymentInfoCache, + commonsCache, + limit, + trackProgress, + totalUsersCount, + globalState, + checkpointInterval, +}) { + const usersById = await getUsersByIds(userIds) + const subscriptions = await getActiveSubscriptionsForUserIds(userIds) + const activeSubscriptions = subscriptions.filter(isActivePaidSubscription) + const userCandidates = buildUserCandidatesMap(activeSubscriptions, userIds) + + const rows = await Promise.all( + userIds.map(userId => + limit(async () => { + const user = usersById.get(userId) + if (!user?.email) { + return null + } + + const candidates = userCandidates.get(userId) || [] + const bestCandidate = chooseBestCandidate(candidates) + + const hasCommons = await userHasCurrentInstitutionLicence( + userId, + commonsCache + ) + const commonsBeatsBestSubscription = + CustomerIoPlanHelpers.shouldUseCommonsBestSubscription( + hasCommons, + bestCandidate, + COMMONS_PLAN + ) + + const resolvedSubscription = commonsBeatsBestSubscription + ? null + : bestCandidate?.subscription + const resolvedPlan = commonsBeatsBestSubscription + ? COMMONS_PLAN + : bestCandidate?.plan + + const bestSubscriptionForPlanType = commonsBeatsBestSubscription + ? { type: 'commons', plan: COMMONS_PLAN } + : resolvedSubscription + ? { + type: getPlanType(resolvedSubscription), + plan: resolvedPlan, + } + : { type: 'free' } + const planType = CustomerIoPlanHelpers.normalizePlanType( + bestSubscriptionForPlanType + ) + const displayPlanType = + CustomerIoPlanHelpers.getFriendlyPlanName(planType) || '' + const planTerm = resolvedSubscription + ? getPlanCadence(resolvedSubscription, resolvedPlan) + : '' + + const userIsMemberOfGroupSubscription = + isMemberOfGroupSubscription(candidates) + const userHasActiveOverleafSubscription = bestCandidate != null + let nextRenewalDate = '' + let expiryDate = '' + let paymentRecord = null + if (resolvedSubscription) { + const paymentInfo = await getSubscriptionPaymentInfo( + userId, + resolvedSubscription, + paymentInfoCache + ) + if (paymentInfo.skip) { + return null + } + nextRenewalDate = paymentInfo.nextRenewalDate + paymentRecord = paymentInfo.paymentRecord + expiryDate = + CustomerIoPlanHelpers.getExpiryDateFromPaymentRecord( + paymentRecord + ) || '' + } + + const aiPlan = getAiPlanForUser({ + bestSubscription: bestSubscriptionForPlanType, + planType, + individualSubscription: resolvedSubscription, + paymentRecord, + user, + userIsMemberOfGroupSubscription, + userHasActiveOverleafSubscription, + }) + const aiPlanTerm = CustomerIoPlanHelpers.getAiPlanCadence( + aiPlan, + bestSubscriptionForPlanType, + resolvedSubscription, + paymentRecord + ) + const groupAIEnabled = getGroupAiEnabled(candidates) + const groupRole = getGroupRole(candidates, userId) + + return { + userId, + email: user.email, + planType, + displayPlanType, + preMigrationPlanType: planType, + preMigrationDisplayPlanType: displayPlanType, + planTerm, + aiPlan, + aiPlanTerm: aiPlanTerm || '', + nextRenewalDate, + expiryDate, + groupAIEnabled, + groupRole, + } + }) + ) + ) + + const csvRows = [] + + for (const row of rows) { + globalState.processedCount += 1 + if (row) { + csvRows.push(row) + globalState.writtenCount += 1 + } else { + globalState.skippedCount += 1 + } + } + + writeCsvRows(writeStream, csvRows, globalState.shouldWriteHeader) + if (csvRows.length > 0) { + globalState.shouldWriteHeader = false + } + + globalState.lastProcessedUserId = userIds[userIds.length - 1] + + if (globalState.processedCount % checkpointInterval < userIds.length) { + await trackProgress( + `Checkpoint: processed=${globalState.processedCount}/${totalUsersCount || '?'} written=${globalState.writtenCount} skipped=${globalState.skippedCount} resumeAfterUserId=${globalState.lastProcessedUserId}` + ) + } +} + +async function main(trackProgress) { + const { + outputPath, + concurrency, + batchSize, + resumeAfterUserId, + checkpointInterval, + append, + } = parseArgs() + + await trackProgress( + 'Building target user cursor (subscriptions + supplementary users)' + ) + if (resumeAfterUserId) { + await trackProgress( + `Resume enabled: starting after user id ${resumeAfterUserId}` + ) + } + + const paymentInfoCache = new Map() + const commonsCache = new Map() + const limit = pLimit(Number(concurrency) || 5) + const resolvedBatchSize = Math.max(1, Number(batchSize) || 500) + const resolvedCheckpointInterval = Math.max( + 100, + Number(checkpointInterval) || 10000 + ) + + const outputFileExists = fs.existsSync(outputPath) + const outputFileHasContent = + outputFileExists && fs.statSync(outputPath).size > 0 + const writeStream = fs.createWriteStream(outputPath, { + flags: append ? 'a' : 'w', + }) + + const globalState = { + processedCount: 0, + writtenCount: 0, + skippedCount: 0, + lastProcessedUserId: resumeAfterUserId || null, + shouldWriteHeader: !append || !outputFileHasContent, + } + + let pendingUserIds = [] + const targetUserIdsCursor = getTargetUserIdsCursor(resumeAfterUserId) + + for await (const doc of targetUserIdsCursor) { + pendingUserIds.push(doc._id.toString()) + + if (pendingUserIds.length >= resolvedBatchSize) { + await processUserBatch({ + userIds: pendingUserIds, + writeStream, + paymentInfoCache, + commonsCache, + limit, + trackProgress, + totalUsersCount: null, + globalState, + checkpointInterval: resolvedCheckpointInterval, + }) + pendingUserIds = [] + } + } + + if (pendingUserIds.length > 0) { + await processUserBatch({ + userIds: pendingUserIds, + writeStream, + paymentInfoCache, + commonsCache, + limit, + trackProgress, + totalUsersCount: null, + globalState, + checkpointInterval: resolvedCheckpointInterval, + }) + } + + if (globalState.shouldWriteHeader) { + writeCsvHeader(writeStream) + globalState.shouldWriteHeader = false + } + + writeStream.end() + + await trackProgress( + `Final checkpoint: processed=${globalState.processedCount} written=${globalState.writtenCount} skipped=${globalState.skippedCount} resumeAfterUserId=${globalState.lastProcessedUserId}` + ) + await trackProgress(`CSV generated: ${outputPath}`) + console.log(`✅ Export complete: ${outputPath}`) + console.log(`Rows written: ${globalState.writtenCount}`) +} + +try { + await scriptRunner(main) + process.exit(0) +} catch (error) { + console.error(error) + process.exit(1) +} diff --git a/services/web/test/unit/src/Project/ProjectListController.test.mjs b/services/web/test/unit/src/Project/ProjectListController.test.mjs index c966e0a237..989f1afd53 100644 --- a/services/web/test/unit/src/Project/ProjectListController.test.mjs +++ b/services/web/test/unit/src/Project/ProjectListController.test.mjs @@ -467,6 +467,34 @@ describe('ProjectListController', function () { await ctx.ProjectListController.projectListPage(ctx.req, ctx.res) }) + it('should send groupRole to customer.io for group admins', async function (ctx) { + ctx.Features.hasFeature.withArgs('saas').returns(true) + ctx.SubscriptionViewModelBuilder.promises.getUsersSubscriptionDetails.resolves( + { + bestSubscription: { type: 'free' }, + individualSubscription: null, + memberGroupSubscriptions: [], + managedGroupSubscriptions: [ + { + planCode: 'group_professional', + membersLimit: 12, + }, + ], + } + ) + ctx.res.render = () => {} + + await ctx.ProjectListController.projectListPage(ctx.req, ctx.res) + + expect(ctx.Modules.promises.hooks.fire).to.have.been.calledWith( + 'setUserProperties', + ctx.user._id, + sinon.match({ + group_role: 'admin', + }) + ) + }) + it('should show INR Banner for Indian users with free account', async function (ctx) { // usersBestSubscription is only available when saas feature is present ctx.Features.hasFeature.withArgs('saas').returns(true) diff --git a/services/web/test/unit/src/Subscription/FeaturesUpdater.test.mjs b/services/web/test/unit/src/Subscription/FeaturesUpdater.test.mjs index e8cef6f13b..f60d2ad451 100644 --- a/services/web/test/unit/src/Subscription/FeaturesUpdater.test.mjs +++ b/services/web/test/unit/src/Subscription/FeaturesUpdater.test.mjs @@ -9,6 +9,7 @@ const MODULE_PATH = '../../../../app/src/Features/Subscription/FeaturesUpdater' describe('FeaturesUpdater', function () { beforeEach(async function (ctx) { + ctx.renewalDate = new Date('2099-04-01T00:00:00Z') ctx.v1UserId = 12345 ctx.user = { _id: new ObjectId(), @@ -17,7 +18,12 @@ describe('FeaturesUpdater', function () { } ctx.aiAddOn = { addOnCode: AI_ADD_ON_CODE, quantity: 1 } ctx.subscriptions = { - individual: { planCode: 'individual-plan' }, + individual: { + planCode: 'individual-plan', + groupPlan: false, + recurlySubscription_id: 'sub-individual', + recurlyStatus: { state: 'active' }, + }, group1: { planCode: 'group-plan-1', groupPlan: true }, group2: { planCode: 'group-plan-2', groupPlan: true }, noDropbox: { planCode: 'no-dropbox' }, @@ -115,6 +121,7 @@ describe('FeaturesUpdater', function () { ctx.UserGetter = { promises: { getUser: sinon.stub().resolves(null), + getWritefullData: sinon.stub().resolves(null), }, } ctx.UserGetter.promises.getUser.withArgs(ctx.user._id).resolves(ctx.user) @@ -126,12 +133,25 @@ describe('FeaturesUpdater', function () { setUserPropertyForUserInBackground: sinon.stub(), } ctx.Modules = { - promises: { hooks: { fire: sinon.stub().resolves() } }, + promises: { hooks: { fire: sinon.stub().resolves([]) } }, } + ctx.Modules.promises.hooks.fire + .withArgs('getPaymentFromRecordPromise', ctx.subscriptions.individual) + .resolves([ + { + subscription: { + state: 'active', + periodEnd: ctx.renewalDate, + }, + }, + ]) ctx.SubscriptionViewModelBuilder = { promises: { getUsersSubscriptionDetails: sinon.stub().resolves({ bestSubscription: { type: 'individual' }, + individualSubscription: ctx.subscriptions.individual, + memberGroupSubscriptions: [], + managedGroupSubscriptions: [], }), }, } @@ -419,36 +439,115 @@ describe('FeaturesUpdater', function () { ).to.have.been.calledWith(ctx.user._id, 'feature-set', 'all') }) - it('should sync features to customer.io', function (ctx) { + it('should sync subscription properties to customer.io', function (ctx) { expect(ctx.Modules.promises.hooks.fire).to.have.been.calledWith( 'setUserProperties', ctx.user._id, - { - features: ctx.Settings.features.all, - 'best-subscription-type': 'individual', - overleafId: ctx.user._id, - } + sinon.match({ + plan_type: 'individual', + display_plan_type: 'individual', + ai_plan: 'none', + next_renewal_date: Math.floor(ctx.renewalDate.getTime() / 1000), + expiry_date: '', + features: sinon.match.object, + }) ) }) }) - describe('when user has an email', function () { + describe('when the individual subscription has a pending cancellation', function () { beforeEach(async function (ctx) { - ctx.user.email = 'user@example.com' + const pendingCancellationSubscription = { + ...ctx.subscriptions.individual, + recurlyStatus: { state: 'canceled' }, + } + ctx.SubscriptionViewModelBuilder.promises.getUsersSubscriptionDetails.resolves( + { + bestSubscription: { type: 'individual' }, + individualSubscription: pendingCancellationSubscription, + memberGroupSubscriptions: [], + managedGroupSubscriptions: [], + } + ) + ctx.Modules.promises.hooks.fire + .withArgs( + 'getPaymentFromRecordPromise', + pendingCancellationSubscription + ) + .resolves([ + { + subscription: { + state: 'canceled', + periodEnd: ctx.renewalDate, + }, + }, + ]) + await ctx.FeaturesUpdater.promises.refreshFeatures(ctx.user._id, 'test') }) - it('should include email in customer.io properties', function (ctx) { + it('should sync expiry_date and blank next_renewal_date in customer.io', function (ctx) { expect(ctx.Modules.promises.hooks.fire).to.have.been.calledWith( 'setUserProperties', ctx.user._id, + sinon.match({ + plan_type: 'individual', + display_plan_type: 'individual', + ai_plan: 'none', + next_renewal_date: '', + expiry_date: Math.floor(ctx.renewalDate.getTime() / 1000), + features: sinon.match.object, + }) + ) + }) + }) + + describe('when the user is in a group subscription', function () { + beforeEach(async function (ctx) { + ctx.SubscriptionViewModelBuilder.promises.getUsersSubscriptionDetails.resolves( { - features: ctx.Settings.features.all, - 'best-subscription-type': 'individual', - overleafId: ctx.user._id, - email: 'user@example.com', + bestSubscription: { + type: 'group', + plan: { + planCode: 'group-plan-1', + groupPlan: true, + membersLimit: 5, + }, + subscription: { + teamName: 'Team Alpha', + }, + }, + memberGroupSubscriptions: [ + { + planCode: 'group-plan-1', + teamName: 'Team Alpha', + membersLimit: 8, + }, + ], + managedGroupSubscriptions: [], + individualSubscription: null, } ) + await ctx.FeaturesUpdater.promises.refreshFeatures(ctx.user._id, 'test') + }) + + it('should sync groupSize to customer.io', function (ctx) { + expect(ctx.Modules.promises.hooks.fire).to.have.been.calledWith( + 'setUserProperties', + ctx.user._id, + sinon.match({ + plan_type: 'group-standard', + display_plan_type: 'Group Standard', + plan_term_label: 'monthly', + ai_plan: 'none', + group_ai_enabled: false, + group_size: 8, + next_renewal_date: '', + expiry_date: '', + features: sinon.match.object, + overleaf_id: ctx.user._id, + }) + ) }) }) diff --git a/services/web/test/unit/src/Templates/TemplatesController.test.mjs b/services/web/test/unit/src/Templates/TemplatesController.test.mjs index 5d354718b5..1ced1067c4 100644 --- a/services/web/test/unit/src/Templates/TemplatesController.test.mjs +++ b/services/web/test/unit/src/Templates/TemplatesController.test.mjs @@ -1,6 +1,5 @@ import { vi, expect } from 'vitest' import sinon from 'sinon' -import ProjectHelper from '../../../../app/src/Features/Project/ProjectHelper.mjs' const modulePath = '../../../../app/src/Features/Templates/TemplatesController.mjs' @@ -9,8 +8,12 @@ describe('TemplatesController', function () { beforeEach(async function (ctx) { ctx.user_id = 'user-id' + ctx.ProjectHelper = { + compilerFromV1Engine: sinon.stub(), + } + vi.doMock('../../../../app/src/Features/Project/ProjectHelper', () => ({ - default: ProjectHelper, + default: ctx.ProjectHelper, })) vi.doMock(