diff --git a/services/web/app/src/Features/SplitTests/SplitTestCache.js b/services/web/app/src/Features/SplitTests/SplitTestCache.js new file mode 100644 index 0000000000..57b447a7b9 --- /dev/null +++ b/services/web/app/src/Features/SplitTests/SplitTestCache.js @@ -0,0 +1,25 @@ +const SplitTestManager = require('./SplitTestManager') +const { SplitTest } = require('../../models/SplitTest') +const { CacheLoader } = require('cache-flow') + +class SplitTestCache extends CacheLoader { + constructor() { + super('split-test', { + expirationTime: 60, // 1min in seconds + }) + } + + async load(name) { + return await SplitTestManager.getSplitTestByName(name) + } + + serialize(value) { + return value.toObject() + } + + deserialize(value) { + return new SplitTest(value) + } +} + +module.exports = new SplitTestCache() diff --git a/services/web/app/src/Features/SplitTests/SplitTestController.js b/services/web/app/src/Features/SplitTests/SplitTestController.js new file mode 100644 index 0000000000..fd429cb86d --- /dev/null +++ b/services/web/app/src/Features/SplitTests/SplitTestController.js @@ -0,0 +1,77 @@ +const SplitTestManager = require('./SplitTestManager') + +async function getSplitTests(req, res, next) { + try { + const splitTests = await SplitTestManager.getSplitTests() + res.send(splitTests) + } catch (error) { + res.status(500).json({ + error: `Error while fetching split tests list: ${error.message}`, + }) + } +} + +async function createSplitTest(req, res, next) { + const { name, configuration } = req.body + try { + const splitTest = await SplitTestManager.createSplitTest( + name, + configuration + ) + res.send(splitTest) + } catch (error) { + res + .status(500) + .json({ error: `Error while creating split test: ${error.message}` }) + } +} + +async function updateSplitTest(req, res, next) { + const { name, configuration } = req.body + try { + const splitTest = await SplitTestManager.updateSplitTest( + name, + configuration + ) + res.send(splitTest) + } catch (error) { + res + .status(500) + .json({ error: `Error while updating split test: ${error.message}` }) + } +} + +async function switchToNextPhase(req, res, next) { + const { name } = req.body + try { + const splitTest = await SplitTestManager.switchToNextPhase(name) + res.send(splitTest) + } catch (error) { + res.status(500).json({ + error: `Error while switching split test to next phase: ${error.message}`, + }) + } +} + +async function revertToPreviousVersion(req, res, next) { + const { name, versionNumber } = req.body + try { + const splitTest = await SplitTestManager.revertToPreviousVersion( + name, + versionNumber + ) + res.send(splitTest) + } catch (error) { + res.status(500).json({ + error: `Error while reverting to previous version: ${error.message}`, + }) + } +} + +module.exports = { + getSplitTests, + createSplitTest, + updateSplitTest, + switchToNextPhase, + revertToPreviousVersion, +} diff --git a/services/web/app/src/Features/SplitTests/SplitTestHandler.js b/services/web/app/src/Features/SplitTests/SplitTestHandler.js index ec60b04937..0a08aad2de 100644 --- a/services/web/app/src/Features/SplitTests/SplitTestHandler.js +++ b/services/web/app/src/Features/SplitTests/SplitTestHandler.js @@ -114,8 +114,14 @@ function _getPercentile(userId, splitTestId) { } module.exports = { + /** + * @deprecated: use SplitTestV2Handler.getAssignment instead + */ getTestSegmentation: callbackify(getTestSegmentation), promises: { + /** + * @deprecated: use SplitTestV2Handler.promises.getAssignment instead + */ getTestSegmentation, }, } diff --git a/services/web/app/src/Features/SplitTests/SplitTestManager.js b/services/web/app/src/Features/SplitTests/SplitTestManager.js new file mode 100644 index 0000000000..6a321a377b --- /dev/null +++ b/services/web/app/src/Features/SplitTests/SplitTestManager.js @@ -0,0 +1,230 @@ +const { SplitTest } = require('../../models/SplitTest') +const OError = require('@overleaf/o-error') +const _ = require('lodash') + +const ALPHA_PHASE = 'alpha' +const BETA_PHASE = 'beta' +const RELEASE_PHASE = 'release' + +async function getSplitTests() { + try { + return await SplitTest.find().exec() + } catch (error) { + throw OError.tag(error, 'Failed to get split tests list') + } +} + +async function getSplitTestByName(name) { + try { + return await SplitTest.findOne({ name }).exec() + } catch (error) { + throw OError.tag(error, 'Failed to get split test', { name }) + } +} + +async function createSplitTest(name, configuration) { + const stripedVariants = [] + let stripeStart = 0 + _checkNewVariantsConfiguration([], configuration.variants) + for (const variant of configuration.variants) { + stripedVariants.push({ + name: variant.name, + active: variant.active, + rolloutPercent: variant.rolloutPercent, + rolloutStripes: [ + { + start: stripeStart, + end: stripeStart + variant.rolloutPercent, + }, + ], + }) + stripeStart += variant.rolloutPercent + } + const splitTest = new SplitTest({ + name, + versions: [ + { + versionNumber: 1, + phase: configuration.phase, + active: configuration.active, + variants: stripedVariants, + }, + ], + }) + return _saveSplitTest(splitTest) +} + +async function updateSplitTest(name, configuration) { + const splitTest = await getSplitTestByName(name) + if (splitTest) { + const lastVersion = splitTest.getCurrentVersion().toObject() + if (configuration.phase !== lastVersion.phase) { + throw new OError( + `Cannot update with different phase - use switchToNextPhase endpoint instead` + ) + } + _checkNewVariantsConfiguration(lastVersion.variants, configuration.variants) + const updatedVariants = _updateVariantsWithNewConfiguration( + lastVersion.variants, + configuration.variants + ) + splitTest.versions.push({ + versionNumber: lastVersion.versionNumber + 1, + phase: configuration.phase, + active: configuration.active, + variants: updatedVariants, + }) + return _saveSplitTest(splitTest) + } else { + throw new OError(`Cannot update split test '${name}': not found`) + } +} + +async function switchToNextPhase(name) { + const splitTest = await getSplitTestByName(name) + if (splitTest) { + const lastVersionCopy = splitTest.getCurrentVersion().toObject() + lastVersionCopy.versionNumber++ + if (lastVersionCopy.phase === ALPHA_PHASE) { + lastVersionCopy.phase = BETA_PHASE + } else if (lastVersionCopy.phase === BETA_PHASE) { + if (splitTest.forbidReleasePhase) { + throw new OError('Switch to release phase is disabled for this test') + } + lastVersionCopy.phase = RELEASE_PHASE + } else if (splitTest.phase === RELEASE_PHASE) { + throw new OError( + `Split test with ID '${name}' is already in the release phase` + ) + } + for (const variant of lastVersionCopy.variants) { + variant.rolloutPercent = 0 + variant.rolloutStripes = [] + } + splitTest.versions.push(lastVersionCopy) + return _saveSplitTest(splitTest) + } else { + throw new OError( + `Cannot switch split test with ID '${name}' to next phase: not found` + ) + } +} + +async function revertToPreviousVersion(name, versionNumber) { + const splitTest = await getSplitTestByName(name) + if (splitTest) { + if (splitTest.versions.length <= 1) { + throw new OError( + `Cannot revert split test with ID '${name}' to previous version: split test must have at least 2 versions` + ) + } + const previousVersion = splitTest.getVersion(versionNumber) + if (!previousVersion) { + throw new OError( + `Cannot revert split test with ID '${name}' to version number ${versionNumber}: version not found` + ) + } + const lastVersion = splitTest.getCurrentVersion() + if ( + lastVersion.phase === RELEASE_PHASE && + previousVersion.phase !== RELEASE_PHASE + ) { + splitTest.forbidReleasePhase = true + } + const previousVersionCopy = previousVersion.toObject() + previousVersionCopy.versionNumber = lastVersion.versionNumber + 1 + splitTest.versions.push(previousVersionCopy) + return _saveSplitTest(splitTest) + } else { + throw new OError( + `Cannot revert split test with ID '${name}' to previous version: not found` + ) + } +} + +function _checkNewVariantsConfiguration(variants, newVariantsConfiguration) { + const totalRolloutPercentage = _getTotalRolloutPercentage( + newVariantsConfiguration + ) + if (totalRolloutPercentage > 100) { + throw new OError(`Total variants rollout percentage cannot exceed 100`) + } + for (const variant of variants) { + const newVariantConfiguration = _.find(newVariantsConfiguration, { + name: variant.name, + }) + if (!newVariantConfiguration) { + throw new OError( + `Variant defined in previous version as ${JSON.stringify( + variant + )} cannot be removed in new configuration: either set it inactive or create a new split test` + ) + } + if (newVariantConfiguration.rolloutPercent < variant.rolloutPercent) { + throw new OError( + `Rollout percentage for variant defined in previous version as ${JSON.stringify( + variant + )} cannot be decreased: revert to a previous configuration instead` + ) + } + } +} + +function _updateVariantsWithNewConfiguration( + variants, + newVariantsConfiguration +) { + let totalRolloutPercentage = _getTotalRolloutPercentage(variants) + const variantsCopy = _.clone(variants) + for (const newVariantConfig of newVariantsConfiguration) { + const variant = _.find(variantsCopy, { name: newVariantConfig.name }) + if (!variant) { + variantsCopy.push({ + name: newVariantConfig.name, + active: newVariantConfig.active, + rolloutPercent: newVariantConfig.rolloutPercent, + rolloutStripes: [ + { + start: totalRolloutPercentage, + end: totalRolloutPercentage + newVariantConfig.rolloutPercent, + }, + ], + }) + totalRolloutPercentage += newVariantConfig.rolloutPercent + } else if (variant.rolloutPercent < newVariantConfig.rolloutPercent) { + const newStripeSize = + newVariantConfig.rolloutPercent - variant.rolloutPercent + variant.active = newVariantConfig.active + variant.rolloutPercent = newVariantConfig.rolloutPercent + variant.rolloutStripes.push({ + start: totalRolloutPercentage, + end: totalRolloutPercentage + newStripeSize, + }) + totalRolloutPercentage += newStripeSize + } + } + return variantsCopy +} + +function _getTotalRolloutPercentage(variants) { + return _.sumBy(variants, 'rolloutPercent') +} + +async function _saveSplitTest(splitTest) { + try { + return (await splitTest.save()).toObject() + } catch (error) { + throw OError.tag(error, 'Failed to save split test', { + splitTest: JSON.stringify(splitTest), + }) + } +} + +module.exports = { + getSplitTestByName, + getSplitTests, + createSplitTest, + updateSplitTest, + switchToNextPhase, + revertToPreviousVersion, +} diff --git a/services/web/app/src/Features/SplitTests/SplitTestRouter.js b/services/web/app/src/Features/SplitTests/SplitTestRouter.js new file mode 100644 index 0000000000..af84174886 --- /dev/null +++ b/services/web/app/src/Features/SplitTests/SplitTestRouter.js @@ -0,0 +1,49 @@ +const SplitTestController = require('./SplitTestController') +const AuthorizationMiddleware = require('../Authorization/AuthorizationMiddleware') +const Features = require('../../infrastructure/Features') + +module.exports = { + apply(webRouter) { + if (Features.hasFeature('saas')) { + webRouter.get( + '/admin/splitTests', + AuthorizationMiddleware.ensureUserIsSiteAdmin, + SplitTestController.getSplitTests + ) + + webRouter.post( + '/admin/createSplitTest', + AuthorizationMiddleware.ensureUserIsSiteAdmin, + SplitTestController.createSplitTest + ) + webRouter.csrf.disableDefaultCsrfProtection('/admin/splitTest', 'PUT') + + webRouter.post( + '/admin/updateSplitTest', + AuthorizationMiddleware.ensureUserIsSiteAdmin, + SplitTestController.updateSplitTest + ) + webRouter.csrf.disableDefaultCsrfProtection('/admin/splitTest', 'POST') + + webRouter.post( + '/admin/splitTest/switchToNextPhase', + AuthorizationMiddleware.ensureUserIsSiteAdmin, + SplitTestController.switchToNextPhase + ) + webRouter.csrf.disableDefaultCsrfProtection( + '/admin/splitTest/switchToNextPhase', + 'POST' + ) + + webRouter.post( + '/admin/splitTest/revertToPreviousVersion', + AuthorizationMiddleware.ensureUserIsSiteAdmin, + SplitTestController.revertToPreviousVersion + ) + webRouter.csrf.disableDefaultCsrfProtection( + '/admin/splitTest/revertToPreviousVersion', + 'POST' + ) + } + }, +} diff --git a/services/web/app/src/Features/SplitTests/SplitTestV2Handler.js b/services/web/app/src/Features/SplitTests/SplitTestV2Handler.js new file mode 100644 index 0000000000..01bcc889fc --- /dev/null +++ b/services/web/app/src/Features/SplitTests/SplitTestV2Handler.js @@ -0,0 +1,177 @@ +const UserGetter = require('../User/UserGetter') +const UserUpdater = require('../User/UserUpdater') +const AnalyticsManager = require('../Analytics/AnalyticsManager') +const crypto = require('crypto') +const _ = require('lodash') +const { callbackify } = require('util') +const splitTestCache = require('./SplitTestCache') + +const DEFAULT_VARIANT = 'default' +const ALPHA_PHASE = 'alpha' +const BETA_PHASE = 'beta' + +/** + * Get the assignment of a user to a split test. + * + * @example + * // Assign user and record an event + * + * const assignment = await SplitTestV2Handler.getAssignment(userId, 'example-project') + * if (assignment.variant === 'awesome-new-version') { + * // execute my awesome change + * } + * else { + * // execute the default behaviour (control group) + * } + * // then record an event + * AnalyticsManager.recordEvent(userId, 'example-project-created', { + * projectId: project._id, + * ...assignment.analytics.segmentation + * }) + * + * @param userId the user's ID + * @param splitTestName the unique name of the split test + * @param options {sync: boolean} - for test purposes only, to force the synchronous update of the user's profile + * @returns {Promise<{analytics: {segmentation: {}}, variant: string}|{analytics: {segmentation: {phase, splitTest, variant: string, versionNumber}}, variant: string}>} + */ +async function getAssignment(userId, splitTestName, options) { + const splitTest = await splitTestCache.get(splitTestName) + + if (splitTest) { + const currentVersion = splitTest.getCurrentVersion() + if (currentVersion.active) { + const { + activeForUser, + selectedVariantName, + phase, + versionNumber, + } = await _getAssignmentMetadata(userId, splitTest) + if (activeForUser) { + const assignmentConfig = { + userId, + splitTestName, + variantName: selectedVariantName, + phase, + versionNumber, + } + if (options && options.sync === true) { + await _updateVariantAssignment(assignmentConfig) + } else { + _updateVariantAssignment(assignmentConfig) + } + return { + variant: selectedVariantName, + analytics: { + segmentation: { + splitTest: splitTestName, + variant: selectedVariantName, + phase, + versionNumber, + }, + }, + } + } + } + } + return { + variant: DEFAULT_VARIANT, + analytics: { + segmentation: {}, + }, + } +} + +async function _getAssignmentMetadata(userId, splitTest) { + const currentVersion = splitTest.getCurrentVersion() + const phase = currentVersion.phase + if ([ALPHA_PHASE, BETA_PHASE].includes(phase)) { + const user = await _getUser(userId) + if ( + (phase === ALPHA_PHASE && !(user && user.alphaProgram)) || + (phase === BETA_PHASE && !(user && user.betaProgram)) + ) { + return { + activeForUser: false, + } + } + } + const percentile = _getPercentile(userId, splitTest.name, phase) + const selectedVariantName = _getVariantFromPercentile( + currentVersion.variants, + percentile + ) + return { + activeForUser: true, + selectedVariantName: selectedVariantName || DEFAULT_VARIANT, + phase, + versionNumber: currentVersion.versionNumber, + } +} + +function _getPercentile(userId, splitTestName, splitTestPhase) { + const hash = crypto + .createHash('md5') + .update(userId + splitTestName + splitTestPhase) + .digest('hex') + const hashPrefix = hash.substr(0, 8) + return Math.floor( + ((parseInt(hashPrefix, 16) % 0xffffffff) / 0xffffffff) * 100 + ) +} + +function _getVariantFromPercentile(variants, percentile) { + for (const variant of variants) { + for (const stripe of variant.rolloutStripes) { + if (percentile >= stripe.start && percentile < stripe.end) { + return variant.name + } + } + } +} + +async function _updateVariantAssignment({ + userId, + splitTestName, + phase, + versionNumber, + variantName, +}) { + const user = await _getUser(userId) + if (user) { + const assignedSplitTests = user.splitTests || [] + const assignmentLog = assignedSplitTests[splitTestName] || [] + const existingAssignment = _.find(assignmentLog, { versionNumber }) + if (!existingAssignment) { + await UserUpdater.promises.updateUser(userId, { + $addToSet: { + [`splitTests.${splitTestName}`]: { + variantName, + versionNumber, + phase, + assignedAt: new Date(), + }, + }, + }) + AnalyticsManager.setUserProperty( + userId, + `split-test-${splitTestName}-${versionNumber}`, + variantName + ) + } + } +} + +async function _getUser(id) { + return UserGetter.promises.getUser(id, { + splitTests: 1, + alphaProgram: 1, + betaProgram: 1, + }) +} + +module.exports = { + getAssignment: callbackify(getAssignment), + promises: { + getAssignment, + }, +} diff --git a/services/web/app/src/models/SplitTest.js b/services/web/app/src/models/SplitTest.js new file mode 100644 index 0000000000..dc998f6d4d --- /dev/null +++ b/services/web/app/src/models/SplitTest.js @@ -0,0 +1,111 @@ +const mongoose = require('../infrastructure/Mongoose') +const { Schema } = mongoose +const _ = require('lodash') + +const MIN_NAME_LENGTH = 3 +const MAX_NAME_LENGTH = 200 +const MIN_VARIANT_NAME_LENGTH = 3 +const MAX_VARIANT_NAME_LENGTH = 255 +const NAME_REGEX = /^[a-zA-Z0-9\-_]+$/ + +const RolloutPercentType = { + type: Number, + default: 0, + min: [0, 'Rollout percentage must be between 0 and 100, got {VALUE}'], + max: [100, 'Rollout percentage must be between 0 and 100, got {VALUE}'], + required: true, +} + +const VariantSchema = new Schema( + { + name: { + type: String, + minLength: MIN_VARIANT_NAME_LENGTH, + maxLength: MAX_VARIANT_NAME_LENGTH, + required: true, + validate: { + validator: function (input) { + return input !== null && input !== 'default' && NAME_REGEX.test(input) + }, + message: `invalid, cannot be 'default' and must match: ${NAME_REGEX}, got {VALUE}`, + }, + }, + active: { + type: Boolean, + default: true, + required: true, + }, + rolloutPercent: RolloutPercentType, + rolloutStripes: [ + { + start: RolloutPercentType, + end: RolloutPercentType, + }, + ], + }, + { _id: false } +) + +const VersionSchema = new Schema( + { + versionNumber: { + type: Number, + default: 1, + min: [1, 'must be 1 or higher, got {VALUE}'], + required: true, + }, + phase: { + type: String, + default: 'alpha', + enum: ['alpha', 'beta', 'release'], + required: true, + }, + active: { + type: Boolean, + default: true, + required: true, + }, + variants: [VariantSchema], + }, + { _id: false } +) + +const SplitTestSchema = new Schema({ + name: { + type: String, + minLength: MIN_NAME_LENGTH, + maxlength: MAX_NAME_LENGTH, + required: true, + unique: true, + validate: { + validator: function (input) { + return input !== null && NAME_REGEX.test(input) + }, + message: `invalid, must match: ${NAME_REGEX}`, + }, + }, + versions: [VersionSchema], + forbidReleasePhase: { + type: Boolean, + required: false, + }, +}) + +SplitTestSchema.methods.getCurrentVersion = function () { + if (this.versions && this.versions.length > 0) { + return _.maxBy(this.versions, 'versionNumber') + } else { + return undefined + } +} + +SplitTestSchema.methods.getVersion = function (versionNumber) { + return _.find(this.versions || [], { + versionNumber, + }) +} + +module.exports = { + SplitTest: mongoose.model('SplitTest', SplitTestSchema), + SplitTestSchema, +} diff --git a/services/web/app/src/router.js b/services/web/app/src/router.js index 0e5657c85e..38d8df3d07 100644 --- a/services/web/app/src/router.js +++ b/services/web/app/src/router.js @@ -49,6 +49,7 @@ const InstitutionsController = require('./Features/Institutions/InstitutionsCont const UserMembershipRouter = require('./Features/UserMembership/UserMembershipRouter') const SystemMessageController = require('./Features/SystemMessages/SystemMessageController') const AnalyticsRegistrationSourceMiddleware = require('./Features/Analytics/AnalyticsRegistrationSourceMiddleware') +const SplitTestRouter = require('./Features/SplitTests/SplitTestRouter') const { Joi, validate } = require('./infrastructure/Validation') const { renderUnsupportedBrowserPage, @@ -107,6 +108,7 @@ function initialize(webRouter, privateApiRouter, publicApiRouter) { LinkedFilesRouter.apply(webRouter, privateApiRouter, publicApiRouter) TemplatesRouter.apply(webRouter) UserMembershipRouter.apply(webRouter) + SplitTestRouter.apply(webRouter) Modules.applyRouter(webRouter, privateApiRouter, publicApiRouter) diff --git a/services/web/package-lock.json b/services/web/package-lock.json index 2fb0d81388..498e48d425 100644 --- a/services/web/package-lock.json +++ b/services/web/package-lock.json @@ -14856,6 +14856,74 @@ } } }, + "cache-flow": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/cache-flow/-/cache-flow-1.7.4.tgz", + "integrity": "sha512-EbU+Gcqddasv1hkUtqMTcEjP5HF+Gu7o0qDikaahsYwnu8ysusJfGa1GtUM/rfimFY7QrOO3p5sIFALrztyt0w==", + "requires": { + "cluster": "^0.7.7", + "date-fns": "^2.16.1", + "ioredis": "^4.19.2", + "lru-cache-for-clusters-as-promised": "^1.5.24", + "object-hash": "^2.0.3", + "typescript": "^4.0.5" + }, + "dependencies": { + "debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "requires": { + "ms": "2.1.2" + } + }, + "ioredis": { + "version": "4.27.6", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.27.6.tgz", + "integrity": "sha512-6W3ZHMbpCa8ByMyC1LJGOi7P2WiOKP9B3resoZOVLDhi+6dDBOW+KNsRq3yI36Hmnb2sifCxHX+YSarTeXh48A==", + "requires": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "p-map": "^2.1.0", + "redis-commands": "1.7.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "p-map": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", + "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==" + }, + "redis-commands": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", + "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "requires": { + "redis-errors": "^1.0.0" + } + }, + "standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + } + } + }, "cacheable-request": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", @@ -15049,6 +15117,15 @@ "check-error": "^1.0.2" } }, + "chai-exclude": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/chai-exclude/-/chai-exclude-2.0.3.tgz", + "integrity": "sha512-6VuTQX25rsh4hKPdLzsOtL20k9+tszksLQrLtsu6szTmSVJP9+gUkqYUsyM+xqCeGZKeRJCsamCMRUQJhWsQ+g==", + "dev": true, + "requires": { + "fclone": "^1.0.11" + } + }, "chaid": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/chaid/-/chaid-1.0.2.tgz", @@ -15467,6 +15544,15 @@ } } }, + "cluster": { + "version": "0.7.7", + "resolved": "https://registry.npmjs.org/cluster/-/cluster-0.7.7.tgz", + "integrity": "sha1-5JfiZ8yVa9CwUTrbSqOTNX0Ahe8=", + "requires": { + "log": ">= 1.2.0", + "mkdirp": ">= 0.0.1" + } + }, "cluster-key-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", @@ -16405,6 +16491,14 @@ } } }, + "cron": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/cron/-/cron-1.8.2.tgz", + "integrity": "sha512-Gk2c4y6xKEO8FSAUTklqtfSr7oTq0CiPQeLBG5Fl0qoXpZyMcj1SG59YL+hqq04bu6/IuEA7lMkYDAplQNKkyg==", + "requires": { + "moment-timezone": "^0.5.x" + } + }, "cron-parser": { "version": "2.16.3", "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.16.3.tgz", @@ -16878,6 +16972,15 @@ "integrity": "sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A==", "dev": true }, + "d": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", + "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", + "requires": { + "es5-ext": "^0.10.50", + "type": "^1.0.1" + } + }, "d3": { "version": "3.5.16", "resolved": "https://registry.npmjs.org/d3/-/d3-3.5.16.tgz", @@ -16886,7 +16989,7 @@ "d64": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/d64/-/d64-1.0.0.tgz", - "integrity": "sha512-5eNy3WZziVYnrogqgXhcdEmqcDB2IHurTqLcrgssJsfkMVCUoUaZpK6cJjxxvLV2dUm5SuJMNcYfVGoin9UIRw==" + "integrity": "sha1-QAKofoUMv8n52XBrYPymE6MzbpA=" }, "damerau-levenshtein": { "version": "1.0.6", @@ -16923,6 +17026,11 @@ "resolved": "https://registry.npmjs.org/date-and-time/-/date-and-time-0.14.2.tgz", "integrity": "sha512-EFTCh9zRSEpGPmJaexg7HTuzZHh6cnJj1ui7IGCFNXzd2QdpsNh05Db5TF3xzJm30YN+A8/6xHSuRcQqoc3kFA==" }, + "date-fns": { + "version": "2.22.1", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.22.1.tgz", + "integrity": "sha512-yUFPQjrxEmIsMqlHhAhmxkuH769baF21Kk+nZwZGyrMoyLA+LugaQtC0+Tqf9CBUUULWwUJt6Q5ySI3LJDDCGg==" + }, "date-format": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/date-format/-/date-format-2.1.0.tgz", @@ -17693,6 +17801,15 @@ } } }, + "duration": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/duration/-/duration-0.2.2.tgz", + "integrity": "sha512-06kgtea+bGreF5eKYgI/36A6pLXggY7oR4p1pq4SmdFBn1ReOL5D8RhG64VrqfTTKNucqqtBAwEj8aB88mcqrg==", + "requires": { + "d": "1", + "es5-ext": "~0.10.46" + } + }, "east": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/east/-/east-2.0.2.tgz", @@ -18160,12 +18277,32 @@ "is-symbol": "^1.0.1" } }, + "es5-ext": { + "version": "0.10.53", + "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz", + "integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==", + "requires": { + "es6-iterator": "~2.0.3", + "es6-symbol": "~3.1.3", + "next-tick": "~1.0.0" + } + }, "es5-shim": { "version": "4.5.15", "resolved": "https://registry.npmjs.org/es5-shim/-/es5-shim-4.5.15.tgz", "integrity": "sha512-FYpuxEjMeDvU4rulKqFdukQyZSTpzhg4ScQHrAosrlVpR6GFyaw14f74yn2+4BugniIS0Frpg7TvwZocU4ZMTw==", "dev": true }, + "es6-iterator": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", + "integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=", + "requires": { + "d": "1", + "es5-ext": "^0.10.35", + "es6-symbol": "^3.1.1" + } + }, "es6-promise": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", @@ -18177,6 +18314,15 @@ "integrity": "sha512-EmTr31wppcaIAgblChZiuN/l9Y7DPyw8Xtbg7fIVngn6zMW+IEBJDJngeKC3x6wr0V/vcA2wqeFnaw1bFJbDdA==", "dev": true }, + "es6-symbol": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", + "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", + "requires": { + "d": "^1.0.1", + "ext": "^1.1.2" + } + }, "escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", @@ -19206,6 +19352,15 @@ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==" }, + "event-emitter": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", + "integrity": "sha1-34xp7vFkeSPHFXuc6DhAYQsCzDk=", + "requires": { + "d": "1", + "es5-ext": "~0.10.14" + } + }, "event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", @@ -19545,6 +19700,21 @@ "resolved": "https://registry.npmjs.org/expressionify/-/expressionify-0.9.3.tgz", "integrity": "sha1-/iJnx+hpRXfxP02oML/DyNgXf5I=" }, + "ext": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/ext/-/ext-1.4.0.tgz", + "integrity": "sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A==", + "requires": { + "type": "^2.0.0" + }, + "dependencies": { + "type": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/type/-/type-2.5.0.tgz", + "integrity": "sha512-180WMDQaIMm3+7hGXWf12GtdniDEy7nYcyFMKJn/eZz/6tSLXrUN9V0wKSbMjej0I1WHWbpREDEKHtqPQa9NNw==" + } + } + }, "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -19791,6 +19961,12 @@ "bser": "2.1.1" } }, + "fclone": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/fclone/-/fclone-1.0.11.tgz", + "integrity": "sha1-EOhdo4v+p/xZk0HClu4ddyZu5kA=", + "dev": true + }, "fd-slicer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", @@ -24967,7 +25143,7 @@ "lodash.camelcase": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", - "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" + "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=" }, "lodash.debounce": { "version": "4.0.8", @@ -25073,6 +25249,19 @@ "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==", "dev": true }, + "log": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/log/-/log-6.0.0.tgz", + "integrity": "sha512-sxChESNYJ/EcQv8C7xpmxhtTOngoXuMEqGDAkhXBEmt3MAzM3SM/TmIBOqnMEVdrOv1+VgZoYbo6U2GemQiU4g==", + "requires": { + "d": "^1.0.0", + "duration": "^0.2.2", + "es5-ext": "^0.10.49", + "event-emitter": "^0.3.5", + "sprintf-kit": "^2.0.0", + "type": "^1.0.1" + } + }, "log-driver": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/log-driver/-/log-driver-1.2.7.tgz", @@ -25208,6 +25397,37 @@ "yallist": "^4.0.0" } }, + "lru-cache-for-clusters-as-promised": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/lru-cache-for-clusters-as-promised/-/lru-cache-for-clusters-as-promised-1.7.1.tgz", + "integrity": "sha512-BSfUYlSHpCrPSc20F0mhk+EDSYhTuM5AK0gvj63mtADIdZVcq3KVR1pxdmgeAEXvsV/o0qyHMKbuhVg0OHWWFg==", + "requires": { + "cron": "1.8.2", + "debug": "4.3.1", + "lru-cache": "6.0.0", + "uuid": "8.3.2" + }, + "dependencies": { + "debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + } + } + }, "lz-string": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.4.4.tgz", @@ -26637,6 +26857,11 @@ "integrity": "sha512-AO81vsIO1k1sM4Zrd6Hu7regmJN1NSiAja10gc4bX3F0wd+9rQmcuHQaHVQCYIEC8iFXnE+mavh23GOt7wBgug==", "dev": true }, + "next-tick": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", + "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=" + }, "ngcomponent": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/ngcomponent/-/ngcomponent-4.1.0.tgz", @@ -33791,6 +34016,14 @@ "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true }, + "sprintf-kit": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/sprintf-kit/-/sprintf-kit-2.0.1.tgz", + "integrity": "sha512-2PNlcs3j5JflQKcg4wpdqpZ+AjhQJ2OZEo34NXDtlB0tIPG84xaaXhpA8XFacFiwjKA4m49UOYG83y3hbMn/gQ==", + "requires": { + "es5-ext": "^0.10.53" + } + }, "srcset": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/srcset/-/srcset-2.0.1.tgz", @@ -35805,7 +36038,7 @@ "timed-out": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", - "integrity": "sha512-G7r3AhovYtr5YKOWQkta8RKAPb+J9IsO4uVmzjl8AZwfhs8UcUwTiD6gcJYSgOtzyjvQKrKYn41syHbUWMkafA==" + "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=" }, "timekeeper": { "version": "2.2.0", @@ -36108,6 +36341,11 @@ "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" }, + "type": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", + "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" + }, "type-check": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", @@ -36166,6 +36404,11 @@ "is-typedarray": "^1.0.0" } }, + "typescript": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.5.tgz", + "integrity": "sha512-DqQgihaQ9cUrskJo9kIyW/+g0Vxsk8cDtZ52a3NGh0YNTfpUSArXSohyUGnvbPazEPLu398C0UxmKSOrPumUzA==" + }, "ua-parser-js": { "version": "0.7.21", "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.21.tgz", diff --git a/services/web/package.json b/services/web/package.json index 34419faba1..70d57dc6a5 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -77,6 +77,7 @@ "bufferedstream": "1.6.0", "bull": "^3.18.0", "bunyan": "^1.8.15", + "cache-flow": "^1.7.4", "celebrate": "^10.0.1", "classnames": "^2.2.6", "codemirror": "^5.33.0", @@ -194,6 +195,7 @@ "c8": "^7.2.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", + "chai-exclude": "^2.0.3", "chaid": "^1.0.2", "cheerio": "^1.0.0-rc.3", "copy-webpack-plugin": "^5.1.1", diff --git a/services/web/test/acceptance/src/SplitTestTests.js b/services/web/test/acceptance/src/SplitTestTests.js new file mode 100644 index 0000000000..f6fc97af3a --- /dev/null +++ b/services/web/test/acceptance/src/SplitTestTests.js @@ -0,0 +1,1416 @@ +const SplitTestV2Handler = require('../../../app/src/Features/SplitTests/SplitTestV2Handler') +const User = require('./helpers/User') +const { db } = require('../../../app/src/infrastructure/mongodb') +const { ObjectId } = require('mongodb') +const chai = require('chai') +const chaiExclude = require('chai-exclude') +const { promisify } = require('../../../app/src/util/promises') +const { assert } = chai +const { CacheFlow } = require('cache-flow') +chai.use(chaiExclude) + +describe('SplitTest', function () { + beforeEach(async function () { + this.res = { + send: obj => { + this.sent = obj + }, + status: status => { + this.sentStatus = status + return this.res + }, + } + this.sent = undefined + const UserPromises = User.promises + this.adminUser = new UserPromises({ email: 'admin@example.com' }) + await this.adminUser.ensureUserExists() + await this.adminUser.ensureAdmin() + await this.adminUser.login() + + this.sendAdminRequest = async function ({ method, path, payload }) { + return this.adminUser.doRequest(method, { + uri: path, + json: payload, + }) + } + this.expectResponse = async function ( + { method, path, payload }, + { status, body, excluding, excludingEvery } + ) { + const result = await this.sendAdminRequest({ method, path, payload }) + + assert.equal(result.response.statusCode, status) + if (body) { + if (excludingEvery) { + assert.deepEqualExcludingEvery(result.body, body, excludingEvery) + } else if (excluding) { + assert.deepEqualExcludingEvery(result.body, body, excluding) + } else { + assert.deepEqual(result.body, body) + } + } + } + }) + + describe('Manage split test lifecycle', function () { + it('Create, update and revert', async function () { + const config = { + name: 'split-test-1', + configuration: { + active: true, + phase: 'alpha', + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 10, + }, + ], + }, + } + + const version1 = { + versionNumber: 1, + phase: 'alpha', + active: true, + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 10, + rolloutStripes: [ + { + start: 0, + end: 10, + }, + ], + }, + ], + } + + await this.expectResponse( + { + method: 'POST', + path: '/admin/createSplitTest', + payload: config, + }, + { + status: 200, + body: { + name: 'split-test-1', + versions: [version1], + }, + excludingEvery: ['_id', '__v'], + } + ) + + config.configuration.variants[0].rolloutPercent = 20 + config.configuration.variants.push({ + name: 'variant-2', + active: true, + rolloutPercent: 5, + }) + + const version2 = { + versionNumber: 2, + phase: 'alpha', + active: true, + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 20, + rolloutStripes: [ + { + start: 0, + end: 10, + }, + { + start: 10, + end: 20, + }, + ], + }, + { + name: 'variant-2', + active: true, + rolloutPercent: 5, + rolloutStripes: [ + { + start: 20, + end: 25, + }, + ], + }, + ], + } + await this.expectResponse( + { + method: 'POST', + path: '/admin/updateSplitTest', + payload: config, + }, + { + status: 200, + body: { + name: 'split-test-1', + versions: [version1, version2], + }, + excludingEvery: ['_id', '__v'], + } + ) + + config.configuration.active = false + config.configuration.variants[0].active = false + config.configuration.variants[0].rolloutPercent = 30 + config.configuration.variants[1].rolloutPercent = 30 + + const version3 = { + versionNumber: 3, + phase: 'alpha', + active: false, + variants: [ + { + name: 'variant-1', + active: false, + rolloutPercent: 30, + rolloutStripes: [ + { + start: 0, + end: 10, + }, + { + start: 10, + end: 20, + }, + { + start: 25, + end: 35, + }, + ], + }, + { + name: 'variant-2', + active: true, + rolloutPercent: 30, + rolloutStripes: [ + { + start: 20, + end: 25, + }, + { + start: 35, + end: 60, + }, + ], + }, + ], + } + await this.expectResponse( + { + method: 'POST', + path: '/admin/updateSplitTest', + payload: config, + }, + { + status: 200, + body: { + name: 'split-test-1', + versions: [version1, version2, version3], + }, + excludingEvery: ['_id', '__v'], + } + ) + + const version4 = { + versionNumber: 4, + phase: 'alpha', + active: true, + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 20, + rolloutStripes: [ + { + start: 0, + end: 10, + }, + { + start: 10, + end: 20, + }, + ], + }, + { + name: 'variant-2', + active: true, + rolloutPercent: 5, + rolloutStripes: [ + { + start: 20, + end: 25, + }, + ], + }, + ], + } + await this.expectResponse( + { + method: 'POST', + path: '/admin/splitTest/revertToPreviousVersion', + payload: { + name: 'split-test-1', + versionNumber: 2, + }, + }, + { + status: 200, + body: { + name: 'split-test-1', + versions: [version1, version2, version3, version4], + }, + excludingEvery: ['_id', '__v'], + } + ) + }) + + it('Switch to next phase', async function () { + const version1 = { + versionNumber: 1, + phase: 'alpha', + active: true, + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 50, + rolloutStripes: [ + { + start: 0, + end: 50, + }, + ], + }, + { + name: 'variant-2', + active: true, + rolloutPercent: 50, + rolloutStripes: [ + { + start: 50, + end: 100, + }, + ], + }, + ], + } + await this.expectResponse( + { + method: 'POST', + path: '/admin/createSplitTest', + payload: { + name: 'split-test-3', + configuration: { + active: true, + phase: 'alpha', + variants: [ + { + name: 'variant-1', + rolloutPercent: 50, + active: true, + }, + { + name: 'variant-2', + rolloutPercent: 50, + active: true, + }, + ], + }, + }, + }, + { + status: 200, + body: { + name: 'split-test-3', + versions: [version1], + }, + excluding: ['_id', '__v'], + } + ) + + const version2 = { + versionNumber: 2, + phase: 'beta', + active: true, + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 0, + rolloutStripes: [], + }, + { + name: 'variant-2', + active: true, + rolloutPercent: 0, + rolloutStripes: [], + }, + ], + } + + await this.expectResponse( + { + method: 'POST', + path: '/admin/splitTest/switchToNextPhase', + payload: { + name: 'split-test-3', + }, + }, + { + status: 200, + body: { + name: 'split-test-3', + versions: [version1, version2], + }, + excludingEvery: ['_id', '__v'], + } + ) + + const version3 = { + versionNumber: 3, + phase: 'release', + active: true, + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 0, + rolloutStripes: [], + }, + { + name: 'variant-2', + active: true, + rolloutPercent: 0, + rolloutStripes: [], + }, + ], + } + await this.expectResponse( + { + method: 'POST', + path: '/admin/splitTest/switchToNextPhase', + payload: { + name: 'split-test-3', + }, + }, + { + status: 200, + body: { + name: 'split-test-3', + versions: [version1, version2, version3], + }, + excludingEvery: ['_id', '__v'], + } + ) + }) + + it('Error - update with different rollout phase is not allowed', async function () { + await this.expectResponse( + { + method: 'POST', + path: '/admin/createSplitTest', + payload: { + name: 'split-test-2', + configuration: { + active: true, + phase: 'alpha', + variants: [], + }, + }, + }, + { + status: 200, + body: { + name: 'split-test-2', + versions: [ + { + versionNumber: 1, + phase: 'alpha', + active: true, + variants: [], + }, + ], + }, + excludingEvery: ['_id', '__v'], + } + ) + await this.expectResponse( + { + method: 'POST', + path: '/admin/updateSplitTest', + payload: { + name: 'split-test-2', + configuration: { + active: true, + phase: 'beta', + variants: [], + }, + }, + }, + { + status: 500, + body: { + error: + 'Error while updating split test: Cannot update with different phase - use switchToNextPhase endpoint instead', + }, + } + ) + }) + + it('Error - update with non existing name', async function () { + await this.expectResponse( + { + method: 'POST', + path: '/admin/updateSplitTest', + payload: { + name: 'does-not-exist', + configuration: { + active: true, + phase: 'beta', + variants: [ + { + name: 'foo', + active: true, + rolloutPercent: 50, + }, + ], + }, + }, + }, + { + status: 500, + body: { + error: + "Error while updating split test: Cannot update split test 'does-not-exist': not found", + }, + } + ) + }) + + it('Error - create with missing name in configuration', async function () { + await this.expectResponse( + { + method: 'POST', + path: '/admin/createSplitTest', + payload: { + configuration: { + active: true, + phase: 'beta', + variants: [ + { + name: 'foo', + active: true, + rolloutPercent: 50, + }, + ], + }, + }, + }, + { + status: 500, + body: { + error: + 'Error while creating split test: SplitTest validation failed: name: Path `name` is required.', + }, + } + ) + }) + + it('Error - missing variants in configuration', async function () { + await this.expectResponse( + { + method: 'POST', + path: '/admin/createSplitTest', + payload: { + name: 'split-test-1', + configuration: {}, + }, + }, + { + status: 500, + body: { + error: + 'Error while creating split test: configuration.variants is not iterable', + }, + } + ) + }) + + it('Error - create with total rollout percent exceeding 100', async function () { + await this.expectResponse( + { + method: 'POST', + path: '/admin/createSplitTest', + payload: { + name: 'split-test-1', + configuration: { + active: true, + phase: 'alpha', + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 51, + }, + { + name: 'variant-2', + active: true, + rolloutPercent: 50, + }, + ], + }, + }, + }, + { + status: 500, + body: { + error: + 'Error while creating split test: Total variants rollout percentage cannot exceed 100', + }, + } + ) + }) + + it('Error - create with empty split test name', async function () { + await this.expectResponse( + { + method: 'POST', + path: '/admin/createSplitTest', + payload: { + name: '', + configuration: { + active: true, + phase: 'alpha', + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 50, + }, + ], + }, + }, + }, + { + status: 500, + body: { + error: + 'Error while creating split test: SplitTest validation failed: name: Path `name` is required.', + }, + } + ) + }) + + it('Error - create with empty variant name', async function () { + await this.expectResponse( + { + method: 'POST', + path: '/admin/createSplitTest', + payload: { + name: 'split-test-7', + configuration: { + active: true, + phase: 'alpha', + variants: [ + { + name: '', + active: true, + rolloutPercent: 50, + }, + ], + }, + }, + }, + { + status: 500, + body: { + error: + 'Error while creating split test: SplitTest validation failed: versions.0.variants.0.name: Path `name` is required.', + }, + } + ) + }) + + it('Error - cannot switch to release phase after revert to beta', async function () { + const version1 = { + versionNumber: 1, + phase: 'alpha', + active: true, + variants: [], + } + await this.expectResponse( + { + method: 'POST', + path: '/admin/createSplitTest', + payload: { + name: 'split-test-8', + configuration: { + active: true, + phase: 'alpha', + variants: [], + }, + }, + }, + { + status: 200, + body: { + name: 'split-test-8', + versions: [version1], + }, + excludingEvery: ['__v', '_id'], + } + ) + + const version2 = { + versionNumber: 2, + phase: 'beta', + active: true, + variants: [], + } + await this.expectResponse( + { + method: 'POST', + path: '/admin/splitTest/switchToNextPhase', + payload: { + name: 'split-test-8', + }, + }, + { + status: 200, + body: { + name: 'split-test-8', + versions: [version1, version2], + }, + excludingEvery: ['__v', '_id'], + } + ) + + const version3 = { + versionNumber: 3, + phase: 'release', + active: true, + variants: [], + } + await this.expectResponse( + { + method: 'POST', + path: '/admin/splitTest/switchToNextPhase', + payload: { + name: 'split-test-8', + }, + }, + { + status: 200, + body: { + name: 'split-test-8', + versions: [version1, version2, version3], + }, + excludingEvery: ['__v', '_id'], + } + ) + + const version4 = { + versionNumber: 4, + phase: 'beta', + active: true, + variants: [], + } + await this.expectResponse( + { + method: 'POST', + path: '/admin/splitTest/revertToPreviousVersion', + payload: { + name: 'split-test-8', + versionNumber: 2, + }, + }, + { + status: 200, + body: { + name: 'split-test-8', + forbidReleasePhase: true, + versions: [version1, version2, version3, version4], + }, + excludingEvery: ['__v', '_id'], + } + ) + + await this.expectResponse( + { + method: 'POST', + path: '/admin/splitTest/switchToNextPhase', + payload: { + name: 'split-test-8', + }, + }, + { + status: 500, + body: { + error: + 'Error while switching split test to next phase: Switch to release phase is disabled for this test', + }, + } + ) + }) + }) + + describe('Assign users to variants', async function () { + it('Assign to default variant when test is inactive', async function () { + await this.sendAdminRequest({ + method: 'POST', + path: '/admin/createSplitTest', + payload: { + name: 'user-test-inactive', + configuration: { + active: false, + phase: 'release', + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 100, + }, + ], + }, + }, + }) + + const user = await createUserWithId('foobarbazcar', { alpha: true }) + const assignment = await SplitTestV2Handler.promises.getAssignment( + user._id, + 'user-test-inactive' + ) + assert.deepEqual(assignment, { + variant: 'default', + analytics: { + segmentation: {}, + }, + }) + }) + + it('Assign to correct variant when active for alpha/non-alpha user', async function () { + await this.sendAdminRequest({ + method: 'POST', + path: '/admin/createSplitTest', + payload: { + name: 'user-test-1', + configuration: { + active: true, + phase: 'alpha', + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 30, + }, + ], + }, + }, + }) + + const user1 = await createUserWithId('abc123abc123', { alpha: true }) + const assignment1 = await SplitTestV2Handler.promises.getAssignment( + user1._id, + 'user-test-1' + ) // percentile: 24 + assert.deepEqual(assignment1, { + variant: 'variant-1', + analytics: { + segmentation: { + splitTest: 'user-test-1', + phase: 'alpha', + versionNumber: 1, + variant: 'variant-1', + }, + }, + }) + + const user2 = await createUserWithId('xxx123abc123', { + alpha: false, + beta: true, + }) + const assignment2 = await SplitTestV2Handler.promises.getAssignment( + user2._id, + 'user-test-1' + ) // percentile: 70 + assert.deepEqual(assignment2, { + variant: 'default', + analytics: { + segmentation: {}, + }, + }) + }) + + it('Assign to correct variant when active for beta user', async function () { + await this.sendAdminRequest({ + method: 'POST', + path: '/admin/createSplitTest', + payload: { + name: 'user-test-2', + configuration: { + active: true, + phase: 'beta', + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 30, + }, + ], + }, + }, + }) + + const user1 = await createUserWithId('abc123abc123', { beta: true }) + const assignment1 = await SplitTestV2Handler.promises.getAssignment( + user1._id, + 'user-test-2' + ) // percentile: 62 + assert.deepEqual(assignment1, { + variant: 'default', + analytics: { + segmentation: { + splitTest: 'user-test-2', + phase: 'beta', + versionNumber: 1, + variant: 'default', + }, + }, + }) + + const user2 = await createUserWithId('abc234abc234', { beta: false }) + const assignment2 = await SplitTestV2Handler.promises.getAssignment( + user2._id, + 'user-test-2' + ) // percentile: 34 + assert.deepEqual(assignment2, { + variant: 'default', + analytics: { + segmentation: {}, + }, + }) + }) + + it('Assign to correct variant when active in release phase', async function () { + await this.sendAdminRequest({ + method: 'POST', + path: '/admin/createSplitTest', + payload: { + name: 'user-test-3', + configuration: { + active: true, + phase: 'release', + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 50, + }, + ], + }, + }, + }) + + const user1 = await createUserWithId('abc123abc123') + const assignment1 = await SplitTestV2Handler.promises.getAssignment( + user1._id, + 'user-test-3' + ) // percentile: 33 + assert.deepEqual(assignment1, { + variant: 'variant-1', + analytics: { + segmentation: { + splitTest: 'user-test-3', + phase: 'release', + versionNumber: 1, + variant: 'variant-1', + }, + }, + }) + + const user2 = await createUserWithId('abc234abc234') + const assignment2 = await SplitTestV2Handler.promises.getAssignment( + user2._id, + 'user-test-3' + ) // percentile: 81 + assert.deepEqual(assignment2, { + variant: 'default', + analytics: { + segmentation: { + splitTest: 'user-test-3', + phase: 'release', + versionNumber: 1, + variant: 'default', + }, + }, + }) + }) + + it('Split test is cached for 1min after update', async function () { + await this.sendAdminRequest({ + method: 'POST', + path: '/admin/createSplitTest', + payload: { + name: 'user-test-4', + configuration: { + active: true, + phase: 'release', + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 10, + }, + ], + }, + }, + }) + + const user1 = await createUserWithId('abc123abc123') + const assignment1 = await SplitTestV2Handler.promises.getAssignment( + user1._id, + 'user-test-4' + ) // percentile: 38 + assert.deepEqual(assignment1, { + variant: 'default', + analytics: { + segmentation: { + splitTest: 'user-test-4', + phase: 'release', + versionNumber: 1, + variant: 'default', + }, + }, + }) + + await this.sendAdminRequest({ + method: 'POST', + path: '/admin/updateSplitTest', + payload: { + name: 'user-test-4', + configuration: { + active: true, + phase: 'release', + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 100, + }, + ], + }, + }, + }) + + const assignment2 = await SplitTestV2Handler.promises.getAssignment( + user1._id, + 'user-test-4' + ) // percentile: 38 + assert.deepEqual(assignment2, { + variant: 'default', + analytics: { + segmentation: { + splitTest: 'user-test-4', + phase: 'release', + versionNumber: 1, + variant: 'default', + }, + }, + }) + }) + + it('User is reassigned if split test cache is cleared after update', async function () { + await this.sendAdminRequest({ + method: 'POST', + path: '/admin/createSplitTest', + payload: { + name: 'user-test-5', + configuration: { + active: true, + phase: 'release', + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 10, + }, + ], + }, + }, + }) + + let user1 = await createUserWithId('abc123abc123') + const assignment1 = await SplitTestV2Handler.promises.getAssignment( + user1._id, + 'user-test-5', + { sync: true } + ) // percentile: 84 + assert.deepEqual(assignment1, { + variant: 'default', + analytics: { + segmentation: { + splitTest: 'user-test-5', + phase: 'release', + versionNumber: 1, + variant: 'default', + }, + }, + }) + user1 = await getUser('abc123abc123') + assert.deepEqualExcludingEvery( + user1.splitTests, + { + 'user-test-5': [ + { + variantName: 'default', + versionNumber: 1, + phase: 'release', + }, + ], + }, + ['assignedAt'] + ) + + await this.sendAdminRequest({ + method: 'POST', + path: '/admin/updateSplitTest', + payload: { + name: 'user-test-5', + configuration: { + active: true, + phase: 'release', + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 100, + }, + ], + }, + }, + }) + + await CacheFlow.reset('split-test') + + const assignment2 = await SplitTestV2Handler.promises.getAssignment( + user1._id, + 'user-test-5', + { sync: true } + ) // percentile: 84 + assert.deepEqual(assignment2, { + variant: 'variant-1', + analytics: { + segmentation: { + splitTest: 'user-test-5', + phase: 'release', + versionNumber: 2, + variant: 'variant-1', + }, + }, + }) + user1 = await getUser('abc123abc123') + assert.deepEqualExcludingEvery( + user1.splitTests, + { + 'user-test-5': [ + { + variantName: 'default', + versionNumber: 1, + phase: 'release', + }, + { + variantName: 'variant-1', + versionNumber: 2, + phase: 'release', + }, + ], + }, + ['assignedAt'] + ) + }) + + it('User is reassigned from control to striped variant after update, and reverted to control', async function () { + await this.sendAdminRequest({ + method: 'POST', + path: '/admin/createSplitTest', + payload: { + name: 'user-test-6', + configuration: { + active: true, + phase: 'alpha', + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 10, + }, + { + name: 'variant-2', + active: true, + rolloutPercent: 10, + }, + ], + }, + }, + }) + await this.sendAdminRequest({ + method: 'POST', + path: '/admin/splitTest/switchToNextPhase', + payload: { + name: 'user-test-6', + }, + }) + await this.sendAdminRequest({ + method: 'POST', + path: '/admin/splitTest/switchToNextPhase', + payload: { + name: 'user-test-6', + }, + }) + await this.sendAdminRequest({ + method: 'POST', + path: '/admin/updateSplitTest', + payload: { + name: 'user-test-6', + configuration: { + active: true, + phase: 'release', + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 10, + }, + { + name: 'variant-2', + active: true, + rolloutPercent: 10, + }, + ], + }, + }, + }) + + let user1 = await createUserWithId('abc123abc123') + const assignment1 = await SplitTestV2Handler.promises.getAssignment( + user1._id, + 'user-test-6', + { sync: true } + ) // percentile: 47 + assert.deepEqual(assignment1, { + variant: 'default', + analytics: { + segmentation: { + splitTest: 'user-test-6', + phase: 'release', + versionNumber: 4, + variant: 'default', + }, + }, + }) + user1 = await getUser('abc123abc123') + assert.deepEqualExcludingEvery( + user1.splitTests, + { + 'user-test-6': [ + { + variantName: 'default', + versionNumber: 4, + phase: 'release', + }, + ], + }, + ['assignedAt'] + ) + + await this.sendAdminRequest({ + method: 'POST', + path: '/admin/updateSplitTest', + payload: { + name: 'user-test-6', + configuration: { + active: true, + phase: 'release', + variants: [ + { + name: 'variant-1', + active: true, + rolloutPercent: 40, + }, + { + name: 'variant-2', + active: true, + rolloutPercent: 40, + }, + ], + }, + }, + }) + await CacheFlow.reset('split-test') + + const assignment2 = await SplitTestV2Handler.promises.getAssignment( + user1._id, + 'user-test-6', + { sync: true } + ) // percentile: 47 + assert.deepEqual(assignment2, { + variant: 'variant-1', + analytics: { + segmentation: { + splitTest: 'user-test-6', + phase: 'release', + versionNumber: 5, + variant: 'variant-1', + }, + }, + }) + user1 = await getUser('abc123abc123') + assert.deepEqualExcludingEvery( + user1.splitTests, + { + 'user-test-6': [ + { + variantName: 'default', + versionNumber: 4, + phase: 'release', + }, + { + variantName: 'variant-1', + versionNumber: 5, + phase: 'release', + }, + ], + }, + ['assignedAt'] + ) + + await this.sendAdminRequest({ + method: 'POST', + path: '/admin/splitTest/revertToPreviousVersion', + payload: { + name: 'user-test-6', + versionNumber: 4, + }, + }) + await CacheFlow.reset('split-test') + + const assignment3 = await SplitTestV2Handler.promises.getAssignment( + user1._id, + 'user-test-6', + { sync: true } + ) // percentile: 47 + assert.deepEqual(assignment3, { + variant: 'default', + analytics: { + segmentation: { + splitTest: 'user-test-6', + phase: 'release', + versionNumber: 6, + variant: 'default', + }, + }, + }) + user1 = await getUser('abc123abc123') + assert.deepEqualExcludingEvery( + user1.splitTests, + { + 'user-test-6': [ + { + variantName: 'default', + versionNumber: 4, + phase: 'release', + }, + { + variantName: 'variant-1', + versionNumber: 5, + phase: 'release', + }, + { + variantName: 'default', + versionNumber: 6, + phase: 'release', + }, + ], + }, + ['assignedAt'] + ) + + await this.sendAdminRequest({ + method: 'POST', + path: '/admin/splitTest/revertToPreviousVersion', + payload: { + name: 'user-test-6', + versionNumber: 2, + }, + }) + await CacheFlow.reset('split-test') + const assignment4 = await SplitTestV2Handler.promises.getAssignment( + user1._id, + 'user-test-6', + { sync: true } + ) // percentile: 47 + assert.deepEqual(assignment4, { + variant: 'default', + analytics: { + segmentation: {}, + }, + }) + user1 = await getUser('abc123abc123') + assert.deepEqualExcludingEvery( + user1.splitTests, + { + 'user-test-6': [ + { + variantName: 'default', + versionNumber: 4, + phase: 'release', + }, + { + variantName: 'variant-1', + versionNumber: 5, + phase: 'release', + }, + { + variantName: 'default', + versionNumber: 6, + phase: 'release', + }, + ], + }, + ['assignedAt'] + ) + }) + }) +}) + +async function createUserWithId(id, { alpha = false, beta = false } = {}) { + const newUser = new User() + const user = await promisify(newUser.register).bind(newUser)() + await db.users.remove({ _id: user._id }) + user._id = ObjectId(id) + user.alphaProgram = alpha + user.betaProgram = beta + await db.users.insert(user) + return db.users.findOne( + { _id: ObjectId(id) }, + { splitTests: 1, alphaProgram: 1, betaProgram: 1 } + ) +} + +async function getUser(id) { + return db.users.findOne( + { _id: ObjectId(id) }, + { splitTests: 1, alphaProgram: 1, betaProgram: 1 } + ) +}