diff --git a/services/history-v1/api/app/rollout.js b/services/history-v1/api/app/rollout.js new file mode 100644 index 0000000000..24ca0409f8 --- /dev/null +++ b/services/history-v1/api/app/rollout.js @@ -0,0 +1,76 @@ +const crypto = require('node:crypto') + +class Rollout { + constructor(config) { + // The history buffer level is used to determine whether to queue changes + // in Redis or persist them directly to the chunk store. + // If defaults to 0 (no queuing) if not set. + this.historyBufferLevel = config.has('historyBufferLevel') + ? parseInt(config.get('historyBufferLevel'), 10) + : 0 + // The forcePersistBuffer flag will ensure the buffer is fully persisted before + // any persist operation. Set this to true if you want to make the persisted-version + // in Redis match the endVersion of the latest chunk. This should be set to true + // when downgrading from a history buffer level that queues changes in Redis + // without persisting them immediately. + this.forcePersistBuffer = config.has('forcePersistBuffer') + ? config.get('forcePersistBuffer') === 'true' + : false + + // Support gradual rollout of the next history buffer level + // with a percentage of projects using it. + this.nextHistoryBufferLevel = config.has('nextHistoryBufferLevel') + ? parseInt(config.get('nextHistoryBufferLevel'), 10) + : null + this.nextHistoryBufferLevelRolloutPercentage = config.has( + 'nextHistoryBufferLevelRolloutPercentage' + ) + ? parseInt(config.get('nextHistoryBufferLevelRolloutPercentage'), 10) + : 0 + } + + report(logger) { + logger.info( + { + historyBufferLevel: this.historyBufferLevel, + forcePersistBuffer: this.forcePersistBuffer, + nextHistoryBufferLevel: this.nextHistoryBufferLevel, + nextHistoryBufferLevelRolloutPercentage: + this.nextHistoryBufferLevelRolloutPercentage, + }, + this.historyBufferLevel > 0 || this.forcePersistBuffer + ? 'using history buffer' + : 'history buffer disabled' + ) + } + + /** + * Get the history buffer level for a project. + * @param {string} projectId + * @returns {Object} - An object containing the history buffer level and force persist buffer flag. + * @property {number} historyBufferLevel - The history buffer level to use for processing changes. + * @property {boolean} forcePersistBuffer - If true, forces the buffer to be persisted before any operation. + */ + getHistoryBufferLevelOptions(projectId) { + if ( + this.nextHistoryBufferLevel > this.historyBufferLevel && + this.nextHistoryBufferLevelRolloutPercentage > 0 + ) { + const hash = crypto.createHash('sha1').update(projectId).digest('hex') + const percentage = parseInt(hash.slice(0, 8), 16) % 100 + // If the project is in the rollout percentage, we use the next history buffer level. + if (percentage < this.nextHistoryBufferLevelRolloutPercentage) { + return { + historyBufferLevel: this.nextHistoryBufferLevel, + forcePersistBuffer: this.forcePersistBuffer, + } + } + } + return { + historyBufferLevel: this.historyBufferLevel, + forcePersistBuffer: this.forcePersistBuffer, + } + } +} + +module.exports = Rollout diff --git a/services/history-v1/api/controllers/project_import.js b/services/history-v1/api/controllers/project_import.js index 80e7a25492..99f56cb7e4 100644 --- a/services/history-v1/api/controllers/project_import.js +++ b/services/history-v1/api/controllers/project_import.js @@ -2,6 +2,7 @@ 'use strict' +const config = require('config') const { expressify } = require('@overleaf/promise-utils') const HTTPStatus = require('http-status') @@ -26,29 +27,10 @@ const persistBuffer = storage.persistBuffer const InvalidChangeError = storage.InvalidChangeError const render = require('./render') +const Rollout = require('../app/rollout') -const config = require('config') -// The history buffer level is used to determine whether to queue changes -// in Redis or persist them directly to the chunk store. -// If defaults to 0 (no queuing) if not set. -const historyBufferLevel = config.has('historyBufferLevel') - ? parseInt(config.historyBufferLevel, 10) - : 0 -// The forcePersistBuffer flag will ensure the buffer is fully persisted before -// any persist operation. Set this to true if you want to make the persisted-version -// in Redis match the endVersion of the latest chunk. This should be set to true -// when downgrading from a history buffer level that queues changes in Redis -// without persisting them immediately. -const forcePersistBuffer = config.has('forcePersistBuffer') - ? config.get('forcePersistBuffer') === 'true' - : false - -logger.info( - { historyBufferLevel, forcePersistBuffer }, - historyBufferLevel > 0 || forcePersistBuffer - ? 'using history buffer' - : 'history buffer disabled' -) +const rollout = new Rollout(config) +rollout.report(logger) // display the rollout configuration in the logs async function importSnapshot(req, res) { const projectId = req.swagger.params.project_id.value @@ -134,6 +116,8 @@ async function importChanges(req, res, next) { let result try { + const { historyBufferLevel, forcePersistBuffer } = + rollout.getHistoryBufferLevelOptions(projectId) result = await commitChanges(projectId, changes, limits, endVersion, { historyBufferLevel, forcePersistBuffer, diff --git a/services/history-v1/config/custom-environment-variables.json b/services/history-v1/config/custom-environment-variables.json index 7fc2114699..f0827fc538 100644 --- a/services/history-v1/config/custom-environment-variables.json +++ b/services/history-v1/config/custom-environment-variables.json @@ -86,6 +86,8 @@ "httpRequestTimeout": "HTTP_REQUEST_TIMEOUT", "historyBufferLevel": "HISTORY_BUFFER_LEVEL", "forcePersistBuffer": "FORCE_PERSIST_BUFFER", + "nextHistoryBufferLevel": "NEXT_HISTORY_BUFFER_LEVEL", + "nextHistoryBufferLevelRolloutPercentage": "NEXT_HISTORY_BUFFER_LEVEL_ROLLOUT_PERCENTAGE", "redis": { "queue": { "host": "QUEUES_REDIS_HOST", diff --git a/services/history-v1/test/acceptance/js/api/rollout.test.js b/services/history-v1/test/acceptance/js/api/rollout.test.js new file mode 100644 index 0000000000..f1a65e5aff --- /dev/null +++ b/services/history-v1/test/acceptance/js/api/rollout.test.js @@ -0,0 +1,115 @@ +const config = require('config') +const sinon = require('sinon') +const { expect } = require('chai') + +const cleanup = require('../storage/support/cleanup') +const Rollout = require('../../../../api/app/rollout') + +describe('rollout', function () { + beforeEach(cleanup.everything) + beforeEach('Set up stubs', function () { + sinon.stub(config, 'has').callThrough() + sinon.stub(config, 'get').callThrough() + }) + afterEach(sinon.restore) + + it('should return a valid history buffer level', function () { + setMockConfig('historyBufferLevel', '2') + setMockConfig('forcePersistBuffer', 'false') + + const rollout = new Rollout(config) + const { historyBufferLevel, forcePersistBuffer } = + rollout.getHistoryBufferLevelOptions('test-project-id') + expect(historyBufferLevel).to.equal(2) + expect(forcePersistBuffer).to.be.false + }) + + it('should return a valid history buffer level and force persist buffer options', function () { + setMockConfig('historyBufferLevel', '1') + setMockConfig('forcePersistBuffer', 'true') + const rollout = new Rollout(config) + const { historyBufferLevel, forcePersistBuffer } = + rollout.getHistoryBufferLevelOptions('test-project-id') + expect(historyBufferLevel).to.equal(1) + expect(forcePersistBuffer).to.be.true + }) + + describe('with a higher next history buffer level rollout', function () { + beforeEach(function () { + setMockConfig('historyBufferLevel', '2') + setMockConfig('forcePersistBuffer', 'false') + setMockConfig('nextHistoryBufferLevel', '3') + }) + it('should return the expected history buffer level when the rollout percentage is zero', function () { + setMockConfig('nextHistoryBufferLevelRolloutPercentage', '0') + const rollout = new Rollout(config) + for (let i = 0; i < 1000; i++) { + const { historyBufferLevel, forcePersistBuffer } = + rollout.getHistoryBufferLevelOptions(`test-project-id-${i}`) + expect(historyBufferLevel).to.equal(2) + expect(forcePersistBuffer).to.be.false + } + }) + + it('should return the expected distribution of levels when the rollout percentage is 10%', function () { + setMockConfig('nextHistoryBufferLevelRolloutPercentage', '10') + const rollout = new Rollout(config) + let currentLevel = 0 + let nextLevel = 0 + for (let i = 0; i < 1000; i++) { + const { historyBufferLevel } = rollout.getHistoryBufferLevelOptions( + `test-project-id-${i}` + ) + switch (historyBufferLevel) { + case 2: + currentLevel++ + break + case 3: + nextLevel++ + break + default: + expect.fail( + `Unexpected history buffer level: ${historyBufferLevel}` + ) + } + } + const twoPercentage = (currentLevel / 1000) * 100 + const threePercentage = (nextLevel / 1000) * 100 + expect(twoPercentage).to.be.closeTo(90, 5) // 90% for level 2 + expect(threePercentage).to.be.closeTo(10, 5) // 10% for level 3 + }) + }) + describe('with a next history buffer level lower than the current level', function () { + beforeEach(function () { + setMockConfig('historyBufferLevel', '3') + setMockConfig('forcePersistBuffer', 'false') + setMockConfig('nextHistoryBufferLevel', '2') + }) + it('should always return the current level when the rollout percentage is zero', function () { + setMockConfig('nextHistoryBufferLevelRolloutPercentage', '0') + const rollout = new Rollout(config) + for (let i = 0; i < 1000; i++) { + const { historyBufferLevel, forcePersistBuffer } = + rollout.getHistoryBufferLevelOptions(`test-project-id-${i}`) + expect(historyBufferLevel).to.equal(3) + expect(forcePersistBuffer).to.be.false + } + }) + + it('should always return the current level regardless of the rollout percentage', function () { + setMockConfig('nextHistoryBufferLevelRolloutPercentage', '10') + const rollout = new Rollout(config) + for (let i = 0; i < 1000; i++) { + const { historyBufferLevel } = rollout.getHistoryBufferLevelOptions( + `test-project-id-${i}` + ) + expect(historyBufferLevel).to.equal(3) + } + }) + }) +}) + +function setMockConfig(path, value) { + config.has.withArgs(path).returns(true) + config.get.withArgs(path).returns(value) +}