From cf472f54d045c5d33ff00afdaf464a7dbe22a4cd Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 7 Jul 2025 10:29:19 +0200 Subject: [PATCH] [history-v1] use POST requests for expiring redis buffer from cron (#26568) * [history-v1] use POST requests for expiring redis buffer from cron (cherry picked from commit 15780ac54e36b96e1aed9fd9eb6dfe9d4fbf842f) * [history-v1] remove double claim of expire job GitOrigin-RevId: 8b2eab07006a5819a47eed3f646b2a4d75f86e5b --- .../api/controllers/project_import.js | 8 ++ .../history-v1/api/swagger/project_import.js | 36 ++++++++ .../storage/scripts/expire_redis_chunks.js | 29 ++++++- .../scripts/persist_and_expire_queues.sh | 2 +- .../acceptance/js/api/project_expiry.test.js | 82 +++++++++++++++++++ 5 files changed, 154 insertions(+), 3 deletions(-) create mode 100644 services/history-v1/test/acceptance/js/api/project_expiry.test.js diff --git a/services/history-v1/api/controllers/project_import.js b/services/history-v1/api/controllers/project_import.js index 638873d105..02fb793c87 100644 --- a/services/history-v1/api/controllers/project_import.js +++ b/services/history-v1/api/controllers/project_import.js @@ -28,6 +28,7 @@ const InvalidChangeError = storage.InvalidChangeError const render = require('./render') const Rollout = require('../app/rollout') +const redisBackend = require('../../storage/lib/chunk_store/redis') const rollout = new Rollout(config) rollout.report(logger) // display the rollout configuration in the logs @@ -177,6 +178,13 @@ async function flushChanges(req, res, next) { } } +async function expireProject(req, res, next) { + const projectId = req.swagger.params.project_id.value + await redisBackend.expireProject(projectId) + res.status(HTTPStatus.OK).end() +} + exports.importSnapshot = expressify(importSnapshot) exports.importChanges = expressify(importChanges) exports.flushChanges = expressify(flushChanges) +exports.expireProject = expressify(expireProject) diff --git a/services/history-v1/api/swagger/project_import.js b/services/history-v1/api/swagger/project_import.js index 6103eed74b..043dc70667 100644 --- a/services/history-v1/api/swagger/project_import.js +++ b/services/history-v1/api/swagger/project_import.js @@ -174,10 +174,46 @@ const flushChanges = { ], } +const expireProject = { + 'x-swagger-router-controller': 'project_import', + operationId: 'expireProject', + tags: ['ProjectImport'], + description: 'Expire project changes from buffer.', + parameters: [ + { + name: 'project_id', + in: 'path', + description: 'project id', + required: true, + type: 'string', + }, + ], + responses: { + 200: { + description: 'Success', + schema: { + $ref: '#/definitions/Project', + }, + }, + 404: { + description: 'Not Found', + schema: { + $ref: '#/definitions/Error', + }, + }, + }, + security: [ + { + basic: [], + }, + ], +} + exports.paths = { '/projects/{project_id}/import': { post: importSnapshot }, '/projects/{project_id}/legacy_import': { post: importSnapshot }, '/projects/{project_id}/changes': { get: getChanges, post: importChanges }, '/projects/{project_id}/legacy_changes': { post: importChanges }, '/projects/{project_id}/flush': { post: flushChanges }, + '/projects/{project_id}/expire': { post: expireProject }, } diff --git a/services/history-v1/storage/scripts/expire_redis_chunks.js b/services/history-v1/storage/scripts/expire_redis_chunks.js index 60ce4c66f6..cb6d689e2c 100644 --- a/services/history-v1/storage/scripts/expire_redis_chunks.js +++ b/services/history-v1/storage/scripts/expire_redis_chunks.js @@ -3,23 +3,48 @@ const commandLineArgs = require('command-line-args') const redis = require('../lib/redis') const { scanAndProcessDueItems } = require('../lib/scan') const { expireProject, claimExpireJob } = require('../lib/chunk_store/redis') +const config = require('config') +const { fetchNothing } = require('@overleaf/fetch-utils') const rclient = redis.rclientHistory -const optionDefinitions = [{ name: 'dry-run', alias: 'd', type: Boolean }] +const optionDefinitions = [ + { name: 'dry-run', alias: 'd', type: Boolean }, + { name: 'post-request', type: Boolean }, +] const options = commandLineArgs(optionDefinitions) const DRY_RUN = options['dry-run'] || false +const POST_REQUEST = options['post-request'] || false +const HISTORY_V1_URL = `http://${process.env.HISTORY_V1_HOST || 'localhost'}:${process.env.PORT || 3100}` logger.initialize('expire-redis-chunks') async function expireProjectAction(projectId) { const job = await claimExpireJob(projectId) - await expireProject(projectId) + if (POST_REQUEST) { + await requestProjectExpiry(projectId) + } else { + await expireProject(projectId) + } if (job && job.close) { await job.close() } } +async function requestProjectExpiry(projectId) { + logger.debug({ projectId }, 'sending project expire request') + const url = `${HISTORY_V1_URL}/api/projects/${projectId}/expire` + const credentials = Buffer.from( + `staging:${config.get('basicHttpAuth.password')}` + ).toString('base64') + await fetchNothing(url, { + method: 'POST', + headers: { + Authorization: `Basic ${credentials}`, + }, + }) +} + async function runExpireChunks() { await scanAndProcessDueItems( rclient, diff --git a/services/history-v1/storage/scripts/persist_and_expire_queues.sh b/services/history-v1/storage/scripts/persist_and_expire_queues.sh index d5789541da..35b057f52c 100644 --- a/services/history-v1/storage/scripts/persist_and_expire_queues.sh +++ b/services/history-v1/storage/scripts/persist_and_expire_queues.sh @@ -1,3 +1,3 @@ #!/bin/sh node storage/scripts/persist_redis_chunks.mjs --queue --max-time 270 -node storage/scripts/expire_redis_chunks.js +node storage/scripts/expire_redis_chunks.js --post-request diff --git a/services/history-v1/test/acceptance/js/api/project_expiry.test.js b/services/history-v1/test/acceptance/js/api/project_expiry.test.js new file mode 100644 index 0000000000..efa589ec71 --- /dev/null +++ b/services/history-v1/test/acceptance/js/api/project_expiry.test.js @@ -0,0 +1,82 @@ +'use strict' + +const BPromise = require('bluebird') +const { expect } = require('chai') +const HTTPStatus = require('http-status') +const fetch = require('node-fetch') +const fs = BPromise.promisifyAll(require('node:fs')) + +const cleanup = require('../storage/support/cleanup') +const fixtures = require('../storage/support/fixtures') +const testFiles = require('../storage/support/test_files') +const testProjects = require('./support/test_projects') +const testServer = require('./support/test_server') + +const { Change, File, Operation } = require('overleaf-editor-core') +const queueChanges = require('../../../../storage/lib/queue_changes') +const { getState } = require('../../../../storage/lib/chunk_store/redis') + +describe('project expiry', function () { + beforeEach(cleanup.everything) + beforeEach(fixtures.create) + + it('expire redis buffer', async function () { + const basicAuthClient = testServer.basicAuthClient + const projectId = await testProjects.createEmptyProject() + + // upload an empty file + const response = await fetch( + testServer.url( + `/api/projects/${projectId}/blobs/${File.EMPTY_FILE_HASH}`, + { qs: { pathname: 'main.tex' } } + ), + { + method: 'PUT', + body: fs.createReadStream(testFiles.path('empty.tex')), + headers: { + Authorization: testServer.basicAuthHeader, + }, + } + ) + expect(response.ok).to.be.true + + const testFile = File.fromHash(File.EMPTY_FILE_HASH) + const testChange = new Change( + [Operation.addFile('main.tex', testFile)], + new Date() + ) + await queueChanges(projectId, [testChange], 0) + + // Verify that the changes are queued and not yet persisted + const initialState = await getState(projectId) + expect(initialState.persistedVersion).to.be.null + expect(initialState.changes).to.have.lengthOf(1) + + const importResponse = + await basicAuthClient.apis.ProjectImport.flushChanges({ + project_id: projectId, + }) + + expect(importResponse.status).to.equal(HTTPStatus.OK) + + // Verify that the changes were persisted to the chunk store + const flushedState = await getState(projectId) + expect(flushedState.persistedVersion).to.equal(1) + + const expireResponse = + await basicAuthClient.apis.ProjectImport.expireProject({ + project_id: projectId, + }) + expect(expireResponse.status).to.equal(HTTPStatus.OK) + + const finalState = await getState(projectId) + expect(finalState).to.deep.equal({ + changes: [], + expireTime: null, + headSnapshot: null, + headVersion: null, + persistTime: null, + persistedVersion: null, + }) + }) +})