diff --git a/services/history-v1/api/controllers/project_import.js b/services/history-v1/api/controllers/project_import.js index 02fb793c87..638873d105 100644 --- a/services/history-v1/api/controllers/project_import.js +++ b/services/history-v1/api/controllers/project_import.js @@ -28,7 +28,6 @@ const InvalidChangeError = storage.InvalidChangeError const render = require('./render') const Rollout = require('../app/rollout') -const redisBackend = require('../../storage/lib/chunk_store/redis') const rollout = new Rollout(config) rollout.report(logger) // display the rollout configuration in the logs @@ -178,13 +177,6 @@ async function flushChanges(req, res, next) { } } -async function expireProject(req, res, next) { - const projectId = req.swagger.params.project_id.value - await redisBackend.expireProject(projectId) - res.status(HTTPStatus.OK).end() -} - exports.importSnapshot = expressify(importSnapshot) exports.importChanges = expressify(importChanges) exports.flushChanges = expressify(flushChanges) -exports.expireProject = expressify(expireProject) diff --git a/services/history-v1/api/swagger/project_import.js b/services/history-v1/api/swagger/project_import.js index 043dc70667..6103eed74b 100644 --- a/services/history-v1/api/swagger/project_import.js +++ b/services/history-v1/api/swagger/project_import.js @@ -174,46 +174,10 @@ const flushChanges = { ], } -const expireProject = { - 'x-swagger-router-controller': 'project_import', - operationId: 'expireProject', - tags: ['ProjectImport'], - description: 'Expire project changes from buffer.', - parameters: [ - { - name: 'project_id', - in: 'path', - description: 'project id', - required: true, - type: 'string', - }, - ], - responses: { - 200: { - description: 'Success', - schema: { - $ref: '#/definitions/Project', - }, - }, - 404: { - description: 'Not Found', - schema: { - $ref: '#/definitions/Error', - }, - }, - }, - security: [ - { - basic: [], - }, - ], -} - exports.paths = { '/projects/{project_id}/import': { post: importSnapshot }, '/projects/{project_id}/legacy_import': { post: importSnapshot }, '/projects/{project_id}/changes': { get: getChanges, post: importChanges }, '/projects/{project_id}/legacy_changes': { post: importChanges }, '/projects/{project_id}/flush': { post: flushChanges }, - '/projects/{project_id}/expire': { post: expireProject }, } diff --git a/services/history-v1/storage/scripts/expire_redis_chunks.js b/services/history-v1/storage/scripts/expire_redis_chunks.js index e4c595a249..60ce4c66f6 100644 --- a/services/history-v1/storage/scripts/expire_redis_chunks.js +++ b/services/history-v1/storage/scripts/expire_redis_chunks.js @@ -3,47 +3,18 @@ const commandLineArgs = require('command-line-args') const redis = require('../lib/redis') const { scanAndProcessDueItems } = require('../lib/scan') const { expireProject, claimExpireJob } = require('../lib/chunk_store/redis') -const config = require('config') -const { fetchNothing } = require('@overleaf/fetch-utils') const rclient = redis.rclientHistory -const optionDefinitions = [ - { name: 'dry-run', alias: 'd', type: Boolean }, - { name: 'post-request', type: Boolean }, -] +const optionDefinitions = [{ name: 'dry-run', alias: 'd', type: Boolean }] const options = commandLineArgs(optionDefinitions) const DRY_RUN = options['dry-run'] || false -const POST_REQUEST = options['post-request'] || false -const HISTORY_V1_URL = `http://${process.env.HISTORY_V1_HOST || 'localhost'}:${process.env.PORT || 3100}` logger.initialize('expire-redis-chunks') async function expireProjectAction(projectId) { const job = await claimExpireJob(projectId) - if (POST_REQUEST) { - await requestProjectExpiry(projectId) - } else { - await expireProject(projectId) - } - if (job && job.close) { - await job.close() - } -} - -async function requestProjectExpiry(projectId) { - const job = await claimExpireJob(projectId) - logger.debug({ projectId }, 'sending project expire request') - const url = `${HISTORY_V1_URL}/api/projects/${projectId}/expire` - const credentials = Buffer.from( - `staging:${config.get('basicHttpAuth.password')}` - ).toString('base64') - await fetchNothing(url, { - method: 'POST', - headers: { - Authorization: `Basic ${credentials}`, - }, - }) + await expireProject(projectId) if (job && job.close) { await job.close() } diff --git a/services/history-v1/storage/scripts/persist_and_expire_queues.sh b/services/history-v1/storage/scripts/persist_and_expire_queues.sh index 35b057f52c..d5789541da 100644 --- a/services/history-v1/storage/scripts/persist_and_expire_queues.sh +++ b/services/history-v1/storage/scripts/persist_and_expire_queues.sh @@ -1,3 +1,3 @@ #!/bin/sh node storage/scripts/persist_redis_chunks.mjs --queue --max-time 270 -node storage/scripts/expire_redis_chunks.js --post-request +node storage/scripts/expire_redis_chunks.js diff --git a/services/history-v1/test/acceptance/js/api/project_expiry.test.js b/services/history-v1/test/acceptance/js/api/project_expiry.test.js deleted file mode 100644 index efa589ec71..0000000000 --- a/services/history-v1/test/acceptance/js/api/project_expiry.test.js +++ /dev/null @@ -1,82 +0,0 @@ -'use strict' - -const BPromise = require('bluebird') -const { expect } = require('chai') -const HTTPStatus = require('http-status') -const fetch = require('node-fetch') -const fs = BPromise.promisifyAll(require('node:fs')) - -const cleanup = require('../storage/support/cleanup') -const fixtures = require('../storage/support/fixtures') -const testFiles = require('../storage/support/test_files') -const testProjects = require('./support/test_projects') -const testServer = require('./support/test_server') - -const { Change, File, Operation } = require('overleaf-editor-core') -const queueChanges = require('../../../../storage/lib/queue_changes') -const { getState } = require('../../../../storage/lib/chunk_store/redis') - -describe('project expiry', function () { - beforeEach(cleanup.everything) - beforeEach(fixtures.create) - - it('expire redis buffer', async function () { - const basicAuthClient = testServer.basicAuthClient - const projectId = await testProjects.createEmptyProject() - - // upload an empty file - const response = await fetch( - testServer.url( - `/api/projects/${projectId}/blobs/${File.EMPTY_FILE_HASH}`, - { qs: { pathname: 'main.tex' } } - ), - { - method: 'PUT', - body: fs.createReadStream(testFiles.path('empty.tex')), - headers: { - Authorization: testServer.basicAuthHeader, - }, - } - ) - expect(response.ok).to.be.true - - const testFile = File.fromHash(File.EMPTY_FILE_HASH) - const testChange = new Change( - [Operation.addFile('main.tex', testFile)], - new Date() - ) - await queueChanges(projectId, [testChange], 0) - - // Verify that the changes are queued and not yet persisted - const initialState = await getState(projectId) - expect(initialState.persistedVersion).to.be.null - expect(initialState.changes).to.have.lengthOf(1) - - const importResponse = - await basicAuthClient.apis.ProjectImport.flushChanges({ - project_id: projectId, - }) - - expect(importResponse.status).to.equal(HTTPStatus.OK) - - // Verify that the changes were persisted to the chunk store - const flushedState = await getState(projectId) - expect(flushedState.persistedVersion).to.equal(1) - - const expireResponse = - await basicAuthClient.apis.ProjectImport.expireProject({ - project_id: projectId, - }) - expect(expireResponse.status).to.equal(HTTPStatus.OK) - - const finalState = await getState(projectId) - expect(finalState).to.deep.equal({ - changes: [], - expireTime: null, - headSnapshot: null, - headVersion: null, - persistTime: null, - persistedVersion: null, - }) - }) -})