Merge pull request #26539 from overleaf/jpa-post

[history-v1] use POST requests for expiring redis buffer from cron

GitOrigin-RevId: 51c9a25b998e581ed20c0e113bd4989537a1e6ef

services/history-v1/api/controllers/project_import.js

@@ -28,6 +28,7 @@ const InvalidChangeError = storage.InvalidChangeError
 
 const render = require('./render')
 const Rollout = require('../app/rollout')
+const redisBackend = require('../../storage/lib/chunk_store/redis')
 
 const rollout = new Rollout(config)
 rollout.report(logger) // display the rollout configuration in the logs
@@ -177,6 +178,13 @@ async function flushChanges(req, res, next) {
   }
 }
 
+async function expireProject(req, res, next) {
+  const projectId = req.swagger.params.project_id.value
+  await redisBackend.expireProject(projectId)
+  res.status(HTTPStatus.OK).end()
+}
+
 exports.importSnapshot = expressify(importSnapshot)
 exports.importChanges = expressify(importChanges)
 exports.flushChanges = expressify(flushChanges)
+exports.expireProject = expressify(expireProject)

services/history-v1/api/swagger/project_import.js

@@ -174,10 +174,46 @@ const flushChanges = {
   ],
 }
 
+const expireProject = {
+  'x-swagger-router-controller': 'project_import',
+  operationId: 'expireProject',
+  tags: ['ProjectImport'],
+  description: 'Expire project changes from buffer.',
+  parameters: [
+    {
+      name: 'project_id',
+      in: 'path',
+      description: 'project id',
+      required: true,
+      type: 'string',
+    },
+  ],
+  responses: {
+    200: {
+      description: 'Success',
+      schema: {
+        $ref: '#/definitions/Project',
+      },
+    },
+    404: {
+      description: 'Not Found',
+      schema: {
+        $ref: '#/definitions/Error',
+      },
+    },
+  },
+  security: [
+    {
+      basic: [],
+    },
+  ],
+}
+
 exports.paths = {
   '/projects/{project_id}/import': { post: importSnapshot },
   '/projects/{project_id}/legacy_import': { post: importSnapshot },
   '/projects/{project_id}/changes': { get: getChanges, post: importChanges },
   '/projects/{project_id}/legacy_changes': { post: importChanges },
   '/projects/{project_id}/flush': { post: flushChanges },
+  '/projects/{project_id}/expire': { post: expireProject },
 }

services/history-v1/storage/scripts/expire_redis_chunks.js

@@ -3,18 +3,47 @@ const commandLineArgs = require('command-line-args')
 const redis = require('../lib/redis')
 const { scanAndProcessDueItems } = require('../lib/scan')
 const { expireProject, claimExpireJob } = require('../lib/chunk_store/redis')
+const config = require('config')
+const { fetchNothing } = require('@overleaf/fetch-utils')
 
 const rclient = redis.rclientHistory
 
-const optionDefinitions = [{ name: 'dry-run', alias: 'd', type: Boolean }]
+const optionDefinitions = [
+  { name: 'dry-run', alias: 'd', type: Boolean },
+  { name: 'post-request', type: Boolean },
+]
 const options = commandLineArgs(optionDefinitions)
 const DRY_RUN = options['dry-run'] || false
+const POST_REQUEST = options['post-request'] || false
+const HISTORY_V1_URL = `http://${process.env.HISTORY_V1_HOST || 'localhost'}:${process.env.PORT || 3100}`
 
 logger.initialize('expire-redis-chunks')
 
 async function expireProjectAction(projectId) {
   const job = await claimExpireJob(projectId)
-  await expireProject(projectId)
+  if (POST_REQUEST) {
+    await requestProjectExpiry(projectId)
+  } else {
+    await expireProject(projectId)
+  }
+  if (job && job.close) {
+    await job.close()
+  }
+}
+
+async function requestProjectExpiry(projectId) {
+  const job = await claimExpireJob(projectId)
+  logger.debug({ projectId }, 'sending project expire request')
+  const url = `${HISTORY_V1_URL}/api/projects/${projectId}/expire`
+  const credentials = Buffer.from(
+    `staging:${config.get('basicHttpAuth.password')}`
+  ).toString('base64')
+  await fetchNothing(url, {
+    method: 'POST',
+    headers: {
+      Authorization: `Basic ${credentials}`,
+    },
+  })
   if (job && job.close) {
     await job.close()
   }

services/history-v1/storage/scripts/persist_and_expire_queues.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 node storage/scripts/persist_redis_chunks.mjs --queue --max-time 270
-node storage/scripts/expire_redis_chunks.js
+node storage/scripts/expire_redis_chunks.js --post-request

services/history-v1/test/acceptance/js/api/project_expiry.test.js

@@ -0,0 +1,82 @@
+'use strict'
+
+const BPromise = require('bluebird')
+const { expect } = require('chai')
+const HTTPStatus = require('http-status')
+const fetch = require('node-fetch')
+const fs = BPromise.promisifyAll(require('node:fs'))
+
+const cleanup = require('../storage/support/cleanup')
+const fixtures = require('../storage/support/fixtures')
+const testFiles = require('../storage/support/test_files')
+const testProjects = require('./support/test_projects')
+const testServer = require('./support/test_server')
+
+const { Change, File, Operation } = require('overleaf-editor-core')
+const queueChanges = require('../../../../storage/lib/queue_changes')
+const { getState } = require('../../../../storage/lib/chunk_store/redis')
+
+describe('project expiry', function () {
+  beforeEach(cleanup.everything)
+  beforeEach(fixtures.create)
+
+  it('expire redis buffer', async function () {
+    const basicAuthClient = testServer.basicAuthClient
+    const projectId = await testProjects.createEmptyProject()
+
+    // upload an empty file
+    const response = await fetch(
+      testServer.url(
+        `/api/projects/${projectId}/blobs/${File.EMPTY_FILE_HASH}`,
+        { qs: { pathname: 'main.tex' } }
+      ),
+      {
+        method: 'PUT',
+        body: fs.createReadStream(testFiles.path('empty.tex')),
+        headers: {
+          Authorization: testServer.basicAuthHeader,
+        },
+      }
+    )
+    expect(response.ok).to.be.true
+
+    const testFile = File.fromHash(File.EMPTY_FILE_HASH)
+    const testChange = new Change(
+      [Operation.addFile('main.tex', testFile)],
+      new Date()
+    )
+    await queueChanges(projectId, [testChange], 0)
+
+    // Verify that the changes are queued and not yet persisted
+    const initialState = await getState(projectId)
+    expect(initialState.persistedVersion).to.be.null
+    expect(initialState.changes).to.have.lengthOf(1)
+
+    const importResponse =
+      await basicAuthClient.apis.ProjectImport.flushChanges({
+        project_id: projectId,
+      })
+
+    expect(importResponse.status).to.equal(HTTPStatus.OK)
+
+    // Verify that the changes were persisted to the chunk store
+    const flushedState = await getState(projectId)
+    expect(flushedState.persistedVersion).to.equal(1)
+
+    const expireResponse =
+      await basicAuthClient.apis.ProjectImport.expireProject({
+        project_id: projectId,
+      })
+    expect(expireResponse.status).to.equal(HTTPStatus.OK)
+
+    const finalState = await getState(projectId)
+    expect(finalState).to.deep.equal({
+      changes: [],
+      expireTime: null,
+      headSnapshot: null,
+      headVersion: null,
+      persistTime: null,
+      persistedVersion: null,
+    })
+  })
+})