[document-updater] add endpoint for project wide last updated timestamp (#24352)

* [document-updater] fix acceptance test for sending two updates

The Object.create() invocation yields an empty object. The following v
assignment works as expected. The effective update is { v: 43 }.
Processing that fails as no operations were included.

* [document-updater] add endpoint project wide last updated timestamp

* [document-updater] apply review feedback

Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com>

---------

Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com>
GitOrigin-RevId: 81397537bfd85c2077f19669860b1391c15b34a3

services/document-updater/app.js

@@ -147,6 +147,10 @@ app.post(
   '/project/:project_id/get_and_flush_if_old',
   HttpController.getProjectDocsAndFlushIfOld
 )
+app.get(
+  '/project/:project_id/last_updated_at',
+  HttpController.getProjectLastUpdatedAt
+)
 app.post('/project/:project_id/clearState', HttpController.clearProjectState)
 app.post('/project/:project_id/doc/:doc_id', HttpController.setDoc)
 app.post('/project/:project_id/doc/:doc_id/append', HttpController.appendToDoc)

services/document-updater/app/js/HttpController.js

@@ -129,6 +129,22 @@ function getProjectDocsAndFlushIfOld(req, res, next) {
   )
 }
 
+function getProjectLastUpdatedAt(req, res, next) {
+  const projectId = req.params.project_id
+  ProjectManager.getProjectDocsTimestamps(projectId, (err, timestamps) => {
+    if (err) return next(err)
+
+    // Filter out nulls. This can happen when
+    // - docs get flushed between the listing and getting the individual docs ts
+    // - a doc flush failed half way (doc keys removed, project tracking not updated)
+    timestamps = timestamps.filter(ts => !!ts)
+
+    timestamps = timestamps.map(ts => parseInt(ts, 10))
+    timestamps.sort((a, b) => (a > b ? 1 : -1))
+    res.json({ lastUpdatedAt: timestamps.pop() })
+  })
+}
+
 function clearProjectState(req, res, next) {
   const projectId = req.params.project_id
   const timer = new Metrics.Timer('http.clearProjectState')
@@ -521,6 +537,7 @@ module.exports = {
   getDoc,
   peekDoc,
   getProjectDocsAndFlushIfOld,
+  getProjectLastUpdatedAt,
   clearProjectState,
   appendToDoc,
   setDoc,

services/document-updater/test/acceptance/js/ApplyingUpdatesToADocTests.js

@@ -109,11 +109,40 @@ describe('Applying updates to a doc', function () {
       )
     })
 
+    it('should yield last updated time', function (done) {
+      DocUpdaterClient.getProjectLastUpdatedAt(
+        this.project_id,
+        (error, res, body) => {
+          if (error != null) {
+            throw error
+          }
+          res.statusCode.should.equal(200)
+          body.lastUpdatedAt.should.be.within(this.startTime, Date.now())
+          done()
+        }
+      )
+    })
+
+    it('should yield no last updated time for another project', function (done) {
+      DocUpdaterClient.getProjectLastUpdatedAt(
+        DocUpdaterClient.randomId(),
+        (error, res, body) => {
+          if (error != null) {
+            throw error
+          }
+          res.statusCode.should.equal(200)
+          body.should.deep.equal({})
+          done()
+        }
+      )
+    })
+
     describe('when sending another update', function () {
       before(function (done) {
-        this.timeout = 10000
-        this.second_update = Object.create(this.update)
+        this.timeout(10000)
+        this.second_update = Object.assign({}, this.update)
         this.second_update.v = this.version + 1
+        this.secondStartTime = Date.now()
         DocUpdaterClient.sendUpdate(
           this.project_id,
           this.doc_id,
@@ -127,6 +156,24 @@ describe('Applying updates to a doc', function () {
         )
       })
 
+      it('should update the doc', function (done) {
+        DocUpdaterClient.getDoc(
+          this.project_id,
+          this.doc_id,
+          (error, res, doc) => {
+            if (error) done(error)
+            doc.lines.should.deep.equal([
+              'one',
+              'one and a half',
+              'one and a half',
+              'two',
+              'three',
+            ])
+            done()
+          }
+        )
+      })
+
       it('should not change the first op timestamp', function (done) {
         rclientProjectHistory.get(
           ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
@@ -142,6 +189,23 @@ describe('Applying updates to a doc', function () {
           }
         )
       })
+
+      it('should yield last updated time', function (done) {
+        DocUpdaterClient.getProjectLastUpdatedAt(
+          this.project_id,
+          (error, res, body) => {
+            if (error != null) {
+              throw error
+            }
+            res.statusCode.should.equal(200)
+            body.lastUpdatedAt.should.be.within(
+              this.secondStartTime,
+              Date.now()
+            )
+            done()
+          }
+        )
+      })
     })
   })
 

services/document-updater/test/acceptance/js/helpers/DocUpdaterClient.js

@@ -119,6 +119,18 @@ module.exports = DocUpdaterClient = {
     )
   },
 
+  getProjectLastUpdatedAt(projectId, callback) {
+    request.get(
+      `http://127.0.0.1:3003/project/${projectId}/last_updated_at`,
+      (error, res, body) => {
+        if (body != null && res.statusCode >= 200 && res.statusCode < 300) {
+          body = JSON.parse(body)
+        }
+        callback(error, res, body)
+      }
+    )
+  },
+
   preloadDoc(projectId, docId, callback) {
     DocUpdaterClient.getDoc(projectId, docId, callback)
   },

services/web/app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js

@@ -11,6 +11,22 @@ const ProjectGetter = require('../Project/ProjectGetter')
 const FileStoreHandler = require('../FileStore/FileStoreHandler')
 const Features = require('../../infrastructure/Features')
 
+function getProjectLastUpdatedAt(projectId, callback) {
+  _makeRequest(
+    {
+      path: `/project/${projectId}/last_updated_at`,
+      method: 'GET',
+      json: true,
+    },
+    projectId,
+    'project.redis.last_updated_at',
+    (err, body) => {
+      if (err || !body?.lastUpdatedAt) return callback(err, null)
+      callback(null, new Date(body.lastUpdatedAt))
+    }
+  )
+}
+
 /**
  * @param {string} projectId
  */
@@ -597,6 +613,7 @@ module.exports = {
   deleteDoc,
   getComment,
   getDocument,
+  getProjectLastUpdatedAt,
   setDocument,
   appendToDocument,
   getProjectDocsIfMatch,
@@ -624,6 +641,7 @@ module.exports = {
     ]),
     setDocument: promisify(setDocument),
     getProjectDocsIfMatch: promisify(getProjectDocsIfMatch),
+    getProjectLastUpdatedAt: promisify(getProjectLastUpdatedAt),
     clearProjectState: promisify(clearProjectState),
     acceptChanges: promisify(acceptChanges),
     resolveThread: promisify(resolveThread),