Merge pull request #28945 from overleaf/mj-tear-down-server-side-referencing

[web] Tear down server side referencing

GitOrigin-RevId: 37feac39cc7bf219a2cbc463191163534434f267

services/web/app/src/Features/LinkedFiles/LinkedFilesController.mjs

@@ -19,12 +19,10 @@ import LinkedFilesErrors from './LinkedFilesErrors.mjs'
 import {
   OutputFileFetchFailedError,
   FileTooLargeError,
-  OError,
 } from '../Errors/Errors.js'
 import Modules from '../../infrastructure/Modules.js'
 import { plainTextResponse } from '../../infrastructure/Response.js'
 import { z, zz, validateReq } from '../../infrastructure/Validation.js'
-import ReferencesHandler from '../References/ReferencesHandler.mjs'
 import EditorRealTimeController from '../Editor/EditorRealTimeController.js'
 import { expressify } from '@overleaf/promise-utils'
 import ProjectOutputFileAgent from './ProjectOutputFileAgent.mjs'
@@ -142,26 +140,16 @@ async function refreshLinkedFile(req, res, next) {
   }
 
   if (req.body.shouldReindexReferences) {
-    let data
-    try {
-      data = await ReferencesHandler.promises.indexAll(projectId)
-    } catch (error) {
-      OError.tag(error, 'failed to index references', {
-        projectId,
-      })
-      return next(error)
-    }
+    // Signal to clients that they should re-index references
     EditorRealTimeController.emitToRoom(
       projectId,
       'references:keys:updated',
-      data.keys,
+      [],
       true,
       clientId
     )
-    res.json({ new_file_id: newFileId })
-  } else {
-    res.json({ new_file_id: newFileId })
   }
+  res.json({ new_file_id: newFileId })
 }
 
 export default LinkedFilesController = {

services/web/app/src/Features/References/ReferencesController.mjs

@@ -1,63 +1,20 @@
-/* eslint-disable
-    max-len,
-    no-unused-vars,
-*/
-// TODO: This file was created by bulk-decaffeinate.
-// Fix any style issues and re-enable lint.
-/*
- * decaffeinate suggestions:
- * DS102: Remove unnecessary code created because of implicit returns
- * DS207: Consider shorter variations of null checks
- * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
- */
-import ReferencesHandler from './ReferencesHandler.mjs'
 import EditorRealTimeController from '../Editor/EditorRealTimeController.js'
-import { OError } from '../Errors/Errors.js'
 
-let ReferencesController
-
-export default ReferencesController = {
+export default {
   indexAll(req, res, next) {
     const projectId = req.params.Project_id
     const { shouldBroadcast, clientId } = req.body
-    return ReferencesHandler.indexAll(projectId, function (error, data) {
-      if (error) {
-        OError.tag(error, 'failed to index references', { projectId })
-        return next(error)
-      }
-      return ReferencesController._handleIndexResponse(
-        req,
-        res,
-        projectId,
-        shouldBroadcast,
-        true,
-        data,
-        clientId
-      )
-    })
-  },
-
-  _handleIndexResponse(
-    req,
-    res,
-    projectId,
-    shouldBroadcast,
-    isAllDocs,
-    data,
-    clientId
-  ) {
-    if (data == null || data.keys == null) {
-      return res.json({ projectId, keys: [] })
-    }
+    // We've migrated to client side indexing, so we only use the message for
+    // broadcasting that the clients need to re-index.
     if (shouldBroadcast) {
       EditorRealTimeController.emitToRoom(
         projectId,
         'references:keys:updated',
-        data.keys,
-        isAllDocs,
+        [],
+        true,
         clientId
       )
     }
-    return res.json(data)
+    res.json({ projectId, keys: [] })
   },
 }

services/web/app/src/Features/References/ReferencesHandler.mjs

@@ -1,214 +0,0 @@
-/* eslint-disable
-    n/handle-callback-err,
-    max-len,
-    no-unused-vars,
-*/
-// TODO: This file was created by bulk-decaffeinate.
-// Fix any style issues and re-enable lint.
-/*
- * decaffeinate suggestions:
- * DS102: Remove unnecessary code created because of implicit returns
- * DS103: Rewrite code to no longer use __guard__
- * DS207: Consider shorter variations of null checks
- * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
- */
-import OError from '@overleaf/o-error'
-import logger from '@overleaf/logger'
-import request from 'request'
-import settings from '@overleaf/settings'
-import Features from '../../infrastructure/Features.js'
-import ProjectGetter from '../Project/ProjectGetter.mjs'
-import UserGetter from '../User/UserGetter.js'
-import DocumentUpdaterHandler from '../DocumentUpdater/DocumentUpdaterHandler.mjs'
-import _ from 'lodash'
-import Async from 'async'
-import Errors from '../Errors/Errors.js'
-import { promisify } from '@overleaf/promise-utils'
-
-let ReferencesHandler
-
-if (!Features.hasFeature('references')) {
-  logger.debug('references search not enabled')
-}
-
-export default ReferencesHandler = {
-  _buildDocUrl(projectId, docId) {
-    return {
-      url: `${settings.apis.docstore.url}/project/${projectId}/doc/${docId}/raw`,
-    }
-  },
-
-  _findBibFileRefs(project) {
-    const fileRefs = []
-    function _process(folder) {
-      _.forEach(folder.fileRefs || [], function (file) {
-        if (
-          __guard__(file != null ? file.name : undefined, x1 =>
-            x1.match(/^.*\.bib$/)
-          )
-        ) {
-          return fileRefs.push(file)
-        }
-      })
-      return _.forEach(folder.folders || [], folder => _process(folder))
-    }
-    _.forEach(project.rootFolder || [], rootFolder => _process(rootFolder))
-    return fileRefs
-  },
-
-  _findBibDocIds(project) {
-    const ids = []
-    function _process(folder) {
-      _.forEach(folder.docs || [], function (doc) {
-        if (
-          __guard__(doc != null ? doc.name : undefined, x1 =>
-            x1.match(/^.*\.bib$/)
-          )
-        ) {
-          return ids.push(doc._id)
-        }
-      })
-      return _.forEach(folder.folders || [], folder => _process(folder))
-    }
-    _.forEach(project.rootFolder || [], rootFolder => _process(rootFolder))
-    return ids
-  },
-
-  _isFullIndex(project, callback) {
-    if (callback == null) {
-      callback = function () {}
-    }
-    return UserGetter.getUser(
-      project.owner_ref,
-      { features: true },
-      function (err, owner) {
-        if (err != null) {
-          return callback(err)
-        }
-        const features = owner != null ? owner.features : undefined
-        return callback(
-          null,
-          (features != null ? features.references : undefined) === true ||
-            (features != null ? features.referencesSearch : undefined) === true
-        )
-      }
-    )
-  },
-
-  indexAll(projectId, callback) {
-    if (callback == null) {
-      callback = function () {}
-    }
-    return ProjectGetter.getProject(
-      projectId,
-      { rootFolder: true, owner_ref: 1, 'overleaf.history.id': 1 },
-      function (err, project) {
-        if (err) {
-          OError.tag(err, 'error finding project', {
-            projectId,
-          })
-          return callback(err)
-        }
-        if (!project) {
-          return callback(
-            new Errors.NotFoundError(`project does not exist: ${projectId}`)
-          )
-        }
-        logger.debug({ projectId }, 'indexing all bib files in project')
-        const docIds = ReferencesHandler._findBibDocIds(project)
-        const fileRefs = ReferencesHandler._findBibFileRefs(project)
-        return ReferencesHandler._doIndexOperation(
-          projectId,
-          project,
-          docIds,
-          fileRefs,
-          callback
-        )
-      }
-    )
-  },
-
-  _doIndexOperation(projectId, project, docIds, fileRefs, callback) {
-    if (!Features.hasFeature('references')) {
-      return callback()
-    }
-    const historyId = project?.overleaf?.history?.id
-    if (!historyId) {
-      return callback(
-        new OError('project does not have a history id', { projectId })
-      )
-    }
-    return ReferencesHandler._isFullIndex(project, function (err, isFullIndex) {
-      if (err) {
-        OError.tag(err, 'error checking whether to do full index', {
-          projectId,
-        })
-        return callback(err)
-      }
-      logger.debug(
-        { projectId, docIds },
-        'flushing docs to mongo before calling references service'
-      )
-      return Async.series(
-        docIds.map(
-          docId => cb =>
-            DocumentUpdaterHandler.flushDocToMongo(projectId, docId, cb)
-        ),
-        function (err) {
-          // continue
-          if (err) {
-            OError.tag(err, 'error flushing docs to mongo', {
-              projectId,
-              docIds,
-            })
-            return callback(err)
-          }
-          const bibDocUrls = docIds.map(docId =>
-            ReferencesHandler._buildDocUrl(projectId, docId)
-          )
-          const bibFileUrls = fileRefs.map(fileRef => ({
-            url: `${settings.apis.project_history.url}/project/${historyId}/blob/${fileRef.hash}`,
-          }))
-          const sourceURLs = bibDocUrls.concat(bibFileUrls)
-          return request.post(
-            {
-              url: `${settings.apis.references.url}/project/${projectId}/index`,
-              json: {
-                docUrls: sourceURLs.map(item => item.url),
-                sourceURLs,
-                fullIndex: isFullIndex,
-              },
-            },
-            function (err, res, data) {
-              if (err) {
-                OError.tag(err, 'error communicating with references api', {
-                  projectId,
-                })
-                return callback(err)
-              }
-              if (res.statusCode >= 200 && res.statusCode < 300) {
-                logger.debug({ projectId }, 'got keys from references api')
-                return callback(null, data)
-              } else {
-                err = new Error(
-                  `references api responded with non-success code: ${res.statusCode}`
-                )
-                return callback(err)
-              }
-            }
-          )
-        }
-      )
-    })
-  },
-}
-
-ReferencesHandler.promises = {
-  indexAll: promisify(ReferencesHandler.indexAll),
-}
-
-function __guard__(value, transform) {
-  return typeof value !== 'undefined' && value !== null
-    ? transform(value)
-    : undefined
-}

services/web/test/unit/src/LinkedFiles/LinkedFilesController.test.mjs

@@ -34,8 +34,7 @@ describe('LinkedFilesController', function () {
     ctx.SessionManager = {
       getLoggedInUserId: sinon.stub().returns(ctx.userId),
     }
-    ctx.EditorRealTimeController = {}
-    ctx.ReferencesHandler = {}
+    ctx.EditorRealTimeController = { emitToRoom: sinon.stub() }
     ctx.UrlAgent = {}
     ctx.ProjectFileAgent = {}
     ctx.ProjectOutputFileAgent = {}
@@ -74,13 +73,6 @@ describe('LinkedFilesController', function () {
       })
     )
 
-    vi.doMock(
-      '../../../../app/src/Features/References/ReferencesHandler',
-      () => ({
-        default: ctx.ReferencesHandler,
-      })
-    )
-
     vi.doMock('../../../../app/src/Features/LinkedFiles/UrlAgent', () => ({
       default: ctx.UrlAgent,
     }))
@@ -200,5 +192,38 @@ describe('LinkedFilesController', function () {
         ctx.LinkedFilesController.refreshLinkedFile(ctx.req, ctx.res, ctx.next)
       })
     })
+
+    describe('when bib file re-indexing is required', function () {
+      const clientId = 'client-id'
+      beforeEach(function (ctx) {
+        ctx.req.body.shouldReindexReferences = true
+        ctx.req.body.clientId = clientId
+      })
+
+      it('informs clients to re-index bib references', async function (ctx) {
+        await new Promise(resolve => {
+          ctx.next = sinon.stub().callsFake(() => resolve('unexpected error'))
+          ctx.res = {
+            json: () => {
+              expect(
+                ctx.EditorRealTimeController.emitToRoom
+              ).to.have.been.calledWith(
+                ctx.projectId,
+                'references:keys:updated',
+                [],
+                true,
+                clientId
+              )
+              resolve()
+            },
+          }
+          ctx.LinkedFilesController.refreshLinkedFile(
+            ctx.req,
+            ctx.res,
+            ctx.next
+          )
+        })
+      })
+    })
   })
 })

services/web/test/unit/src/References/ReferencesController.test.mjs

@@ -15,16 +15,6 @@ describe('ReferencesController', function () {
       }),
     }))
 
-    vi.doMock(
-      '../../../../app/src/Features/References/ReferencesHandler',
-      () => ({
-        default: (ctx.ReferencesHandler = {
-          index: sinon.stub(),
-          indexAll: sinon.stub(),
-        }),
-      })
-    )
-
     vi.doMock(
       '../../../../app/src/Features/Editor/EditorRealTimeController',
       () => ({
@@ -45,16 +35,15 @@ describe('ReferencesController', function () {
     ctx.res.json = sinon.stub()
     ctx.res.sendStatus = sinon.stub()
     ctx.next = sinon.stub()
-    ctx.fakeResponseData = {
+    ctx.expectedResponseData = {
       projectId: ctx.projectId,
-      keys: ['one', 'two', 'three'],
+      keys: [],
     }
   })
 
   describe('indexAll', function () {
     beforeEach(function (ctx) {
       ctx.req.body = { shouldBroadcast: false }
-      ctx.ReferencesHandler.indexAll.callsArgWith(1, null, ctx.fakeResponseData)
       ctx.call = callback => {
         ctx.controller.indexAll(ctx.req, ctx.res, ctx.next)
         return callback()
@@ -72,23 +61,11 @@ describe('ReferencesController', function () {
       })
     })
 
-    it('should return data', async function (ctx) {
+    it('should return expected empty data', async function (ctx) {
       await new Promise(resolve => {
         ctx.call(() => {
           ctx.res.json.callCount.should.equal(1)
-          ctx.res.json.calledWith(ctx.fakeResponseData).should.equal(true)
-          resolve()
-        })
-      })
-    })
-
-    it('should call ReferencesHandler.indexAll', async function (ctx) {
-      await new Promise(resolve => {
-        ctx.call(() => {
-          ctx.ReferencesHandler.indexAll.callCount.should.equal(1)
-          ctx.ReferencesHandler.indexAll
-            .calledWith(ctx.projectId)
-            .should.equal(true)
+          ctx.res.json.calledWith(ctx.expectedResponseData).should.equal(true)
           resolve()
         })
       })
@@ -96,7 +73,6 @@ describe('ReferencesController', function () {
 
     describe('when shouldBroadcast is true', function () {
       beforeEach(function (ctx) {
-        ctx.ReferencesHandler.index.callsArgWith(2, null, ctx.fakeResponseData)
         ctx.req.body.shouldBroadcast = true
       })
 
@@ -120,11 +96,11 @@ describe('ReferencesController', function () {
         })
       })
 
-      it('should still return data', async function (ctx) {
+      it('should still return empty data', async function (ctx) {
         await new Promise(resolve => {
           ctx.call(() => {
             ctx.res.json.callCount.should.equal(1)
-            ctx.res.json.calledWith(ctx.fakeResponseData).should.equal(true)
+            ctx.res.json.calledWith(ctx.expectedResponseData).should.equal(true)
             resolve()
           })
         })
@@ -133,7 +109,6 @@ describe('ReferencesController', function () {
 
     describe('when shouldBroadcast is false', function () {
       beforeEach(function (ctx) {
-        ctx.ReferencesHandler.index.callsArgWith(2, null, ctx.fakeResponseData)
         ctx.req.body.shouldBroadcast = false
       })
 
@@ -157,57 +132,15 @@ describe('ReferencesController', function () {
         })
       })
 
-      it('should still return data', async function (ctx) {
+      it('should still return empty data', async function (ctx) {
         await new Promise(resolve => {
           ctx.call(() => {
             ctx.res.json.callCount.should.equal(1)
-            ctx.res.json.calledWith(ctx.fakeResponseData).should.equal(true)
+            ctx.res.json.calledWith(ctx.expectedResponseData).should.equal(true)
             resolve()
           })
         })
       })
     })
   })
-
-  describe('there is no data', function () {
-    beforeEach(function (ctx) {
-      ctx.ReferencesHandler.indexAll.callsArgWith(1)
-      ctx.call = callback => {
-        ctx.controller.indexAll(ctx.req, ctx.res, ctx.next)
-        callback()
-      }
-    })
-
-    it('should not call EditorRealTimeController.emitToRoom', async function (ctx) {
-      await new Promise(resolve => {
-        ctx.call(() => {
-          ctx.EditorRealTimeController.emitToRoom.callCount.should.equal(0)
-          resolve()
-        })
-      })
-    })
-
-    it('should not produce an error', async function (ctx) {
-      await new Promise(resolve => {
-        ctx.call(() => {
-          ctx.res.sendStatus.callCount.should.equal(0)
-          ctx.res.sendStatus.calledWith(500).should.equal(false)
-          ctx.res.sendStatus.calledWith(400).should.equal(false)
-          resolve()
-        })
-      })
-    })
-
-    it('should send a response with an empty keys list', async function (ctx) {
-      await new Promise(resolve => {
-        ctx.call(() => {
-          ctx.res.json.called.should.equal(true)
-          ctx.res.json
-            .calledWith({ projectId: ctx.projectId, keys: [] })
-            .should.equal(true)
-          resolve()
-        })
-      })
-    })
-  })
 })

services/web/test/unit/src/References/ReferencesHandler.test.mjs

@@ -1,444 +0,0 @@
-import { expect, vi } from 'vitest'
-import sinon from 'sinon'
-import Errors from '../../../../app/src/Features/Errors/Errors.js'
-const modulePath =
-  '../../../../app/src/Features/References/ReferencesHandler.mjs'
-
-vi.mock('../../../../app/src/Features/Errors/Errors.js', () =>
-  vi.importActual('../../../../app/src/Features/Errors/Errors.js')
-)
-
-describe('ReferencesHandler', function () {
-  beforeEach(async function (ctx) {
-    ctx.projectId = '222'
-    ctx.historyId = 42
-    ctx.fakeProject = {
-      _id: ctx.projectId,
-      owner_ref: (ctx.fakeOwner = {
-        _id: 'some_owner',
-        features: {
-          references: false,
-        },
-      }),
-      rootFolder: [
-        {
-          docs: [
-            { name: 'one.bib', _id: 'aaa' },
-            { name: 'two.txt', _id: 'bbb' },
-          ],
-          folders: [
-            {
-              docs: [{ name: 'three.bib', _id: 'ccc' }],
-              fileRefs: [
-                { name: 'four.bib', _id: 'fff', hash: 'abc' },
-                { name: 'five.bib', _id: 'ggg', hash: 'def' },
-              ],
-              folders: [],
-            },
-          ],
-        },
-      ],
-      overleaf: { history: { id: ctx.historyId } },
-    }
-    ctx.docIds = ['aaa', 'ccc']
-
-    vi.doMock('@overleaf/settings', () => ({
-      default: (ctx.settings = {
-        apis: {
-          references: { url: 'http://some.url/references' },
-          docstore: { url: 'http://some.url/docstore' },
-          filestore: { url: 'http://some.url/filestore' },
-          project_history: { url: 'http://project-history.local' },
-        },
-      }),
-    }))
-
-    vi.doMock('request', () => ({
-      default: (ctx.request = {
-        get: sinon.stub(),
-        post: sinon.stub(),
-      }),
-    }))
-
-    vi.doMock('../../../../app/src/Features/Project/ProjectGetter', () => ({
-      default: (ctx.ProjectGetter = {
-        getProject: sinon.stub().callsArgWith(2, null, ctx.fakeProject),
-      }),
-    }))
-
-    vi.doMock('../../../../app/src/Features/User/UserGetter', () => ({
-      default: (ctx.UserGetter = {
-        getUser: sinon.stub(),
-      }),
-    }))
-
-    vi.doMock(
-      '../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler',
-      () => ({
-        default: (ctx.DocumentUpdaterHandler = {
-          flushDocToMongo: sinon.stub().callsArgWith(2, null),
-        }),
-      })
-    )
-
-    vi.doMock('../../../../app/src/infrastructure/Features', () => ({
-      default: (ctx.Features = {
-        hasFeature: sinon.stub().returns(true),
-      }),
-    }))
-
-    ctx.handler = (await import(modulePath)).default
-    ctx.fakeResponseData = {
-      projectId: ctx.projectId,
-      keys: ['k1', 'k2'],
-    }
-  })
-
-  describe('indexAll', function () {
-    beforeEach(function (ctx) {
-      sinon.stub(ctx.handler, '_findBibDocIds').returns(['aaa', 'ccc'])
-      sinon.stub(ctx.handler, '_findBibFileRefs').returns([
-        { _id: 'fff', hash: 'abc' },
-        { _id: 'ggg', hash: 'def' },
-      ])
-      sinon.stub(ctx.handler, '_isFullIndex').callsArgWith(1, null, true)
-      ctx.request.post.callsArgWith(
-        1,
-        null,
-        { statusCode: 200 },
-        ctx.fakeResponseData
-      )
-      return (ctx.call = callback => {
-        return ctx.handler.indexAll(ctx.projectId, callback)
-      })
-    })
-
-    it('should call _findBibDocIds', async function (ctx) {
-      await new Promise(resolve => {
-        return ctx.call((err, data) => {
-          expect(err).to.be.null
-          ctx.handler._findBibDocIds.callCount.should.equal(1)
-          ctx.handler._findBibDocIds
-            .calledWith(ctx.fakeProject)
-            .should.equal(true)
-          return resolve()
-        })
-      })
-    })
-
-    it('should call _findBibFileRefs', async function (ctx) {
-      await new Promise(resolve => {
-        return ctx.call((err, data) => {
-          expect(err).to.be.null
-          ctx.handler._findBibDocIds.callCount.should.equal(1)
-          ctx.handler._findBibDocIds
-            .calledWith(ctx.fakeProject)
-            .should.equal(true)
-          return resolve()
-        })
-      })
-    })
-
-    it('should call DocumentUpdaterHandler.flushDocToMongo', async function (ctx) {
-      await new Promise(resolve => {
-        return ctx.call((err, data) => {
-          expect(err).to.be.null
-          ctx.DocumentUpdaterHandler.flushDocToMongo.callCount.should.equal(2)
-          return resolve()
-        })
-      })
-    })
-
-    it('should make a request to references service', async function (ctx) {
-      await new Promise(resolve => {
-        return ctx.call((err, data) => {
-          expect(err).to.be.null
-          ctx.request.post.callCount.should.equal(1)
-          const arg = ctx.request.post.firstCall.args[0]
-          expect(arg.json).to.have.all.keys(
-            'docUrls',
-            'sourceURLs',
-            'fullIndex'
-          )
-          expect(arg.json.docUrls.length).to.equal(4)
-          expect(arg.json.docUrls).to.deep.equal([
-            `${ctx.settings.apis.docstore.url}/project/${ctx.projectId}/doc/aaa/raw`,
-            `${ctx.settings.apis.docstore.url}/project/${ctx.projectId}/doc/ccc/raw`,
-            `${ctx.settings.apis.project_history.url}/project/${ctx.historyId}/blob/abc`,
-            `${ctx.settings.apis.project_history.url}/project/${ctx.historyId}/blob/def`,
-          ])
-          expect(arg.json.sourceURLs.length).to.equal(4)
-          expect(arg.json.sourceURLs).to.deep.equal([
-            {
-              url: `${ctx.settings.apis.docstore.url}/project/${ctx.projectId}/doc/aaa/raw`,
-            },
-            {
-              url: `${ctx.settings.apis.docstore.url}/project/${ctx.projectId}/doc/ccc/raw`,
-            },
-            {
-              url: `${ctx.settings.apis.project_history.url}/project/${ctx.historyId}/blob/abc`,
-            },
-            {
-              url: `${ctx.settings.apis.project_history.url}/project/${ctx.historyId}/blob/def`,
-            },
-          ])
-          expect(arg.json.fullIndex).to.equal(true)
-          return resolve()
-        })
-      })
-    })
-
-    it('should not produce an error', async function (ctx) {
-      await new Promise(resolve => {
-        return ctx.call((err, data) => {
-          expect(err).to.equal(null)
-          return resolve()
-        })
-      })
-    })
-
-    it('should return data', async function (ctx) {
-      await new Promise(resolve => {
-        return ctx.call((err, data) => {
-          expect(err).to.be.null
-          expect(data).to.not.equal(null)
-          expect(data).to.not.equal(undefined)
-          expect(data).to.equal(ctx.fakeResponseData)
-          return resolve()
-        })
-      })
-    })
-
-    describe('when ProjectGetter.getProject produces an error', function () {
-      beforeEach(function (ctx) {
-        ctx.ProjectGetter.getProject.callsArgWith(2, new Error('woops'))
-      })
-
-      it('should produce an error', async function (ctx) {
-        await new Promise(resolve => {
-          ctx.call((err, data) => {
-            expect(err).to.not.equal(null)
-            expect(err).to.be.instanceof(Error)
-            expect(data).to.equal(undefined)
-            resolve()
-          })
-        })
-      })
-
-      it('should not send request', async function (ctx) {
-        await new Promise(resolve => {
-          ctx.call(() => {
-            ctx.request.post.callCount.should.equal(0)
-            resolve()
-          })
-        })
-      })
-    })
-
-    describe('when ProjectGetter.getProject returns null', function () {
-      beforeEach(function (ctx) {
-        ctx.ProjectGetter.getProject.callsArgWith(2, null)
-      })
-
-      it('should produce an error', async function (ctx) {
-        await new Promise(resolve => {
-          ctx.call((err, data) => {
-            expect(err).to.not.equal(null)
-            expect(err).to.be.instanceof(Errors.NotFoundError)
-            expect(data).to.equal(undefined)
-            resolve()
-          })
-        })
-      })
-
-      it('should not send request', async function (ctx) {
-        await new Promise(resolve => {
-          ctx.call(() => {
-            ctx.request.post.callCount.should.equal(0)
-            resolve()
-          })
-        })
-      })
-    })
-
-    describe('when _isFullIndex produces an error', function () {
-      beforeEach(function (ctx) {
-        ctx.ProjectGetter.getProject.callsArgWith(2, null, ctx.fakeProject)
-        ctx.handler._isFullIndex.callsArgWith(1, new Error('woops'))
-      })
-
-      it('should produce an error', async function (ctx) {
-        await new Promise(resolve => {
-          ctx.call((err, data) => {
-            expect(err).to.not.equal(null)
-            expect(err).to.be.instanceof(Error)
-            expect(data).to.equal(undefined)
-            resolve()
-          })
-        })
-      })
-
-      it('should not send request', async function (ctx) {
-        await new Promise(resolve => {
-          ctx.call(() => {
-            ctx.request.post.callCount.should.equal(0)
-            resolve()
-          })
-        })
-      })
-    })
-
-    describe('when flushDocToMongo produces an error', function () {
-      beforeEach(function (ctx) {
-        ctx.ProjectGetter.getProject.callsArgWith(2, null, ctx.fakeProject)
-        ctx.handler._isFullIndex.callsArgWith(1, false)
-        ctx.DocumentUpdaterHandler.flushDocToMongo.callsArgWith(
-          2,
-          new Error('woops')
-        )
-      })
-
-      it('should produce an error', async function (ctx) {
-        await new Promise(resolve => {
-          ctx.call((err, data) => {
-            expect(err).to.not.equal(null)
-            expect(err).to.be.instanceof(Error)
-            expect(data).to.equal(undefined)
-            resolve()
-          })
-        })
-      })
-
-      it('should not send request', async function (ctx) {
-        await new Promise(resolve => {
-          ctx.call(() => {
-            ctx.request.post.callCount.should.equal(0)
-            resolve()
-          })
-        })
-      })
-    })
-  })
-
-  describe('_findBibDocIds', function () {
-    beforeEach(function (ctx) {
-      ctx.fakeProject = {
-        rootFolder: [
-          {
-            docs: [
-              { name: 'one.bib', _id: 'aaa' },
-              { name: 'two.txt', _id: 'bbb' },
-            ],
-            folders: [
-              { docs: [{ name: 'three.bib', _id: 'ccc' }], folders: [] },
-            ],
-          },
-        ],
-      }
-      ctx.expectedIds = ['aaa', 'ccc']
-    })
-
-    it('should select the correct docIds', function (ctx) {
-      const result = ctx.handler._findBibDocIds(ctx.fakeProject)
-      expect(result).to.deep.equal(ctx.expectedIds)
-    })
-
-    it('should not error with a non array of folders from dirty data', function (ctx) {
-      ctx.fakeProject.rootFolder[0].folders[0].folders = {}
-      const result = ctx.handler._findBibDocIds(ctx.fakeProject)
-      expect(result).to.deep.equal(ctx.expectedIds)
-    })
-  })
-
-  describe('_findBibFileRefs', function () {
-    beforeEach(function (ctx) {
-      ctx.fakeProject = {
-        rootFolder: [
-          {
-            docs: [
-              { name: 'one.bib', _id: 'aaa' },
-              { name: 'two.txt', _id: 'bbb' },
-            ],
-            fileRefs: [{ name: 'other.bib', _id: 'ddd' }],
-            folders: [
-              {
-                docs: [{ name: 'three.bib', _id: 'ccc' }],
-                fileRefs: [{ name: 'four.bib', _id: 'ghg' }],
-                folders: [],
-              },
-            ],
-          },
-        ],
-      }
-      ctx.expectedIds = [
-        ctx.fakeProject.rootFolder[0].fileRefs[0],
-        ctx.fakeProject.rootFolder[0].folders[0].fileRefs[0],
-      ]
-    })
-
-    it('should select the correct docIds', function (ctx) {
-      const result = ctx.handler._findBibFileRefs(ctx.fakeProject)
-      expect(result).to.deep.equal(ctx.expectedIds)
-    })
-  })
-
-  describe('_isFullIndex', function () {
-    beforeEach(function (ctx) {
-      ctx.fakeProject = { owner_ref: (ctx.owner_ref = 'owner-ref-123') }
-      ctx.owner = {
-        features: {
-          references: false,
-        },
-      }
-      ctx.UserGetter.getUser = sinon.stub()
-      ctx.UserGetter.getUser
-        .withArgs(ctx.owner_ref, { features: true })
-        .yields(null, ctx.owner)
-      ctx.call = callback => {
-        ctx.handler._isFullIndex(ctx.fakeProject, callback)
-      }
-    })
-
-    describe('with references feature on', function () {
-      beforeEach(function (ctx) {
-        ctx.owner.features.references = true
-      })
-
-      it('should return true', function (ctx) {
-        ctx.call((err, isFullIndex) => {
-          expect(err).to.equal(null)
-          expect(isFullIndex).to.equal(true)
-        })
-      })
-    })
-
-    describe('with references feature off', function () {
-      beforeEach(function (ctx) {
-        ctx.owner.features.references = false
-      })
-
-      it('should return false', function (ctx) {
-        ctx.call((err, isFullIndex) => {
-          expect(err).to.equal(null)
-          expect(isFullIndex).to.equal(false)
-        })
-      })
-    })
-
-    describe('with referencesSearch', function () {
-      beforeEach(function (ctx) {
-        ctx.owner.features = {
-          referencesSearch: true,
-          references: false,
-        }
-      })
-
-      it('should return true', function (ctx) {
-        ctx.call((err, isFullIndex) => {
-          expect(err).to.equal(null)
-          expect(isFullIndex).to.equal(true)
-        })
-      })
-    })
-  })
-})