From cc818695efe8174d84627ebf3f2d3ce340921400 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Wed, 27 Jan 2016 16:00:53 +0000 Subject: [PATCH] Split References.index into index and indexAll. --- .../ReferencesSearchController.coffee | 26 +++-- .../ReferencesSearchHandler.coffee | 77 ++++++++------- services/web/app/coffee/router.coffee | 1 + .../ReferencesSearchManager.coffee | 17 +++- .../ReferencesSearchControllerTests.coffee | 96 +++++++++++++++---- .../ReferencesSearchHandlerTests.coffee | 76 +++++++++------ 6 files changed, 202 insertions(+), 91 deletions(-) diff --git a/services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchController.coffee b/services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchController.coffee index ae7be5f73e..96f06228c8 100644 --- a/services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchController.coffee +++ b/services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchController.coffee @@ -5,19 +5,33 @@ EditorRealTimeController = require("../Editor/EditorRealTimeController") module.exports = ReferencesSearchController = + index: (req, res) -> projectId = req.params.Project_id shouldBroadcast = req.body.shouldBroadcast docIds = req.body.docIds - if (!docIds or (!(docIds instanceof Array) and (docIds != 'ALL'))) + if (!docIds or !(docIds instanceof Array)) logger.err {projectId, docIds}, "docIds is not valid, should be either Array or String 'ALL'" return res.send 400 logger.log {projectId, docIds: docIds}, "index references for project" ReferencesSearchHandler.index projectId, docIds, (err, data) -> if err - logger.err {err, projectId}, "error indexing references" + logger.err {err, projectId}, "error indexing all references" return res.send 500 - if shouldBroadcast - logger.log {projectId}, "emitting new references keys to connected clients" - EditorRealTimeController.emitToRoom projectId, 'references:keys:updated', data.keys - return res.json data + ReferencesSearchController._handleIndexResponse(req, res, projectId, shouldBroadcast, data) + + indexAll: (req, res) -> + projectId = req.params.Project_id + shouldBroadcast = req.body.shouldBroadcast + logger.log {projectId}, "index all references for project" + ReferencesSearchHandler.indexAll projectId, (err, data) -> + if err + logger.err {err, projectId}, "error indexing all references" + return res.send 500 + ReferencesSearchController._handleIndexResponse(req, res, projectId, shouldBroadcast, data) + + _handleIndexResponse: (req, res, projectId, shouldBroadcast, data) -> + if shouldBroadcast + logger.log {projectId}, "emitting new references keys to connected clients" + EditorRealTimeController.emitToRoom projectId, 'references:keys:updated', data.keys + return res.json data diff --git a/services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchHandler.coffee b/services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchHandler.coffee index 7722c9b241..619d325389 100644 --- a/services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchHandler.coffee +++ b/services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchHandler.coffee @@ -34,45 +34,52 @@ module.exports = ReferencesSearchHandler = owner = project.owner_ref callback(null, owner.features.references == true) - # projectId: String, docIds: List[String]|Null + indexAll: (projectId, callback=(err, data)->) -> + Project.findPopulatedById projectId, (err, project) -> + if err + logger.err {err, projectId}, "error finding project" + return callback(err) + logger.log {projectId}, "indexing all bib files in project" + docIds = ReferencesSearchHandler._findBibDocIds(project) + ReferencesSearchHandler._doIndexOperation(projectId, project, docIds, callback) + index: (projectId, docIds, callback=(err, data)->) -> Project.findPopulatedById projectId, (err, project) -> if err logger.err {err, projectId}, "error finding project" return callback(err) - if docIds == "ALL" - logger.log {projectId}, "indexing all bib files in project" - docIds = ReferencesSearchHandler._findBibDocIds(project) - ReferencesSearchHandler._isFullIndex project, (err, isFullIndex) -> - if err - logger.err {err, projectId}, "error checking whether to do full index" - return callback(err) - # TODO: flush documents to mongo - logger.log {projectId, docIds}, 'flushing docs to mongo before calling references service' - Async.series( - docIds.map((docId) -> (cb) -> DocumentUpdaterHandler.flushDocToMongo(projectId, docId, cb)), - (err) -> - # continue + ReferencesSearchHandler._doIndexOperation(projectId, project, docIds, callback) + + _doIndexOperation: (projectId, project, docIds, callback) -> + ReferencesSearchHandler._isFullIndex project, (err, isFullIndex) -> + if err + logger.err {err, projectId}, "error checking whether to do full index" + return callback(err) + logger.log {projectId, docIds}, 'flushing docs to mongo before calling references service' + Async.series( + docIds.map((docId) -> (cb) -> DocumentUpdaterHandler.flushDocToMongo(projectId, docId, cb)), + (err) -> + # continue + if err + logger.err {err, projectId, docIds}, "error flushing docs to mongo" + return callback(err) + bibDocUrls = docIds.map (docId) -> + ReferencesSearchHandler._buildDocUrl projectId, docId + logger.log {projectId, isFullIndex, docIds, bibDocUrls}, "sending request to references service" + request.post { + url: "#{settings.apis.references.url}/project/#{projectId}/index" + json: + docUrls: bibDocUrls + fullIndex: isFullIndex + }, (err, res, data) -> if err - logger.err {err, projectId, docIds}, "error flushing docs to mongo" + logger.err {err, projectId}, "error communicating with references api" return callback(err) - bibDocUrls = docIds.map (docId) -> - ReferencesSearchHandler._buildDocUrl projectId, docId - logger.log {projectId, isFullIndex, docIds, bibDocUrls}, "sending request to references service" - request.post { - url: "#{settings.apis.references.url}/project/#{projectId}/index" - json: - docUrls: bibDocUrls - fullIndex: isFullIndex - }, (err, res, data) -> - if err - logger.err {err, projectId}, "error communicating with references api" - return callback(err) - if 200 <= res.statusCode < 300 - logger.log {projectId}, "got keys from references api" - return callback(null, data) - else - err = new Error("references api responded with non-success code: #{res.statusCode}") - logger.log {err, projectId}, "error updating references" - return callback(err) - ) + if 200 <= res.statusCode < 300 + logger.log {projectId}, "got keys from references api" + return callback(null, data) + else + err = new Error("references api responded with non-success code: #{res.statusCode}") + logger.log {err, projectId}, "error updating references" + return callback(err) + ) diff --git a/services/web/app/coffee/router.coffee b/services/web/app/coffee/router.coffee index 0d55794d03..f84d2b0564 100644 --- a/services/web/app/coffee/router.coffee +++ b/services/web/app/coffee/router.coffee @@ -172,6 +172,7 @@ module.exports = class Router webRouter.get /learn(\/.*)?/, WikiController.getPage webRouter.post "/project/:Project_id/references/index", SecurityManager.requestCanAccessProject, ReferencesSearchController.index + webRouter.post "/project/:Project_id/references/indexAll", SecurityManager.requestCanAccessProject, ReferencesSearchController.indexAll #Admin Stuff webRouter.get '/admin', SecurityManager.requestIsAdmin, AdminController.index diff --git a/services/web/public/coffee/ide/references-search/ReferencesSearchManager.coffee b/services/web/public/coffee/ide/references-search/ReferencesSearchManager.coffee index d553fa84ab..fbb957707a 100644 --- a/services/web/public/coffee/ide/references-search/ReferencesSearchManager.coffee +++ b/services/web/public/coffee/ide/references-search/ReferencesSearchManager.coffee @@ -15,14 +15,14 @@ define [ # index all references files # and don't broadcast to all clients @$scope.$on 'project:joined', (e) => - @indexReferences("ALL", false) + @indexAllReferences(false) setTimeout( (self) -> self.ide.socket.on 'references:keys:updated', (keys) -> # console.log '>> got keys from socket' self._storeReferencesKeys(keys) - , 100 + , 1000 , this ) @@ -35,7 +35,6 @@ define [ @$scope.$root._references.keys = _.union(oldKeys, newKeys) console.log "end>> #{@$scope.$root._references.keys.length}" - # docIds: List[String]|String('ALL'), shouldBroadcast: Bool indexReferences: (docIds, shouldBroadcast) -> opts = docIds: docIds @@ -48,3 +47,15 @@ define [ # console.log ">> got keys ", data @_storeReferencesKeys(data.keys) ) + + indexAllReferences: (shouldBroadcast) -> + opts = + shouldBroadcast: shouldBroadcast + _csrf: window.csrfToken + $.post( + "/project/#{@$scope.project_id}/references/indexAll", + opts, + (data) => + # console.log ">> got keys ", data + @_storeReferencesKeys(data.keys) + ) diff --git a/services/web/test/UnitTests/coffee/ReferencesSearch/ReferencesSearchControllerTests.coffee b/services/web/test/UnitTests/coffee/ReferencesSearch/ReferencesSearchControllerTests.coffee index dd93ef8626..9b12648520 100644 --- a/services/web/test/UnitTests/coffee/ReferencesSearch/ReferencesSearchControllerTests.coffee +++ b/services/web/test/UnitTests/coffee/ReferencesSearch/ReferencesSearchControllerTests.coffee @@ -20,6 +20,7 @@ describe "ReferencesSearchController", -> }, './ReferencesSearchHandler': @ReferencesSearchHandler = { index: sinon.stub() + indexAll: sinon.stub() }, '../Editor/EditorRealTimeController': @EditorRealTimeController = { emitToRoom: sinon.stub() @@ -36,6 +37,83 @@ describe "ReferencesSearchController", -> projectId: @projectId, keys: ['one', 'two', 'three'] + describe 'indexAll', -> + + beforeEach -> + @req.body = {shouldBroadcast: false} + @ReferencesSearchHandler.indexAll.callsArgWith(1, null, @fakeResponseData) + @call = (callback) => + @controller.indexAll @req, @res + callback() + + it 'should not produce an error', (done) -> + @call () => + @res.send.callCount.should.equal 0 + @res.send.calledWith(500).should.equal false + @res.send.calledWith(400).should.equal false + done() + + it 'should return data', (done) -> + @call () => + @res.json.callCount.should.equal 1 + @res.json.calledWith(@fakeResponseData).should.equal true + done() + + it 'should call ReferencesSearchHandler.indexAll', (done) -> + @call () => + @ReferencesSearchHandler.indexAll.callCount.should.equal 1 + @ReferencesSearchHandler.indexAll.calledWith(@projectId).should.equal true + done() + + describe 'when shouldBroadcast is true', -> + + beforeEach -> + @ReferencesSearchHandler.index.callsArgWith(2, null, @fakeResponseData) + @req.body.shouldBroadcast = true + + it 'should call EditorRealTimeController.emitToRoom', (done) -> + @call () => + @EditorRealTimeController.emitToRoom.callCount.should.equal 1 + done() + + it 'should not produce an error', (done) -> + @call () => + @res.send.callCount.should.equal 0 + @res.send.calledWith(500).should.equal false + @res.send.calledWith(400).should.equal false + done() + + it 'should still return data', (done) -> + @call () => + @res.json.callCount.should.equal 1 + @res.json.calledWith(@fakeResponseData).should.equal true + done() + + describe 'when shouldBroadcast is false', -> + + beforeEach -> + @ReferencesSearchHandler.index.callsArgWith(2, null, @fakeResponseData) + @req.body.shouldBroadcast = false + + it 'should not call EditorRealTimeController.emitToRoom', (done) -> + @call () => + @EditorRealTimeController.emitToRoom.callCount.should.equal 0 + done() + + it 'should not produce an error', (done) -> + @call () => + @res.send.callCount.should.equal 0 + @res.send.calledWith(500).should.equal false + @res.send.calledWith(400).should.equal false + done() + + it 'should still return data', (done) -> + @call () => + @res.json.callCount.should.equal 1 + @res.json.calledWith(@fakeResponseData).should.equal true + done() + + describe 'index', -> describe 'with docIds as an array and shouldBroadcast as false', -> @@ -70,24 +148,6 @@ describe "ReferencesSearchController", -> @EditorRealTimeController.emitToRoom.callCount.should.equal 0 done() - describe 'with docIds set to ALL', -> - - beforeEach -> - @req.body.docIds = 'ALL' - - it 'should still pass the "ALL" value to handler', (done) -> - @call () => - @ReferencesSearchHandler.index.callCount.should.equal 1 - @ReferencesSearchHandler.index.calledWith(@projectId, 'ALL').should.equal true - done() - - it 'should not produce an error', (done) -> - @call () => - @res.send.callCount.should.equal 0 - @res.send.calledWith(500).should.equal false - @res.send.calledWith(400).should.equal false - done() - describe 'when ReferencesSearchHandler.index produces an error', -> beforeEach -> diff --git a/services/web/test/UnitTests/coffee/ReferencesSearch/ReferencesSearchHandlerTests.coffee b/services/web/test/UnitTests/coffee/ReferencesSearch/ReferencesSearchHandlerTests.coffee index faa6e49a62..0fac15b078 100644 --- a/services/web/test/UnitTests/coffee/ReferencesSearch/ReferencesSearchHandlerTests.coffee +++ b/services/web/test/UnitTests/coffee/ReferencesSearch/ReferencesSearchHandlerTests.coffee @@ -76,6 +76,11 @@ describe 'ReferencesSearchHandler', -> @Project.findPopulatedById.calledWith(@projectId).should.equal true done() + it 'should not call _findBibDocIds', (done) -> + @call (err, data) => + @handler._findBibDocIds.callCount.should.equal 0 + done() + it 'should call DocumentUpdaterHandler.flushDocToMongo', (done) -> @call (err, data) => @DocumentUpdaterHandler.flushDocToMongo.callCount.should.equal 2 @@ -104,35 +109,6 @@ describe 'ReferencesSearchHandler', -> expect(data).to.equal @fakeResponseData done() - describe 'with docIds as "ALL"', -> - - beforeEach -> - @docIds = 'ALL' - @handler._findBibDocIds.returns(['aaa', 'ccc']) - - it 'should call _findBibDocIds', (done) -> - @call (err, data) => - @handler._findBibDocIds.callCount.should.equal 1 - @handler._findBibDocIds.calledWith(@fakeProject).should.equal true - done() - - it 'should call DocumentUpdaterHandler.flushDocToMongo', (done) -> - @call (err, data) => - @DocumentUpdaterHandler.flushDocToMongo.callCount.should.equal 2 - done() - - it 'should not produce an error', (done) -> - @call (err, data) => - expect(err).to.equal null - done() - - it 'should return data', (done) -> - @call (err, data) => - expect(data).to.not.equal null - expect(data).to.not.equal undefined - expect(data).to.equal @fakeResponseData - done() - describe 'when Project.findPopulatedById produces an error', -> beforeEach -> @@ -217,6 +193,48 @@ describe 'ReferencesSearchHandler', -> expect(data).to.equal undefined done() + describe 'indexAll', -> + + beforeEach -> + sinon.stub(@handler, '_findBibDocIds').returns(['aaa', 'ccc']) + sinon.stub(@handler, '_isFullIndex').callsArgWith(1, null, true) + @request.post.callsArgWith(1, null, {statusCode: 200}, @fakeResponseData) + @call = (callback) => + @handler.indexAll @projectId, callback + + it 'should call _findBibDocIds', (done) -> + @call (err, data) => + @handler._findBibDocIds.callCount.should.equal 1 + @handler._findBibDocIds.calledWith(@fakeProject).should.equal true + done() + + it 'should call DocumentUpdaterHandler.flushDocToMongo', (done) -> + @call (err, data) => + @DocumentUpdaterHandler.flushDocToMongo.callCount.should.equal 2 + done() + + it 'should make a request to references service', (done) -> + @call (err, data) => + @request.post.callCount.should.equal 1 + arg = @request.post.firstCall.args[0] + expect(arg.json).to.have.all.keys 'docUrls', 'fullIndex' + expect(arg.json.docUrls.length).to.equal 2 + expect(arg.json.fullIndex).to.equal true + done() + + it 'should not produce an error', (done) -> + @call (err, data) => + expect(err).to.equal null + done() + + it 'should return data', (done) -> + @call (err, data) => + expect(data).to.not.equal null + expect(data).to.not.equal undefined + expect(data).to.equal @fakeResponseData + done() + + describe '_findBibDocIds', -> beforeEach ->