diff --git a/services/web/app/coffee/Features/References/ReferencesHandler.coffee b/services/web/app/coffee/Features/References/ReferencesHandler.coffee index c033a4999f..7894ab6915 100644 --- a/services/web/app/coffee/Features/References/ReferencesHandler.coffee +++ b/services/web/app/coffee/Features/References/ReferencesHandler.coffee @@ -16,19 +16,31 @@ module.exports = ReferencesHandler = _buildDocUrl: (projectId, docId) -> "#{settings.apis.docstore.url}/project/#{projectId}/doc/#{docId}/raw" + _buildFileUrl: (projectId, fileId) -> + "#{settings.apis.filestore.url}/project/#{projectId}/file/#{fileId}" + + _findBibFileIds: (project) -> + ids = [] + _process = (folder) -> + (folder.fileRefs or []).forEach (file) -> + if file?.name?.match(/^.*\.bib$/) + ids.push(file._id) + (folder.folders or []).forEach (folder) -> + _process(folder) + (project.rootFolder or []).forEach (rootFolder) -> + _process(rootFolder) + return ids + _findBibDocIds: (project) -> ids = [] - _process = (folder) -> (folder.docs or []).forEach (doc) -> if doc?.name?.match(/^.*\.bib$/) ids.push(doc._id) (folder.folders or []).forEach (folder) -> _process(folder) - (project.rootFolder or []).forEach (rootFolder) -> _process(rootFolder) - return ids _isFullIndex: (project, callback = (err, result) ->) -> @@ -43,16 +55,17 @@ module.exports = ReferencesHandler = return callback(err) logger.log {projectId}, "indexing all bib files in project" docIds = ReferencesHandler._findBibDocIds(project) - ReferencesHandler._doIndexOperation(projectId, project, docIds, callback) + fileIds = ReferencesHandler._findBibFileIds(project) + ReferencesHandler._doIndexOperation(projectId, project, docIds, fileIds, callback) index: (projectId, docIds, callback=(err, data)->) -> ProjectGetter.getProject projectId, {rootFolder: true, owner_ref: 1}, (err, project) -> if err logger.err {err, projectId}, "error finding project" return callback(err) - ReferencesHandler._doIndexOperation(projectId, project, docIds, callback) + ReferencesHandler._doIndexOperation(projectId, project, docIds, [], callback) - _doIndexOperation: (projectId, project, docIds, callback) -> + _doIndexOperation: (projectId, project, docIds, fileIds, callback) -> ReferencesHandler._isFullIndex project, (err, isFullIndex) -> if err logger.err {err, projectId}, "error checking whether to do full index" @@ -67,11 +80,14 @@ module.exports = ReferencesHandler = return callback(err) bibDocUrls = docIds.map (docId) -> ReferencesHandler._buildDocUrl projectId, docId + bibFileUrls = fileIds.map (fileId) -> + ReferencesHandler._buildFileUrl projectId, fileId + allUrls = bibDocUrls.concat(bibFileUrls) logger.log {projectId, isFullIndex, docIds, bibDocUrls}, "sending request to references service" request.post { url: "#{settings.apis.references.url}/project/#{projectId}/index" json: - docUrls: bibDocUrls + docUrls: allUrls fullIndex: isFullIndex }, (err, res, data) -> if err diff --git a/services/web/public/coffee/ide/pdf/controllers/PdfController.coffee b/services/web/public/coffee/ide/pdf/controllers/PdfController.coffee index 4c00ced0b8..ac6303e510 100644 --- a/services/web/public/coffee/ide/pdf/controllers/PdfController.coffee +++ b/services/web/public/coffee/ide/pdf/controllers/PdfController.coffee @@ -97,7 +97,7 @@ define [ $scope.pdf.qs = if qs_args.length then "?" + qs_args.join("&") else "" $scope.pdf.url += $scope.pdf.qs - fetchLogs(fileByPath['output.log']) + fetchLogs(fileByPath['output.log'], fileByPath['output.blg']) IGNORE_FILES = ["output.fls", "output.fdb_latexmk"] $scope.pdf.outputFiles = [] @@ -116,62 +116,77 @@ define [ file.url = file.url + "?clsiserverid=#{response.clsiServerId}" $scope.pdf.outputFiles.push file - fetchLogs = (outputFile) -> - opts = - method:"GET" - params: - build:outputFile.build - clsiserverid:ide.clsiServerId - if outputFile?.build? - opts.url = "/project/#{$scope.project_id}/build/#{outputFile.build}/output/output.log" - else - opts.url = "/project/#{$scope.project_id}/output/output.log" - $http opts - .success (log) -> - #console.log ">>", log - $scope.pdf.rawLog = log - logEntries = LogParser.parse(log, ignoreDuplicates: true) - #console.log ">>", logEntries - $scope.pdf.logEntries = logEntries - $scope.pdf.logEntries.all = logEntries.errors.concat(logEntries.warnings).concat(logEntries.typesetting) - # # # # - proceed = () -> - $scope.pdf.logEntryAnnotations = {} - for entry in logEntries.all - if entry.file? - entry.file = normalizeFilePath(entry.file) - entity = ide.fileTreeManager.findEntityByPath(entry.file) - if entity? - $scope.pdf.logEntryAnnotations[entity.id] ||= [] - $scope.pdf.logEntryAnnotations[entity.id].push { - row: entry.line - 1 - type: if entry.level == "error" then "error" else "warning" - text: entry.message - } - # Get the biber log and parse it - if outputFile?.build? - opts.url = "/project/#{$scope.project_id}/build/#{outputFile.build}/output/output.blg" - else - opts.url = "/project/#{$scope.project_id}/output/output.blg" - $http opts - .success (log) -> - window._s = $scope - biberLogEntries = BibLogParser.parse(log, {}) - if $scope.pdf.logEntries - entries = $scope.pdf.logEntries - all = biberLogEntries.errors.concat(biberLogEntries.warnings) - entries.all = entries.all.concat(all) - entries.errors = entries.errors.concat(biberLogEntries.errors) - entries.warnings = entries.warnings.concat(biberLogEntries.warnings) - proceed() - .error (e) -> - # it's not an error for the output.blg file to not be present - proceed() - # # # # - .error () -> - $scope.pdf.logEntries = [] - $scope.pdf.rawLog = "" + fetchLogs = (logFile, blgFile) -> + + getFile = (name, file) -> + opts = + method:"GET" + params: + build:file.build + clsiserverid:ide.clsiServerId + if file?.build? + opts.url = "/project/#{$scope.project_id}/build/#{file.build}/output/#{name}" + else + opts.url = "/project/#{$scope.project_id}/output/#{name}" + return $http(opts) + + # accumulate the log entries + logEntries = + all: [] + errors: [] + warnings: [] + + accumulateResults = (newEntries) -> + for key in ['all', 'errors', 'warnings'] + logEntries[key] = logEntries[key].concat newEntries[key] + + # use the parsers for each file type + processLog = (log) -> + $scope.pdf.rawLog = log + {errors, warnings, typesetting} = LogParser.parse(log, ignoreDuplicates: true) + all = [].concat errors, warnings, typesetting + accumulateResults {all, errors, warnings} + + processBiber = (log) -> + {errors, warnings} = BibLogParser.parse(log, {}) + all = [].concat errors, warnings + accumulateResults {all, errors, warnings} + + # output the results + handleError = () -> + $scope.pdf.logEntries = [] + $scope.pdf.rawLog = "" + + annotateFiles = () -> + $scope.pdf.logEntries = logEntries + $scope.pdf.logEntryAnnotations = {} + for entry in logEntries.all + if entry.file? + entry.file = normalizeFilePath(entry.file) + entity = ide.fileTreeManager.findEntityByPath(entry.file) + if entity? + $scope.pdf.logEntryAnnotations[entity.id] ||= [] + $scope.pdf.logEntryAnnotations[entity.id].push { + row: entry.line - 1 + type: if entry.level == "error" then "error" else "warning" + text: entry.message + } + + # retrieve the logfile and process it + response = getFile('output.log', logFile) + .success processLog + .error handleError + + if blgFile? # retrieve the blg file if present + response.success () -> + getFile('output.blg', blgFile) + # ignore errors in biber file + .success processBiber + # display the combined result + .then annotateFiles + else # otherwise just display the result + response.success annotateFiles getRootDocOverride_id = () -> doc = ide.editorManager.getCurrentDocValue() diff --git a/services/web/public/coffee/ide/references/ReferencesManager.coffee b/services/web/public/coffee/ide/references/ReferencesManager.coffee index b0f49f54f8..2f1e95c5b1 100644 --- a/services/web/public/coffee/ide/references/ReferencesManager.coffee +++ b/services/web/public/coffee/ide/references/ReferencesManager.coffee @@ -11,6 +11,9 @@ define [ if entity?.name?.match /.*\.bib$/ @indexReferences([doc.doc_id], true) + @$scope.$on 'references:should-reindex', (e, data) => + @indexAllReferences(true) + # When we join the project: # index all references files # and don't broadcast to all clients diff --git a/services/web/test/UnitTests/coffee/References/ReferencesHandlerTests.coffee b/services/web/test/UnitTests/coffee/References/ReferencesHandlerTests.coffee index 53e064d821..c46e71a542 100644 --- a/services/web/test/UnitTests/coffee/References/ReferencesHandlerTests.coffee +++ b/services/web/test/UnitTests/coffee/References/ReferencesHandlerTests.coffee @@ -21,7 +21,11 @@ describe 'ReferencesHandler', -> {name: 'two.txt', _id: 'bbb'}, ] folders: [ - {docs: [{name: 'three.bib', _id: 'ccc'}], folders: []} + { + docs: [{name: 'three.bib', _id: 'ccc'}], + fileRefs: [{name: 'four.bib', _id: 'ghg'}], + folders: [] + } ] ] @docIds = ['aaa', 'ccc'] @@ -34,6 +38,7 @@ describe 'ReferencesHandler', -> apis: references: {url: 'http://some.url/references'} docstore: {url: 'http://some.url/docstore'} + filestore: {url: 'http://some.url/filestore'} } 'request': @request = { get: sinon.stub() @@ -56,6 +61,7 @@ describe 'ReferencesHandler', -> beforeEach -> sinon.stub(@handler, '_findBibDocIds') + sinon.stub(@handler, '_findBibFileIds') sinon.stub(@handler, '_isFullIndex').callsArgWith(1, null, true) @request.post.callsArgWith(1, null, {statusCode: 200}, @fakeResponseData) @call = (callback) => @@ -198,6 +204,7 @@ describe 'ReferencesHandler', -> beforeEach -> sinon.stub(@handler, '_findBibDocIds').returns(['aaa', 'ccc']) + sinon.stub(@handler, '_findBibFileIds').returns(['fff', 'ggg']) sinon.stub(@handler, '_isFullIndex').callsArgWith(1, null, true) @request.post.callsArgWith(1, null, {statusCode: 200}, @fakeResponseData) @call = (callback) => @@ -209,6 +216,12 @@ describe 'ReferencesHandler', -> @handler._findBibDocIds.calledWith(@fakeProject).should.equal true done() + it 'should call _findBibFileIds', (done) -> + @call (err, data) => + @handler._findBibDocIds.callCount.should.equal 1 + @handler._findBibDocIds.calledWith(@fakeProject).should.equal true + done() + it 'should call DocumentUpdaterHandler.flushDocToMongo', (done) -> @call (err, data) => @DocumentUpdaterHandler.flushDocToMongo.callCount.should.equal 2 @@ -219,7 +232,7 @@ describe 'ReferencesHandler', -> @request.post.callCount.should.equal 1 arg = @request.post.firstCall.args[0] expect(arg.json).to.have.all.keys 'docUrls', 'fullIndex' - expect(arg.json.docUrls.length).to.equal 2 + expect(arg.json.docUrls.length).to.equal 4 expect(arg.json.fullIndex).to.equal true done() @@ -309,6 +322,32 @@ describe 'ReferencesHandler', -> result = @handler._findBibDocIds(@fakeProject) expect(result).to.deep.equal @expectedIds + describe '_findBibFileIds', -> + + beforeEach -> + @fakeProject = + rootFolder: [ + docs: [ + {name: 'one.bib', _id: 'aaa'}, + {name: 'two.txt', _id: 'bbb'}, + ] + fileRefs: [ + {name: 'other.bib', _id: 'ddd'} + ], + folders: [ + { + docs: [{name: 'three.bib', _id: 'ccc'}], + fileRefs: [{name: 'four.bib', _id: 'ghg'}], + folders: [] + } + ] + ] + @expectedIds = ['ddd', 'ghg'] + + it 'should select the correct docIds', -> + result = @handler._findBibFileIds(@fakeProject) + expect(result).to.deep.equal @expectedIds + describe '_isFullIndex', -> beforeEach ->