From 9eec24535873996db00cb2c63fb470ea4035716f Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 9 Oct 2017 14:17:28 +0100 Subject: [PATCH 1/2] allow docupdater to flush docs in background --- .../web/app/coffee/Features/Compile/ClsiManager.coffee | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/services/web/app/coffee/Features/Compile/ClsiManager.coffee b/services/web/app/coffee/Features/Compile/ClsiManager.coffee index 46cb4e35e3..d5f9b2e16c 100755 --- a/services/web/app/coffee/Features/Compile/ClsiManager.coffee +++ b/services/web/app/coffee/Features/Compile/ClsiManager.coffee @@ -146,13 +146,8 @@ module.exports = ClsiManager = logger.log project_id: project_id, projectStateHash: projectStateHash, docs: docUpdaterDocs?, "checked project state" # see if we can send an incremental update to the CLSI if docUpdaterDocs? and (options.syncType isnt "full") and not error? - # Workaround: for now, always flush project to mongo on compile - # until we have automatic periodic flushing on the docupdater - # side, to prevent documents staying in redis too long. - DocumentUpdaterHandler.flushProjectToMongo project_id, (error) -> - return callback(error) if error? - Metrics.inc "compile-from-redis" - ClsiManager._buildRequestFromDocupdater project_id, options, project, projectStateHash, docUpdaterDocs, callback + Metrics.inc "compile-from-redis" + ClsiManager._buildRequestFromDocupdater project_id, options, project, projectStateHash, docUpdaterDocs, callback else Metrics.inc "compile-from-mongo" ClsiManager._buildRequestFromMongo project_id, options, project, projectStateHash, callback From cf25ff058ffa1ca8b0fb31a01758e8a60f2c2a2c Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 11 Oct 2017 15:50:21 +0100 Subject: [PATCH 2/2] use new endpoint docupdater get_and_flush_if_old --- .../Features/DocumentUpdater/DocumentUpdaterHandler.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee b/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee index 466152924e..e806ddf8dd 100644 --- a/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee +++ b/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee @@ -128,9 +128,9 @@ module.exports = DocumentUpdaterHandler = # docs from redis via the docupdater. Otherwise we will need to # fall back to getting them from mongo. timer = new metrics.Timer("get-project-docs") - url = "#{settings.apis.documentupdater.url}/project/#{project_id}/doc?state=#{projectStateHash}" + url = "#{settings.apis.documentupdater.url}/project/#{project_id}/get_and_flush_if_old?state=#{projectStateHash}" logger.log project_id:project_id, "getting project docs from document updater" - request.get url, (error, res, body)-> + request.post url, (error, res, body)-> timer.done() if error? logger.error err:error, url:url, project_id:project_id, "error getting project docs from doc updater"