From 53d207431502c04f168109db82803aaab90ae4c6 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 26 Sep 2022 09:39:26 +0100 Subject: [PATCH] Merge pull request #9723 from overleaf/bg-log-large-docs-in-docstore-archive warn about oversized docs in docstore archiving GitOrigin-RevId: 296a49b37b7f6733ef33f682ae40a8d3d478d9e8 --- services/docstore/app/js/DocArchiveManager.js | 16 ++++++++++++++++ services/docstore/package.json | 1 + 2 files changed, 17 insertions(+) diff --git a/services/docstore/app/js/DocArchiveManager.js b/services/docstore/app/js/DocArchiveManager.js index 9fc74c15ff..7d6c3abec1 100644 --- a/services/docstore/app/js/DocArchiveManager.js +++ b/services/docstore/app/js/DocArchiveManager.js @@ -9,6 +9,9 @@ const RangeManager = require('./RangeManager') const PersistorManager = require('./PersistorManager') const pMap = require('p-map') +const Bson = require('bson') +const BSON = new Bson() + const PARALLEL_JOBS = Settings.parallelArchiveJobs const UN_ARCHIVE_BATCH_SIZE = Settings.unArchiveBatchSize @@ -55,6 +58,19 @@ async function archiveDoc(projectId, docId) { throw new Error('doc has no lines') } + // warn about any oversized docs already in mongo + const linesSize = BSON.calculateObjectSize(doc.lines || {}) + const rangesSize = BSON.calculateObjectSize(doc.ranges || {}) + if ( + linesSize > Settings.max_doc_length || + rangesSize > Settings.max_doc_length + ) { + logger.warn( + { project_id: projectId, doc_id: doc._id, linesSize, rangesSize }, + 'large doc found when archiving project' + ) + } + const json = JSON.stringify({ lines: doc.lines, ranges: doc.ranges, diff --git a/services/docstore/package.json b/services/docstore/package.json index 0320deec1e..ed3a6fabca 100644 --- a/services/docstore/package.json +++ b/services/docstore/package.json @@ -23,6 +23,7 @@ "@overleaf/settings": "^3.0.0", "async": "^3.2.2", "body-parser": "^1.19.0", + "bson": "^1.1.4", "bunyan": "^1.8.15", "celebrate": "^13.0.4", "express": "^4.17.1",