diff --git a/services/project-history/app/js/SyncManager.js b/services/project-history/app/js/SyncManager.js index c8d29279c3..1443d6436a 100644 --- a/services/project-history/app/js/SyncManager.js +++ b/services/project-history/app/js/SyncManager.js @@ -222,6 +222,17 @@ async function setResyncState(projectId, syncState) { update, { upsert: true } ) + + if (!syncState.isSyncOngoing()) { + await db.projects.updateOne( + { _id: new ObjectId(projectId) }, + { + $max: { + 'overleaf.history.lastResyncedAt': new Date(), + }, + } + ) + } } async function clearResyncState(projectId) { diff --git a/services/project-history/test/acceptance/js/SyncTests.js b/services/project-history/test/acceptance/js/SyncTests.js index 17314c77d4..108f988b2a 100644 --- a/services/project-history/test/acceptance/js/SyncTests.js +++ b/services/project-history/test/acceptance/js/SyncTests.js @@ -1,7 +1,6 @@ import nock from 'nock' import { expect } from 'chai' import assert from 'node:assert' -import mongodb from 'mongodb-legacy' import logger from '@overleaf/logger' import Settings from '@overleaf/settings' import { @@ -17,7 +16,7 @@ import sinon from 'sinon' import { getFailure } from './helpers/ProjectHistoryClient.js' import { fetchNothing, RequestFailedError } from '@overleaf/fetch-utils' import { _getBlobHashFromString } from '../../../app/js/HashManager.js' -const { ObjectId } = mongodb +import { db, ObjectId } from '../../../app/js/mongodb.js' const EMPTY_FILE_HASH = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391' @@ -45,6 +44,8 @@ describe('Syncing with web and doc-updater', function () { this.doc_id = new ObjectId().toString() this.file_id = new ObjectId().toString() + await db.projects.insertOne({ _id: new ObjectId(this.project_id) }) + MockHistoryStore().post('/api/projects').reply(200, { projectId: historyId, }) @@ -954,6 +955,7 @@ describe('Syncing with web and doc-updater', function () { }) it('should send test updates to the history store', async function () { + const beforeResync = new Date() const addFile = MockHistoryStore() .post(`/api/projects/${historyId}/legacy_changes`, body => { expect(body).to.deep.equal([ @@ -1008,6 +1010,14 @@ describe('Syncing with web and doc-updater', function () { addFile.isDone(), `/api/projects/${historyId}/changes should have been called` ) + + const project = await db.projects.findOne({ + _id: new ObjectId(this.project_id), + }) + assert( + project.overleaf.history.lastResyncedAt > beforeResync, + 'lastResyncedAt should have been updated when resync finished' + ) }) it('should strip non-BMP characters in updates before sending to the history store', async function () { diff --git a/services/project-history/test/unit/js/SyncManager/SyncManagerTests.js b/services/project-history/test/unit/js/SyncManager/SyncManagerTests.js index 66a9a62542..a89325fe03 100644 --- a/services/project-history/test/unit/js/SyncManager/SyncManagerTests.js +++ b/services/project-history/test/unit/js/SyncManager/SyncManagerTests.js @@ -73,6 +73,9 @@ describe('SyncManager', function () { findOne: sinon.stub().resolves(this.syncState), updateOne: sinon.stub().resolves(), }, + projects: { + updateOne: sinon.stub().resolves(), + }, } this.extendLock = sinon.stub().resolves() diff --git a/services/web/app/src/models/Project.mjs b/services/web/app/src/models/Project.mjs index 7692a1dd2d..1d42668687 100644 --- a/services/web/app/src/models/Project.mjs +++ b/services/web/app/src/models/Project.mjs @@ -88,6 +88,7 @@ export const ProjectSchema = new Schema( zipFileArchivedInProject: { type: Boolean }, rangesSupportEnabled: { type: Boolean }, otMigrationStage: { type: Number }, + lastResyncedAt: { type: Date }, }, isDebugCopyOf: { type: ObjectId }, }, diff --git a/services/web/scripts/history/resync_projects.mjs b/services/web/scripts/history/resync_projects.mjs index 36e896d152..c9b8030792 100644 --- a/services/web/scripts/history/resync_projects.mjs +++ b/services/web/scripts/history/resync_projects.mjs @@ -24,6 +24,7 @@ Options: --max-id Migrate projects to this id --last-updated-after Migrate projects last updated after this date --last-updated-before Migrate projects last updated before this date + --skip-resynced-after Skip projects that have already been resynced after this date --commit Actually perform the resync, instead of just checking which projects would be resynced --skip-metadata-checks Skip doing Mongo/Redis-level only checks to determine if the projects needs resyncing (has ranges or linked file data) --concurrency How many jobs to run in parallel @@ -32,7 +33,7 @@ Options: /** * - * @returns {{ projectIds?: string[]; minId?: string; maxId?: string; concurrency: number; commit: boolean; skipMetadataChecks: boolean; lastUpdatedAfter?: string; lastUpdatedBefore?: string; }} + * @returns {{ projectIds?: string[]; minId?: string; maxId?: string; concurrency: number; commit: boolean; skipMetadataChecks: boolean; lastUpdatedAfter?: string; lastUpdatedBefore?: string; skipResyncedAfter?: string; }} */ function parseArgs() { const args = minimist(process.argv.slice(2), { @@ -43,6 +44,7 @@ function parseArgs() { 'max-id', 'last-updated-after', 'last-updated-before', + 'skip-resynced-after', ], }) @@ -59,13 +61,15 @@ function parseArgs() { const concurrency = parseInt(args.concurrency, 10) || 1 const commit = args.commit const skipMetadataChecks = args['skip-metadata-checks'] + const skipResyncedAfter = args['skip-resynced-after'] if ( projectIds == null && minId == null && maxId == null && lastUpdatedAfter == null && - lastUpdatedBefore == null + lastUpdatedBefore == null && + skipResyncedAfter == null ) { console.error('Please specify at least one filter\n') usage() @@ -81,6 +85,7 @@ function parseArgs() { skipMetadataChecks, lastUpdatedAfter, lastUpdatedBefore, + skipResyncedAfter, } } @@ -94,6 +99,7 @@ async function main() { skipMetadataChecks, lastUpdatedAfter, lastUpdatedBefore, + skipResyncedAfter, } = parseArgs() // skip projects that don't have full project history @@ -115,6 +121,13 @@ async function main() { if (lastUpdatedBefore) { clauses.push({ lastUpdated: { $lt: new Date(lastUpdatedBefore) } }) } + if (skipResyncedAfter) { + clauses.push({ + 'overleaf.history.lastResyncedAt': { + $not: { $gt: new Date(skipResyncedAfter) }, + }, + }) + } const filter = { $and: clauses } const projects = db.projects