[history-v1] add migrations record after full binary file migration (#27932)

* [history-v1] add migrations record after full binary file migration

* [server-pro] add hotfix 5.5.5

* [server-ce] test: build hotfix 5.5.5 and use it in tests

GitOrigin-RevId: fb84e5710c59f466a3305de5f32f78e0ac9ce15d

server-ce/hotfix/5.5.5/Dockerfile

@@ -0,0 +1,7 @@
+FROM sharelatex/sharelatex:5.5.4
+
+# ../../bin/import_pr_patch.sh 27932
+# Remove web migrations changes
+# Remove test changes
+COPY *.patch .
+RUN bash -ec 'for p in *.patch; do echo "=== Applying $p ==="; patch -p1 < "$p" && rm $p; done'

server-ce/hotfix/5.5.5/pr_27932.patch

@@ -0,0 +1,85 @@
+diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs
+index 33962c5da7d4..8b25fb8bd603 100644
+--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs
++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs
+@@ -78,11 +78,29 @@ ObjectId.cacheHexString = true
+  * @property {Blob} [blob]
+  */
+
++/**
++ * Start and end for range.
++ * @type {Date}
++ */
++const PUBLIC_LAUNCH_DATE = new Date('2012-01-01T00:00:00Z')
++const DEFAULT_BATCH_RANGE_START_DATE = PUBLIC_LAUNCH_DATE
++const DEFAULT_BATCH_RANGE_END_DATE = new Date()
++
++function usesDefaultBatchRange() {
++  return (
++    BATCH_RANGE_START ===
++      objectIdFromInput(
++        DEFAULT_BATCH_RANGE_START_DATE.toISOString()
++      ).toString() &&
++    BATCH_RANGE_END ===
++      objectIdFromInput(DEFAULT_BATCH_RANGE_END_DATE.toISOString()).toString()
++  )
++}
++
+ /**
+  * @return {{PROJECT_IDS_FROM: string, PROCESS_HASHED_FILES: boolean, LOGGING_IDENTIFIER: string, BATCH_RANGE_START: string, BATCH_RANGE_END: string, PROCESS_NON_DELETED_PROJECTS: boolean, PROCESS_DELETED_PROJECTS: boolean, PROCESS_BLOBS: boolean, DRY_RUN: boolean, OUTPUT_FILE: string, DISPLAY_REPORT: boolean, CONCURRENCY: number, CONCURRENT_BATCHES: number, RETRIES: number, RETRY_DELAY_MS: number, RETRY_FILESTORE_404: boolean, BUFFER_DIR_PREFIX: string, STREAM_HIGH_WATER_MARK: number, LOGGING_INTERVAL: number, SLEEP_BEFORE_EXIT: number }}
+  */
+ function parseArgs() {
+-  const PUBLIC_LAUNCH_DATE = new Date('2012-01-01T00:00:00Z')
+   const DEFAULT_OUTPUT_FILE = `/var/log/overleaf/file-migration-${new Date()
+     .toISOString()
+     .replace(/[:.]/g, '_')}.log`
+@@ -1475,6 +1493,49 @@ try {
+     )
+     code++
+   }
++  console.warn('-'.repeat(79))
++  if (code === 0) {
++    const allProcessed =
++      !DRY_RUN &&
++      PROCESS_NON_DELETED_PROJECTS &&
++      PROCESS_DELETED_PROJECTS &&
++      PROCESS_HASHED_FILES &&
++      !PROJECT_IDS_FROM &&
++      usesDefaultBatchRange()
++    if (allProcessed) {
++      await db
++        .collection('migrations')
++        .updateOne(
++          { name: '20250519101128_binary_files_migration' },
++          { $set: { migratedAt: new Date(DEFAULT_BATCH_RANGE_END_DATE) } },
++          { upsert: true }
++        )
++      console.warn('The binary files migration succeeded.')
++      console.warn(
++        'You can now proceed to OVERLEAF_FILESTORE_MIGRATION_LEVEL=2.'
++      )
++    } else {
++      console.warn(
++        'The binary files migration succeeded on a subset of files (at least one of --dry-run, --skip-hashed-files, --from-file, --BATCH_RANGE_START or --BATCH_RANGE_END is set and --all is not set).'
++      )
++      console.warn(
++        'Once you are done with all the partial runs, you need to run the migration again on all projects/files to ensure that all files are migrated into the full project history system.'
++      )
++      console.warn('The full run will unlock the upgrade to Server Pro 6.0.')
++    }
++  } else {
++    console.warn('The binary files migration failed, see above.')
++    console.warn(
++      'Please review the failures and check the docs on remediating the failures.'
++    )
++    console.warn(
++      'Docs: https://docs.overleaf.com/on-premises/release-notes/release-notes-5.x.x/binary-files-migration#troubleshooting'
++    )
++    console.warn(
++      'In case there is not solution available, please reach out to support as detailed in the docs.'
++    )
++  }
++  console.warn('-'.repeat(79))
+   await setTimeout(SLEEP_BEFORE_EXIT)
+   process.exit(code)
+ } catch (err) {

server-ce/test/Makefile

@@ -82,6 +82,12 @@ prefetch_old_5_0:
 	docker pull $(IMAGE_TAG_PRO:main=5.0.1-RC1)
 	docker pull $(IMAGE_TAG_PRO:main=5.0)
 
+prefetch_custom: build_hotfix_5_5_5
+build_hotfix_5_5_5:
+	docker pull $(IMAGE_TAG_PRO:main=5.5.4)
+	docker tag $(IMAGE_TAG_PRO:main=5.5.4) quay.io/sharelatex/sharelatex-pro:5.5.4
+	cd ../../server-pro/hotfix/5.5.5 && docker build -t $(IMAGE_TAG_PRO:main=5.5.5) .
+
 # Google Cloud Build runs on a very ancient Docker version that does not support the subdir flag.
 # Use services -> mailtrap -> build -> context = https://github.com/dbck/docker-mailtrap.git#v1.5.0:build in docker-compose.yml eventually.
 prefetch_default_compose_build: build_mailtrap

server-ce/test/filestore-migration.spec.ts

@@ -232,11 +232,13 @@ describe('filestore migration', function () {
     // ------------------------------------------
     // Server Pro 5.x + mongo upgrade 6 -> 7 -> 8
     startWith({
+      version: '5.5.5',
       pro: true,
       withDataDir: true,
       mongoVersion: '6.0',
     })
     startWith({
+      version: '5.5.5',
       pro: true,
       withDataDir: true,
       mongoVersion: '7.0',
@@ -245,6 +247,7 @@ describe('filestore migration', function () {
       await setMongoFeatureCompatibilityVersion('7.0')
     })
     startWith({
+      version: '5.5.5',
       pro: true,
       withDataDir: true,
       // implicit mongo upgrade to 8.0
@@ -254,7 +257,7 @@ describe('filestore migration', function () {
     })
   } else {
     // 5.x
-    startWith({ pro: true, withDataDir: true })
+    startWith({ version: '5.5.5', pro: true, withDataDir: true })
     defaultImage = 'frog.jpg'
     ensureUserExists({ email })
     before(function () {
@@ -337,12 +340,13 @@ describe('filestore migration', function () {
   }
 
   describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL not set', function () {
-    startWith({ pro: true, withDataDir: true, vars: {} })
+    startWith({ version: '5.5.5', pro: true, withDataDir: true, vars: {} })
     checkFilesAreAccessible()
   })
 
   describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=0', function () {
     startWith({
+      version: '5.5.5',
       pro: true,
       withDataDir: true,
       vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '0' },
@@ -351,6 +355,7 @@ describe('filestore migration', function () {
 
     describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=1', function () {
       startWith({
+        version: '5.5.5',
         pro: true,
         withDataDir: true,
         vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '1' },
@@ -359,6 +364,7 @@ describe('filestore migration', function () {
 
       describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=2', function () {
         startWith({
+          version: '5.5.5',
           pro: true,
           withDataDir: true,
           vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '1' },
@@ -371,6 +377,7 @@ describe('filestore migration', function () {
           })
         })
         startWith({
+          version: '5.5.5',
           pro: true,
           withDataDir: true,
           vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '2' },
@@ -382,6 +389,15 @@ describe('filestore migration', function () {
             await purgeFilestoreData()
           })
           checkFilesAreAccessible()
+
+          describe('latest', function () {
+            startWith({
+              pro: true,
+              withDataDir: true,
+              vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '2' },
+            })
+            checkFilesAreAccessible()
+          })
         })
       })
     })

server-ce/test/upgrading.spec.ts

@@ -155,11 +155,31 @@ describe('Upgrading', function () {
     },
     newProjectButtonMatcher: /create first project/i,
   }
+  const optionsBinaryFilesMigration = {
+    version: '5.5.5',
+    hook() {
+      before(async function () {
+        await runScript({
+          cwd: 'services/history-v1',
+          script: 'storage/scripts/back_fill_file_hash.mjs',
+          args: ['--all'],
+        })
+      })
+    },
+  }
   describe('from 4.2 to latest', () => {
-    testUpgrade([optionsFourDotTwo, { version: 'latest' }])
+    testUpgrade([
+      optionsFourDotTwo,
+      optionsBinaryFilesMigration,
+      { version: 'latest' },
+    ])
   })
   describe('from 5.0 to latest', () => {
-    testUpgrade([{ version: '5.0' }, { version: 'latest' }])
+    testUpgrade([
+      { version: '5.0' },
+      optionsBinaryFilesMigration,
+      { version: 'latest' },
+    ])
   })
   describe('doc version recovery', () => {
     testUpgrade([
@@ -190,6 +210,7 @@ describe('Upgrading', function () {
           })
         },
       },
+      optionsBinaryFilesMigration,
       {
         version: 'latest',
         hook() {