[server-pro] extend hotfix 5.5.5: avoid creating user_files directory (#28738)

* [WIP] restore 100_make_overleaf_data_dirs.sh

* [WIP] patch for hotfix 5.5.5

* Revert "[WIP] patch for hotfix 5.5.5"

This reverts commit cc83a526ecd86ed7b1a0e061b074d2fb15d4d672.

* Revert "[WIP] restore 100_make_overleaf_data_dirs.sh"

This reverts commit e9b0597fe8451e9ef1deaf7ed31790f7d7fc996e.

* [server-pro] extend hotfix 5.5.5: avoid creating user_files directory

* [server-ce] run binary files migration in CI

GitOrigin-RevId: ac5bb86df92b441201c69dfac6b302058b501d8e

server-ce/hotfix/5.5.5/Dockerfile

@@ -3,5 +3,7 @@ FROM sharelatex/sharelatex:5.5.4
 # ../../bin/import_pr_patch.sh 27932
 # Remove web migrations changes
 # Remove test changes
-COPY *.patch .
-RUN bash -ec 'for p in *.patch; do echo "=== Applying $p ==="; patch -p1 < "$p" && rm $p; done'
+# Add init script change (commit cc83a526ecd86ed7b1a0e061b074d2fb15d4d672 in PR 28738)
+COPY *.patch* .
+RUN bash -ec 'for p in *.patch; do echo "=== Applying $p ==="; patch -p1 < "$p" && rm $p; done' \
+&&  bash -ec 'cd / && for p in /overleaf/*.patch-abs; do echo "=== Applying $p ==="; patch -p1 < "$p" && rm $p; done'

server-ce/hotfix/5.5.5/pr_28738.patch-abs

@@ -0,0 +1,12 @@
+--- a/etc/my_init.d/100_make_overleaf_data_dirs.sh
++++ b/etc/my_init.d/100_make_overleaf_data_dirs.sh
+@@ -4,8 +4,10 @@ set -e
+ mkdir -p /var/lib/overleaf/data
+ chown www-data:www-data /var/lib/overleaf/data
+
+-mkdir -p /var/lib/overleaf/data/user_files
+-chown www-data:www-data /var/lib/overleaf/data/user_files
++if [[ "${OVERLEAF_FILESTORE_MIGRATION_LEVEL:-0}" != "2" ]]; then
++  mkdir -p /var/lib/overleaf/data/user_files
++  chown www-data:www-data /var/lib/overleaf/data/user_files
++fi

server-ce/init_scripts/100_make_overleaf_data_dirs.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 set -e
 
 mkdir -p /var/lib/overleaf/data

server-ce/test/Jenkinsfile

@@ -305,6 +305,22 @@ pipeline {
             }
           }
         }
+        stage('PRO custom 5') {
+          environment {
+            CYPRESS_SHARD = "PRO_CUSTOM_5"
+            COMPOSE_PROJECT_NAME = "test-pro-custom-5"
+          }
+          steps {
+            script {
+              waitUntil {
+                return job_npm_install_done && job_server_pro_build_done && job_prefetch_default_done && job_prefetch_custom_done
+              }
+            }
+            dir('server-ce/test') {
+              sh 'make test-e2e'
+            }
+          }
+        }
       }
     }
   }

server-ce/test/filestore-migration.spec.ts

@@ -28,7 +28,7 @@ function activateUserVersion1x(url: string, password = DEFAULT_PASSWORD) {
 }
 
 describe('filestore migration', function () {
-  if (isExcludedBySharding('LOCAL_ONLY')) return
+  if (isExcludedBySharding('PRO_CUSTOM_5')) return
   const email = 'user@example.com'
   // Branding of env vars changed in 5.x
   const sharelatexBrandedVars = {
@@ -385,16 +385,40 @@ describe('filestore migration', function () {
 
         describe('purge filestore data', function () {
           before(async function () {
-            await purgeFilestoreData()
+            const deleted = await purgeFilestoreData()
+            expect(deleted).to.have.length.greaterThan(
+              previousBinaryFiles.length
+            )
+            expect(deleted).to.include(
+              "removed directory '/var/lib/overleaf/data/user_files'"
+            )
           })
           checkFilesAreAccessible()
 
+          describe('after next restart', function () {
+            startWith({
+              version: '5.5.5',
+              pro: true,
+              withDataDir: true,
+              vars: {
+                OVERLEAF_APP_NAME: 'change-config',
+                OVERLEAF_FILESTORE_MIGRATION_LEVEL: '2',
+              },
+            })
+            it('should not recreate the user_files folder', async function () {
+              expect(await purgeFilestoreData()).to.deep.equal([])
+            })
+          })
+
           describe('latest', function () {
             startWith({
               pro: true,
               withDataDir: true,
               vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '2' },
             })
+            it('should not recreate the user_files folder', async function () {
+              expect(await purgeFilestoreData()).to.deep.equal([])
+            })
             checkFilesAreAccessible()
           })
         })

server-ce/test/helpers/config.ts

@@ -15,6 +15,7 @@ export function isExcludedBySharding(
     | 'PRO_CUSTOM_2'
     | 'PRO_CUSTOM_3'
     | 'PRO_CUSTOM_4'
+    | 'PRO_CUSTOM_5'
 ) {
   const SHARD = Cypress.env('SHARD')
   return SHARD && shard !== SHARD

server-ce/test/helpers/hostAdminClient.ts

@@ -122,9 +122,11 @@ export async function setMongoFeatureCompatibilityVersion(
 }
 
 export async function purgeFilestoreData() {
-  await fetchJSON(`${hostAdminURL}/data/user_files`, {
+  const { stdout } = await fetchJSON(`${hostAdminURL}/data/user_files`, {
     method: 'DELETE',
   })
+  if (!stdout.trim()) return []
+  return stdout.trim().split('\n')
 }
 
 async function sleep(ms: number) {

server-ce/test/host-admin.js

@@ -443,7 +443,7 @@ app.get('/redis/keys', (req, res) => {
 app.delete('/data/user_files', (req, res) => {
   runDockerCompose(
     'exec',
-    ['sharelatex', 'rm', '-rf', '/var/lib/overleaf/data/user_files'],
+    ['sharelatex', 'rm', '-vrf', '/var/lib/overleaf/data/user_files'],
     (error, stdout, stderr) => {
       res.json({ error, stdout, stderr })
     }