[web] download binary files in clsi from filestore via new endpoints (#27505)

* [history-v1] use String.padStart instead of lodash.padStart

* [web] download binary files in clsi from filestore via new endpoints

* [server-ce] tests: Cypress.env() is parsing boolean values

* [server-ce] tests: run history migration as root

GitOrigin-RevId: bdf6c0e542531ccc4b3f13d2ed68ca0d31e580e9

server-ce/config/settings.js

@@ -441,6 +441,8 @@ switch (process.env.OVERLEAF_FILESTORE_BACKEND) {
         user_files: process.env.OVERLEAF_FILESTORE_USER_FILES_BUCKET_NAME,
         template_files:
           process.env.OVERLEAF_FILESTORE_TEMPLATE_FILES_BUCKET_NAME,
+        project_blobs: process.env.OVERLEAF_HISTORY_PROJECT_BLOBS_BUCKET,
+        global_blobs: process.env.OVERLEAF_HISTORY_BLOBS_BUCKET,
       },
       s3: {
         key:
@@ -463,6 +465,15 @@ switch (process.env.OVERLEAF_FILESTORE_BACKEND) {
       stores: {
         user_files: Path.join(DATA_DIR, 'user_files'),
         template_files: Path.join(DATA_DIR, 'template_files'),
+
+        // NOTE: The below paths are hard-coded in server-ce/config/production.json, so hard code them here as well.
+        // We can use DATA_DIR after switching history-v1 from 'config' to '@overleaf/settings'.
+        project_blobs:
+          process.env.OVERLEAF_HISTORY_PROJECT_BLOBS_BUCKET ||
+          '/var/lib/overleaf/data/history/overleaf-project-blobs',
+        global_blobs:
+          process.env.OVERLEAF_HISTORY_BLOBS_BUCKET ||
+          '/var/lib/overleaf/data/history/overleaf-global-blobs',
       },
     }
 }

server-ce/test/docker-compose.yml

@@ -75,6 +75,7 @@ services:
     environment:
       CYPRESS_SHARD:
       CYPRESS_BASE_URL: http://sharelatex
+      CYPRESS_FULL_FILESTORE_MIGRATION:
       SPEC_PATTERN: '**/*.spec.{js,jsx,ts,tsx}'
     depends_on:
       sharelatex:

server-ce/test/filestore-migration.spec.ts

@@ -1,5 +1,6 @@
-import { DEFAULT_PASSWORD, login } from './helpers/login'
+import { DEFAULT_PASSWORD, ensureUserExists, login } from './helpers/login'
 import {
+  createProject,
   expectFileExists,
   openProjectById,
   prepareFileUploadTest,
@@ -35,7 +36,8 @@ describe('filestore migration', function () {
     SHARELATEX_MONGO_URL: 'mongodb://mongo/sharelatex',
     SHARELATEX_REDIS_HOST: 'redis',
   }
-  let projectName: string
+  const projectName = `project-${uuid()}`
+  let defaultImage: string
   let projectId: string
   let waitForCompileRateLimitCoolOff: (fn: () => void) => void
   const previousBinaryFiles: (() => void)[] = []
@@ -49,7 +51,7 @@ describe('filestore migration', function () {
   }
 
   function addNewBinaryFileAndCheckPrevious(
-    universeSelector = 'img[alt="universe.jpg"]'
+    universeSelector = `img[alt="${defaultImage}"]`
   ) {
     before(function () {
       login(email)
@@ -61,7 +63,7 @@ describe('filestore migration', function () {
       for (const check of previousBinaryFiles) {
         check()
       }
-      cy.findByRole('treeitem', { name: 'universe.jpg' }).click()
+      cy.findByRole('treeitem', { name: defaultImage }).click()
       cy.get(universeSelector)
         .should('be.visible')
         .and('have.prop', 'naturalWidth')
@@ -71,184 +73,223 @@ describe('filestore migration', function () {
     })
   }
 
-  // --------------
-  // Server Pro 1.x
-  startWith({
-    pro: true,
-    resetData: true,
-    withDataDir: true,
-    vars: sharelatexBrandedVars,
-    version: '1.2.4',
-    mongoVersion: '5.0',
-  })
-
-  let activateURL: string
-  before(async function () {
-    const { stdout } = await runGruntTask({
-      task: 'user:create-admin',
-      args: ['--email', email],
+  if (Cypress.env('FULL_FILESTORE_MIGRATION')) {
+    // --------------
+    // Server Pro 1.x
+    startWith({
+      pro: true,
+      resetData: true,
+      withDataDir: true,
+      vars: sharelatexBrandedVars,
+      version: '1.2.4',
+      mongoVersion: '5.0',
     })
-    ;[activateURL] = stdout.match(
-      /http:\/\/.+\/user\/password\/set\?passwordResetToken=\S+/
-    )!
-  })
-  before(function () {
-    activateUserVersion1x(activateURL)
-    login(email)
-    projectName = `project-${uuid()}`
-    cy.visit('/project')
+    defaultImage = 'universe.jpg'
 
-    // Legacy angular based UI uses links instead of buttons
-    cy.findByRole('link', {
-      name: /Create First Project|New Project/,
-    }).click()
-    cy.findByRole('link', { name: 'Example Project' }).click()
-    cy.findByPlaceholderText('Project Name').type(projectName)
-    cy.findByRole('button', { name: 'Create' }).click()
-    cy.url()
-      .should('match', /\/project\/[a-fA-F0-9]{24}/)
-      .then(url => (projectId = url.split('/').pop()!))
-    let queueReset
-    ;({ waitForCompileRateLimitCoolOff, queueReset } =
-      prepareWaitForNextCompileSlot())
-    queueReset()
+    let activateURL: string
+    before(async function () {
+      const { stdout } = await runGruntTask({
+        task: 'user:create-admin',
+        args: ['--email', email],
+      })
+      ;[activateURL] = stdout.match(
+        /http:\/\/.+\/user\/password\/set\?passwordResetToken=\S+/
+      )!
+    })
+    before(function () {
+      activateUserVersion1x(activateURL)
+      login(email)
+      cy.visit('/project')
 
-    // Create a new binary file
-    cy.get(`a[tooltip="Upload"]`).click()
-    const name = `${uuid()}.txt`
-    // Binary file detection is not sophisticated in version 1.x
-    const binName = name.replace('.txt', '.bin')
-    const content = `Test File Content ${name} \x00`
-    cy.get('input[type=file]')
-      .first()
-      .selectFile(
+      // Legacy angular based UI uses links instead of buttons
+      cy.findByRole('link', {
+        name: /Create First Project|New Project/,
+      }).click()
+      cy.findByRole('link', { name: 'Example Project' }).click()
+      cy.findByPlaceholderText('Project Name').type(projectName)
+      cy.findByRole('button', { name: 'Create' }).click()
+      cy.url()
+        .should('match', /\/project\/[a-fA-F0-9]{24}/)
+        .then(url => (projectId = url.split('/').pop()!))
+      let queueReset
+      ;({ waitForCompileRateLimitCoolOff, queueReset } =
+        prepareWaitForNextCompileSlot())
+      queueReset()
+
+      // Create a new binary file
+      cy.get(`a[tooltip="Upload"]`).click()
+      const name = `${uuid()}.txt`
+      // Binary file detection is not sophisticated in version 1.x
+      const binName = name.replace('.txt', '.bin')
+      const content = `Test File Content ${name} \x00`
+      cy.get('input[type=file]')
+        .first()
+        .selectFile(
+          {
+            contents: Cypress.Buffer.from(content),
+            fileName: binName,
+            lastModified: Date.now(),
+          },
+          { force: true }
+        )
+      // Rename back to .txt to enable preview
+      cy.findByText(binName).click()
+      cy.findByText(binName).dblclick()
+      cy.focused().type(name + '{del}'.repeat('.bin'.length) + '{enter}')
+      // Switch back and forth
+      cy.findByText('universe.jpg').click()
+      cy.findByText(name).click()
+      cy.findByText(content)
+        .parent()
+        .parent()
+        .should('have.class', 'text-preview')
+
+      previousBinaryFiles.push(() => expectFileExists(name, true, content))
+      avoid502()
+    })
+
+    // --------------
+    // Server Pro 2.x
+    startWith({
+      pro: true,
+      withDataDir: true,
+      vars: sharelatexBrandedVars,
+      version: '2.7.1',
+      mongoVersion: '5.0',
+    })
+    before(function () {
+      // Cypress strips the Content-Length header: https://github.com/cypress-io/cypress/issues/16469
+      // Server Pro 2.x does not gracefully handle a missing value.
+      cy.intercept(
         {
-          contents: Cypress.Buffer.from(content),
-          fileName: binName,
-          lastModified: Date.now(),
+          method: 'HEAD',
+          url: `http://sharelatex/project/${projectId}/file/*`,
+          times: previousBinaryFiles.length + 1,
         },
-        { force: true }
+        req => {
+          req.continue(res => {
+            res.headers['Content-Length'] = '60'
+          })
+        }
       )
-    // Rename back to .txt to enable preview
-    cy.findByText(binName).click()
-    cy.findByText(binName).dblclick()
-    cy.focused().type(name + '{del}'.repeat('.bin'.length) + '{enter}')
-    // Switch back and forth
-    cy.findByText('universe.jpg').click()
-    cy.findByText(name).click()
-    cy.findByText(content)
-      .parent()
-      .parent()
-      .should('have.class', 'text-preview')
+    })
+    // Server Pro 2.x does not have alt tags on images.
+    addNewBinaryFileAndCheckPrevious('img')
 
-    previousBinaryFiles.push(() => expectFileExists(name, true, content))
-    avoid502()
-  })
+    // ----------------------------------
+    // Server Pro 3.x + history migration
+    startWith({
+      pro: true,
+      withDataDir: true,
+      vars: sharelatexBrandedVars,
+      version: '3.5.13',
+      mongoVersion: '5.0',
+    })
+    addNewBinaryFileAndCheckPrevious() // before history migration
+    before(async function () {
+      await runScript({
+        cwd: 'services/web',
+        script: 'scripts/history/migrate_history.js',
+        args: [
+          '--force-clean',
+          '--fix-invalid-characters',
+          '--convert-large-docs-to-file',
+        ],
+        hasOverleafEnv: false,
+        user: 'root',
+      })
+    })
+    before(async function () {
+      await runScript({
+        cwd: 'services/web',
+        script: 'scripts/history/clean_sl_history_data.js',
+        hasOverleafEnv: false,
+      })
+    })
+    addNewBinaryFileAndCheckPrevious() // after history migration
 
-  // --------------
-  // Server Pro 2.x
-  startWith({
-    pro: true,
-    withDataDir: true,
-    vars: sharelatexBrandedVars,
-    version: '2.7.1',
-    mongoVersion: '5.0',
-  })
-  before(function () {
-    // Cypress strips the Content-Length header: https://github.com/cypress-io/cypress/issues/16469
-    // Server Pro 2.x does not gracefully handle a missing value.
-    cy.intercept(
-      {
-        method: 'HEAD',
-        url: `http://sharelatex/project/${projectId}/file/*`,
-        times: previousBinaryFiles.length + 1,
-      },
-      req => {
-        req.continue(res => {
-          res.headers['Content-Length'] = '60'
-        })
+    // ------------------------------
+    // Server Pro 4.x + mongo upgrade
+    startWith({
+      pro: true,
+      withDataDir: true,
+      vars: sharelatexBrandedVars,
+      version: '4.2.9',
+      mongoVersion: '5.0',
+    })
+    startWith({
+      pro: true,
+      withDataDir: true,
+      vars: sharelatexBrandedVars,
+      version: '4.2.9',
+      mongoVersion: '6.0',
+    })
+    before(async function () {
+      await setMongoFeatureCompatibilityVersion('6.0')
+    })
+    addNewBinaryFileAndCheckPrevious()
+
+    // ------------------------------------------
+    // Server Pro 5.x + mongo upgrade 6 -> 7 -> 8
+    startWith({
+      pro: true,
+      withDataDir: true,
+      mongoVersion: '6.0',
+    })
+    startWith({
+      pro: true,
+      withDataDir: true,
+      mongoVersion: '7.0',
+    })
+    before(async function () {
+      await setMongoFeatureCompatibilityVersion('7.0')
+    })
+    startWith({
+      pro: true,
+      withDataDir: true,
+      // implicit mongo upgrade to 8.0
+    })
+    before(async function () {
+      await setMongoFeatureCompatibilityVersion('8.0')
+    })
+  } else {
+    // 5.x
+    startWith({ pro: true, withDataDir: true })
+    defaultImage = 'frog.jpg'
+    ensureUserExists({ email })
+    before(function () {
+      login(email)
+      createProject(projectName, { type: 'Example project', open: false }).then(
+        id => (projectId = id)
+      )
+      ;({ waitForCompileRateLimitCoolOff } = prepareWaitForNextCompileSlot())
+    })
+  }
+  addNewBinaryFileAndCheckPrevious()
+
+  function ensureStopOnFirstErrorIsActive() {
+    cy.findByRole('button', { name: 'Toggle compile options menu' }).click()
+    cy.findByRole('menuitem', {
+      name: 'Stop on first error',
+    }).then(el => {
+      // NOTE: THIS IS BAD, but the selected option is otherwise not accessible :/
+      if (
+        el.get()[0]?.querySelector('.material-symbol')?.textContent !== 'check'
+      ) {
+        cy.findByRole('menuitem', {
+          name: 'Stop on first error',
+        }).click()
+        // Clicking on "Stop on first error" closes the mode. Open it again.
+        cy.findByRole('button', { name: 'Toggle compile options menu' }).click()
       }
-    )
-  })
-  // Server Pro 2.x does not have alt tags on images.
-  addNewBinaryFileAndCheckPrevious('img')
-
-  // ----------------------------------
-  // Server Pro 3.x + history migration
-  startWith({
-    pro: true,
-    withDataDir: true,
-    vars: sharelatexBrandedVars,
-    version: '3.5.13',
-    mongoVersion: '5.0',
-  })
-  addNewBinaryFileAndCheckPrevious() // before history migration
-  before(async function () {
-    await runScript({
-      cwd: 'services/web',
-      script: 'scripts/history/migrate_history.js',
-      args: [
-        '--force-clean',
-        '--fix-invalid-characters',
-        '--convert-large-docs-to-file',
-      ],
-      hasOverleafEnv: false,
     })
-  })
-  before(async function () {
-    await runScript({
-      cwd: 'services/web',
-      script: 'scripts/history/clean_sl_history_data.js',
-      hasOverleafEnv: false,
+    cy.findByRole('menuitem', {
+      name: 'Stop on first error',
+    }).within(() => {
+      cy.findByText('check').should('be.visible')
     })
-  })
-  addNewBinaryFileAndCheckPrevious() // after history migration
-
-  // ------------------------------
-  // Server Pro 4.x + mongo upgrade
-  startWith({
-    pro: true,
-    withDataDir: true,
-    vars: sharelatexBrandedVars,
-    version: '4.2.9',
-    mongoVersion: '5.0',
-  })
-  startWith({
-    pro: true,
-    withDataDir: true,
-    vars: sharelatexBrandedVars,
-    version: '4.2.9',
-    mongoVersion: '6.0',
-  })
-  before(async function () {
-    await setMongoFeatureCompatibilityVersion('6.0')
-  })
-  addNewBinaryFileAndCheckPrevious()
-
-  // ------------------------------------------
-  // Server Pro 5.x + mongo upgrade 6 -> 7 -> 8
-  startWith({
-    pro: true,
-    withDataDir: true,
-    mongoVersion: '6.0',
-  })
-  startWith({
-    pro: true,
-    withDataDir: true,
-    mongoVersion: '7.0',
-  })
-  before(async function () {
-    await setMongoFeatureCompatibilityVersion('7.0')
-  })
-  startWith({
-    pro: true,
-    withDataDir: true,
-    // implicit mongo upgrade to 8.0
-  })
-  before(async function () {
-    await setMongoFeatureCompatibilityVersion('8.0')
-  })
-  addNewBinaryFileAndCheckPrevious()
+    cy.findByRole('button', { name: 'Toggle compile options menu' }).click()
+  }
 
   // -------------------
   // filestore-migration
@@ -257,6 +298,7 @@ describe('filestore migration', function () {
     waitForCompileRateLimitCoolOff(() => {
       openProjectById(projectId)
     })
+    ensureStopOnFirstErrorIsActive()
   })
 
   function checkFilesAreAccessible() {
@@ -267,13 +309,31 @@ describe('filestore migration', function () {
       }
     })
 
-    it('renders universe jpg', () => {
-      cy.findByTestId('file-tree').findByText('universe.jpg').click()
-      cy.get('[alt="universe.jpg"]')
+    it('renders image of example project', () => {
+      cy.findByTestId('file-tree').findByText(defaultImage).click()
+      cy.get(`[alt="${defaultImage}"]`)
         .should('be.visible')
         .and('have.prop', 'naturalWidth')
         .should('be.greaterThan', 0)
     })
+
+    it('can recompile from scratch', function () {
+      const id = uuid()
+      cy.findByText('\\maketitle').parent().click()
+      cy.findByText('\\maketitle')
+        .parent()
+        .type(`\n\\section{{}Test Section ${id}}`)
+
+      waitForCompileRateLimitCoolOff(() => {
+        cy.findByRole('button', { name: 'Toggle compile options menu' }).click()
+
+        cy.findByRole('menuitem', {
+          name: 'Recompile from scratch',
+        }).trigger('click')
+      })
+
+      cy.get('.pdf-viewer').should('contain.text', `Test Section ${id}`)
+    })
   }
 
   describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL not set', function () {

server-ce/test/helpers/hostAdminClient.ts

@@ -65,11 +65,13 @@ export async function runScript({
   cwd,
   script,
   args = [],
+  user = 'www-data',
   hasOverleafEnv = true,
 }: {
   cwd: string
   script: string
   args?: string[]
+  user?: string
   hasOverleafEnv?: boolean
 }) {
   return await fetchJSON(`${hostAdminURL}/run/script`, {
@@ -78,6 +80,7 @@ export async function runScript({
       cwd,
       script,
       args,
+      user,
       hasOverleafEnv,
     }),
   })

server-ce/test/host-admin.js

@@ -107,13 +107,14 @@ app.post(
         cwd: Joi.string().required(),
         script: Joi.string().required(),
         args: Joi.array().items(Joi.string()),
+        user: Joi.string().required(),
         hasOverleafEnv: Joi.boolean().required(),
       },
     },
     { allowUnknown: false }
   ),
   (req, res) => {
-    const { cwd, script, args, hasOverleafEnv } = req.body
+    const { cwd, script, args, user, hasOverleafEnv } = req.body
 
     const env = hasOverleafEnv
       ? 'source /etc/overleaf/env.sh || source /etc/sharelatex/env.sh'
@@ -127,7 +128,7 @@ app.post(
         'sharelatex',
         'bash',
         '-c',
-        `source /etc/container_environment.sh && ${env} && /sbin/setuser www-data node ${JSON.stringify(script)} ${args.map(a => JSON.stringify(a)).join(' ')}`,
+        `source /etc/container_environment.sh && ${env} && /sbin/setuser ${user} node ${script} ${args.map(a => JSON.stringify(a)).join(' ')}`,
       ],
       (error, stdout, stderr) => {
         res.json({