From a877979b11471907816908894a9ad793eb2b897a Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 23 Jul 2025 16:06:47 +0200 Subject: [PATCH] [server-ce] test filestore migration with upgrade from version 1.x (#27342) * [server-ce] test filestore migration with upgrade from version 1.x * [server-ce] tests: drop verbose logs from host-admin in CI * [server-ce] tests: fix flag following rebase GitOrigin-RevId: dc00127fc76f87ee3eb5071fd430f4917e8123ff --- server-ce/test/Makefile | 2 +- server-ce/test/filestore-migration.spec.ts | 263 +++++++++++++++++++-- server-ce/test/helpers/config.ts | 7 + server-ce/test/helpers/hostAdminClient.ts | 33 +++ server-ce/test/helpers/login.ts | 2 +- server-ce/test/helpers/project.ts | 22 +- server-ce/test/host-admin.js | 111 ++++++++- 7 files changed, 403 insertions(+), 37 deletions(-) diff --git a/server-ce/test/Makefile b/server-ce/test/Makefile index fb7c980293..155878d03f 100644 --- a/server-ce/test/Makefile +++ b/server-ce/test/Makefile @@ -26,7 +26,7 @@ test-e2e: test-e2e-open: docker compose up -d host-admin - docker compose up --no-log-prefix --exit-code-from=e2e-open e2e-open + docker compose up --no-log-prefix --exit-code-from=e2e-open e2e-open host-admin clean: docker compose down --volumes --timeout 0 diff --git a/server-ce/test/filestore-migration.spec.ts b/server-ce/test/filestore-migration.spec.ts index 25875ad374..27e93903a0 100644 --- a/server-ce/test/filestore-migration.spec.ts +++ b/server-ce/test/filestore-migration.spec.ts @@ -1,39 +1,259 @@ -import { ensureUserExists, login } from './helpers/login' +import { DEFAULT_PASSWORD, login } from './helpers/login' import { - createProject, + expectFileExists, openProjectById, prepareFileUploadTest, } from './helpers/project' import { isExcludedBySharding, startWith } from './helpers/config' import { prepareWaitForNextCompileSlot } from './helpers/compile' -import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry' import { v4 as uuid } from 'uuid' -import { purgeFilestoreData, runScript } from './helpers/hostAdminClient' +import { + purgeFilestoreData, + runGruntTask, + runScript, + setMongoFeatureCompatibilityVersion, +} from './helpers/hostAdminClient' + +function activateUserVersion1x(url: string, password = DEFAULT_PASSWORD) { + cy.session(url, () => { + cy.visit(url) + cy.url().then(url => { + if (url.includes('/login')) return + cy.url().should('contain', '/user/password/set') + cy.get('input[type="password"]').type(password) + cy.findByRole('button', { name: 'Set new password' }).click() + }) + }) +} describe('filestore migration', function () { - if (isExcludedBySharding('CE_CUSTOM_3')) return - startWith({ withDataDir: true, resetData: true, vars: {} }) - ensureUserExists({ email: 'user@example.com' }) - + if (isExcludedBySharding('LOCAL_ONLY')) return + const email = 'user@example.com' + // Branding of env vars changed in 5.x + const sharelatexBrandedVars = { + SHARELATEX_SITE_URL: 'http://sharelatex', + SHARELATEX_MONGO_URL: 'mongodb://mongo/sharelatex', + SHARELATEX_REDIS_HOST: 'redis', + } let projectName: string let projectId: string let waitForCompileRateLimitCoolOff: (fn: () => void) => void const previousBinaryFiles: (() => void)[] = [] - beforeWithReRunOnTestRetry(function () { + + function avoid502() { + // The next step will likely restart the instance and any following + // requests will fail with a 502/bad gateway. Avoid this by navigating + // away from the editor, which will reload upon receiving a + // 'forceDisconnect' socket.io message. + cy.visit('/project') + } + + function addNewBinaryFileAndCheckPrevious( + universeSelector = 'img[alt="universe.jpg"]' + ) { + before(function () { + login(email) + waitForCompileRateLimitCoolOff(() => { + cy.visit(`/project/${projectId}`) + }) + previousBinaryFiles.push(prepareFileUploadTest(true)) + cy.log('check binary files') + for (const check of previousBinaryFiles) { + check() + } + cy.findByRole('treeitem', { name: 'universe.jpg' }).click() + cy.get(universeSelector) + .should('be.visible') + .and('have.prop', 'naturalWidth') + .should('be.greaterThan', 0) + + avoid502() + }) + } + + // -------------- + // Server Pro 1.x + startWith({ + pro: true, + resetData: true, + withDataDir: true, + vars: sharelatexBrandedVars, + version: '1.2.4', + mongoVersion: '5.0', + }) + + let activateURL: string + before(async function () { + const { stdout } = await runGruntTask({ + task: 'user:create-admin', + args: ['--email', email], + }) + ;[activateURL] = stdout.match( + /http:\/\/.+\/user\/password\/set\?passwordResetToken=\S+/ + )! + }) + before(function () { + activateUserVersion1x(activateURL) + login(email) projectName = `project-${uuid()}` - login('user@example.com') - createProject(projectName, { type: 'Example project' }).then( - id => (projectId = id) - ) + cy.visit('/project') + + // Legacy angular based UI uses links instead of buttons + cy.findByRole('link', { + name: /Create First Project|New Project/, + }).click() + cy.findByRole('link', { name: 'Example Project' }).click() + cy.findByPlaceholderText('Project Name').type(projectName) + cy.findByRole('button', { name: 'Create' }).click() + cy.url() + .should('match', /\/project\/[a-fA-F0-9]{24}/) + .then(url => (projectId = url.split('/').pop()!)) let queueReset ;({ waitForCompileRateLimitCoolOff, queueReset } = prepareWaitForNextCompileSlot()) queueReset() - previousBinaryFiles.push(prepareFileUploadTest(true)) + + // Create a new binary file + cy.get(`a[tooltip="Upload"]`).click() + const name = `${uuid()}.txt` + // Binary file detection is not sophisticated in version 1.x + const binName = name.replace('.txt', '.bin') + const content = `Test File Content ${name} \x00` + cy.get('input[type=file]') + .first() + .selectFile( + { + contents: Cypress.Buffer.from(content), + fileName: binName, + lastModified: Date.now(), + }, + { force: true } + ) + // Rename back to .txt to enable preview + cy.findByText(binName).click() + cy.findByText(binName).dblclick() + cy.focused().type(name + '{del}'.repeat('.bin'.length) + '{enter}') + // Switch back and forth + cy.findByText('universe.jpg').click() + cy.findByText(name).click() + cy.findByText(content) + .parent() + .parent() + .should('have.class', 'text-preview') + + previousBinaryFiles.push(() => expectFileExists(name, true, content)) + avoid502() }) + // -------------- + // Server Pro 2.x + startWith({ + pro: true, + withDataDir: true, + vars: sharelatexBrandedVars, + version: '2.7.1', + mongoVersion: '5.0', + }) + before(function () { + // Cypress strips the Content-Length header: https://github.com/cypress-io/cypress/issues/16469 + // Server Pro 2.x does not gracefully handle a missing value. + cy.intercept( + { + method: 'HEAD', + url: `http://sharelatex/project/${projectId}/file/*`, + times: previousBinaryFiles.length + 1, + }, + req => { + req.continue(res => { + res.headers['Content-Length'] = '60' + }) + } + ) + }) + // Server Pro 2.x does not have alt tags on images. + addNewBinaryFileAndCheckPrevious('img') + + // ---------------------------------- + // Server Pro 3.x + history migration + startWith({ + pro: true, + withDataDir: true, + vars: sharelatexBrandedVars, + version: '3.5.13', + mongoVersion: '5.0', + }) + addNewBinaryFileAndCheckPrevious() // before history migration + before(async function () { + await runScript({ + cwd: 'services/web', + script: 'scripts/history/migrate_history.js', + args: [ + '--force-clean', + '--fix-invalid-characters', + '--convert-large-docs-to-file', + ], + hasOverleafEnv: false, + }) + }) + before(async function () { + await runScript({ + cwd: 'services/web', + script: 'scripts/history/clean_sl_history_data.js', + hasOverleafEnv: false, + }) + }) + addNewBinaryFileAndCheckPrevious() // after history migration + + // ------------------------------ + // Server Pro 4.x + mongo upgrade + startWith({ + pro: true, + withDataDir: true, + vars: sharelatexBrandedVars, + version: '4.2.9', + mongoVersion: '5.0', + }) + startWith({ + pro: true, + withDataDir: true, + vars: sharelatexBrandedVars, + version: '4.2.9', + mongoVersion: '6.0', + }) + before(async function () { + await setMongoFeatureCompatibilityVersion('6.0') + }) + addNewBinaryFileAndCheckPrevious() + + // ------------------------------------------ + // Server Pro 5.x + mongo upgrade 6 -> 7 -> 8 + startWith({ + pro: true, + withDataDir: true, + mongoVersion: '6.0', + }) + startWith({ + pro: true, + withDataDir: true, + mongoVersion: '7.0', + }) + before(async function () { + await setMongoFeatureCompatibilityVersion('7.0') + }) + startWith({ + pro: true, + withDataDir: true, + // implicit mongo upgrade to 8.0 + }) + before(async function () { + await setMongoFeatureCompatibilityVersion('8.0') + }) + addNewBinaryFileAndCheckPrevious() + + // ------------------- + // filestore-migration beforeEach(() => { - login('user@example.com') + login(email) waitForCompileRateLimitCoolOff(() => { openProjectById(projectId) }) @@ -47,9 +267,9 @@ describe('filestore migration', function () { } }) - it('renders frog jpg', () => { - cy.findByTestId('file-tree').findByText('frog.jpg').click() - cy.get('[alt="frog.jpg"]') + it('renders universe jpg', () => { + cy.findByTestId('file-tree').findByText('universe.jpg').click() + cy.get('[alt="universe.jpg"]') .should('be.visible') .and('have.prop', 'naturalWidth') .should('be.greaterThan', 0) @@ -57,12 +277,13 @@ describe('filestore migration', function () { } describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL not set', function () { - startWith({ withDataDir: true, vars: {} }) + startWith({ pro: true, withDataDir: true, vars: {} }) checkFilesAreAccessible() }) describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=0', function () { startWith({ + pro: true, withDataDir: true, vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '0' }, }) @@ -70,6 +291,7 @@ describe('filestore migration', function () { describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=1', function () { startWith({ + pro: true, withDataDir: true, vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '1' }, }) @@ -77,6 +299,7 @@ describe('filestore migration', function () { describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=2', function () { startWith({ + pro: true, withDataDir: true, vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '1' }, }) @@ -84,9 +307,11 @@ describe('filestore migration', function () { await runScript({ cwd: 'services/history-v1', script: 'storage/scripts/back_fill_file_hash.mjs', + args: ['--all'], }) }) startWith({ + pro: true, withDataDir: true, vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '2' }, }) diff --git a/server-ce/test/helpers/config.ts b/server-ce/test/helpers/config.ts index 78e81be1f7..f11a4491aa 100644 --- a/server-ce/test/helpers/config.ts +++ b/server-ce/test/helpers/config.ts @@ -6,6 +6,7 @@ export const STARTUP_TIMEOUT = export function isExcludedBySharding( shard: + | 'LOCAL_ONLY' | 'CE_DEFAULT' | 'CE_CUSTOM_1' | 'CE_CUSTOM_2' @@ -29,6 +30,7 @@ export function startWith({ varsFn = () => ({}), withDataDir = false, resetData = false, + mongoVersion = '', }) { before(async function () { Object.assign(vars, varsFn()) @@ -38,14 +40,18 @@ export function startWith({ vars, withDataDir, resetData, + mongoVersion, }) if (resetData) { + cy.log('resetting data and sessions') resetCreatedUsersCache() resetActivateUserRateLimit() // no return here, always reconfigure when resetting data } else if (previousConfigFrontend === cfg) { + cy.log(`already running with ${cfg}`) return } + cy.log(`starting with ${cfg}`) this.timeout(STARTUP_TIMEOUT) const { previousConfigServer } = await reconfigure({ @@ -54,6 +60,7 @@ export function startWith({ vars, withDataDir, resetData, + mongoVersion, }) if (previousConfigServer !== cfg) { await Cypress.session.clearAllSavedSessions() diff --git a/server-ce/test/helpers/hostAdminClient.ts b/server-ce/test/helpers/hostAdminClient.ts index dadfe2b059..56bd7f2b46 100644 --- a/server-ce/test/helpers/hostAdminClient.ts +++ b/server-ce/test/helpers/hostAdminClient.ts @@ -15,6 +15,7 @@ export async function reconfigure({ vars = {}, withDataDir = false, resetData = false, + mongoVersion = '', }): Promise<{ previousConfigServer: string }> { return await fetchJSON(`${hostAdminURL}/reconfigure`, { method: 'POST', @@ -24,6 +25,7 @@ export async function reconfigure({ vars, withDataDir, resetData, + mongoVersion, }), }) } @@ -63,10 +65,12 @@ export async function runScript({ cwd, script, args = [], + hasOverleafEnv = true, }: { cwd: string script: string args?: string[] + hasOverleafEnv?: boolean }) { return await fetchJSON(`${hostAdminURL}/run/script`, { method: 'POST', @@ -74,6 +78,23 @@ export async function runScript({ cwd, script, args, + hasOverleafEnv, + }), + }) +} + +export async function runGruntTask({ + task, + args = [], +}: { + task: string + args?: string[] +}) { + return await fetchJSON(`${hostAdminURL}/run/gruntTask`, { + method: 'POST', + body: JSON.stringify({ + task, + args, }), }) } @@ -85,6 +106,18 @@ export async function getRedisKeys() { return stdout.split('\n') } +export async function setMongoFeatureCompatibilityVersion( + mongoVersion: string +) { + cy.log(`advancing mongo featureCompatibilityVersion to ${mongoVersion}`) + await fetchJSON(`${hostAdminURL}/mongo/setFeatureCompatibilityVersion`, { + method: 'POST', + body: JSON.stringify({ + mongoVersion, + }), + }) +} + export async function purgeFilestoreData() { await fetchJSON(`${hostAdminURL}/data/user_files`, { method: 'DELETE', diff --git a/server-ce/test/helpers/login.ts b/server-ce/test/helpers/login.ts index fa95abec1d..39759eab76 100644 --- a/server-ce/test/helpers/login.ts +++ b/server-ce/test/helpers/login.ts @@ -1,6 +1,6 @@ import { runScript } from './hostAdminClient' -const DEFAULT_PASSWORD = 'Passw0rd!' +export const DEFAULT_PASSWORD = 'Passw0rd!' const createdUsers = new Set() diff --git a/server-ce/test/helpers/project.ts b/server-ce/test/helpers/project.ts index 4b3197afed..bfa1cb1c83 100644 --- a/server-ce/test/helpers/project.ts +++ b/server-ce/test/helpers/project.ts @@ -217,6 +217,19 @@ export function createNewFile() { return fileName } +export function expectFileExists( + name: string, + binary: boolean, + content: string +) { + cy.findByRole('treeitem', { name }).click() + if (binary) { + cy.findByText(content).should('not.have.class', 'cm-line') + } else { + cy.findByText(content).should('have.class', 'cm-line') + } +} + export function prepareFileUploadTest(binary = false) { const name = `${uuid()}.txt` const content = `Test File Content ${name}${binary ? ' \x00' : ''}` @@ -235,14 +248,7 @@ export function prepareFileUploadTest(binary = false) { // wait for the upload to finish cy.findByRole('treeitem', { name }) - return function check() { - cy.findByRole('treeitem', { name }).click() - if (binary) { - cy.findByText(content).should('not.have.class', 'cm-line') - } else { - cy.findByText(content).should('have.class', 'cm-line') - } - } + return () => expectFileExists(name, binary, content) } export function testNewFileUpload() { diff --git a/server-ce/test/host-admin.js b/server-ce/test/host-admin.js index b3dcd72b1f..799c83b6bb 100644 --- a/server-ce/test/host-admin.js +++ b/server-ce/test/host-admin.js @@ -107,20 +107,62 @@ app.post( cwd: Joi.string().required(), script: Joi.string().required(), args: Joi.array().items(Joi.string()), + hasOverleafEnv: Joi.boolean().required(), }, }, { allowUnknown: false } ), (req, res) => { - const { cwd, script, args } = req.body + const { cwd, script, args, hasOverleafEnv } = req.body + + const env = hasOverleafEnv + ? 'source /etc/overleaf/env.sh || source /etc/sharelatex/env.sh' + : 'true' runDockerCompose( 'exec', [ + '--workdir', + `/overleaf/${cwd}`, 'sharelatex', 'bash', '-c', - `source /etc/container_environment.sh && source /etc/overleaf/env.sh || source /etc/sharelatex/env.sh && cd ${JSON.stringify(cwd)} && node ${JSON.stringify(script)} ${args.map(a => JSON.stringify(a)).join(' ')}`, + `source /etc/container_environment.sh && ${env} && node ${JSON.stringify(script)} ${args.map(a => JSON.stringify(a)).join(' ')}`, + ], + (error, stdout, stderr) => { + res.json({ + error, + stdout, + stderr, + }) + } + ) + } +) + +app.post( + '/run/gruntTask', + validate( + { + body: { + task: Joi.string().required(), + args: Joi.array().items(Joi.string()), + }, + }, + { allowUnknown: false } + ), + (req, res) => { + const { task, args } = req.body + + runDockerCompose( + 'exec', + [ + '--workdir', + '/var/www/sharelatex', + 'sharelatex', + 'bash', + '-c', + `source /etc/container_environment.sh && grunt ${JSON.stringify(task)} ${args.map(a => JSON.stringify(a)).join(' ')}`, ], (error, stdout, stderr) => { res.json({ @@ -178,7 +220,13 @@ const allowedVars = Joi.object( ) ) -function setVarsDockerCompose({ pro, vars, version, withDataDir }) { +function setVarsDockerCompose({ + pro, + vars, + version, + withDataDir, + mongoVersion, +}) { const cfg = readDockerComposeOverride() cfg.services.sharelatex.image = `${pro ? IMAGES.PRO : IMAGES.CE}:${version}` @@ -198,8 +246,8 @@ function setVarsDockerCompose({ pro, vars, version, withDataDir }) { const dataDirInContainer = version === 'latest' || version >= '5.0' - ? '/var/lib/overleaf/data' - : '/var/lib/sharelatex/data' + ? '/var/lib/overleaf' + : '/var/lib/sharelatex' cfg.services.sharelatex.volumes = [] if (withDataDir) { @@ -220,11 +268,19 @@ function setVarsDockerCompose({ pro, vars, version, withDataDir }) { ) if (!withDataDir) { cfg.services.sharelatex.volumes.push( - `${PATHS.SANDBOXED_COMPILES_HOST_DIR}:${dataDirInContainer}/compiles` + `${PATHS.SANDBOXED_COMPILES_HOST_DIR}:${dataDirInContainer}/data/compiles` ) } } + if (mongoVersion) { + cfg.services.mongo = { + image: `mongo:${mongoVersion}`, + } + } else { + delete cfg.services.mongo + } + writeDockerComposeOverride(cfg) } @@ -285,6 +341,7 @@ app.post( { body: { pro: Joi.boolean().required(), + mongoVersion: Joi.string().allow('').optional(), version: Joi.string().required(), vars: allowedVars, withDataDir: Joi.boolean().optional(), @@ -294,7 +351,8 @@ app.post( { allowUnknown: false } ), (req, res) => { - const { pro, version, vars, withDataDir, resetData } = req.body + const { pro, version, vars, withDataDir, resetData, mongoVersion } = + req.body maybeResetData(resetData, (error, stdout, stderr) => { if (error) return res.json({ error, stdout, stderr }) @@ -305,7 +363,7 @@ app.post( } try { - setVarsDockerCompose({ pro, version, vars, withDataDir }) + setVarsDockerCompose({ pro, version, vars, withDataDir, mongoVersion }) } catch (error) { return res.json({ error }) } @@ -323,6 +381,43 @@ app.post( } ) +app.post( + '/mongo/setFeatureCompatibilityVersion', + validate( + { + body: { + mongoVersion: Joi.string().required(), + }, + }, + { allowUnknown: false } + ), + (req, res) => { + const { mongoVersion } = req.body + const mongosh = mongoVersion > '5' ? 'mongosh' : 'mongo' + const params = { + setFeatureCompatibilityVersion: mongoVersion, + } + if (mongoVersion >= '7.0') { + // MongoServerError: Once you have upgraded to 7.0, you will not be able to downgrade FCV and binary version without support assistance. Please re-run this command with 'confirm: true' to acknowledge this and continue with the FCV upgrade. + // NOTE: 6.0 does not know about this flag. So conditionally add it. + // MongoServerError: BSON field 'setFeatureCompatibilityVersion.confirm' is an unknown field. + params.confirm = true + } + runDockerCompose( + 'exec', + [ + 'mongo', + mongosh, + '--eval', + `db.adminCommand(${JSON.stringify(params)})`, + ], + (error, stdout, stderr) => { + res.json({ error, stdout, stderr }) + } + ) + } +) + app.get('/redis/keys', (req, res) => { runDockerCompose( 'exec',