agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-05-23 09:09:36 +02:00
Code Issues Packages Projects Releases Wiki Activity

[server-ce] test filestore migration with upgrade from version 1.x (#27342)

Browse Source 
* [server-ce] test filestore migration with upgrade from version 1.x

* [server-ce] tests: drop verbose logs from host-admin in CI

* [server-ce] tests: fix flag following rebase

GitOrigin-RevId: dc00127fc76f87ee3eb5071fd430f4917e8123ff
This commit is contained in:
Jakob Ackermann
2025-07-23 16:06:47 +02:00
committed by Copybot
parent 26dde244a1
commit a877979b11
7 changed files with 403 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server-ce/test/Makefile
View File

@@ -26,7 +26,7 @@ test-e2e:
test-e2e-open:
docker compose up -d host-admin
docker compose up --no-log-prefix --exit-code-from=e2e-open e2e-open
docker compose up --no-log-prefix --exit-code-from=e2e-open e2e-open host-admin
clean:
docker compose down --volumes --timeout 0

263
server-ce/test/filestore-migration.spec.ts
View File

@@ -1,39 +1,259 @@
import { ensureUserExists, login } from './helpers/login'
import { DEFAULT_PASSWORD, login } from './helpers/login'
import {
createProject,
expectFileExists,
openProjectById,
prepareFileUploadTest,
} from './helpers/project'
import { isExcludedBySharding, startWith } from './helpers/config'
import { prepareWaitForNextCompileSlot } from './helpers/compile'
import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry'
import { v4 as uuid } from 'uuid'
import { purgeFilestoreData, runScript } from './helpers/hostAdminClient'
import {
purgeFilestoreData,
runGruntTask,
runScript,
setMongoFeatureCompatibilityVersion,
} from './helpers/hostAdminClient'
function activateUserVersion1x(url: string, password = DEFAULT_PASSWORD) {
cy.session(url, () => {
cy.visit(url)
cy.url().then(url => {
if (url.includes('/login')) return
cy.url().should('contain', '/user/password/set')
cy.get('input[type="password"]').type(password)
cy.findByRole('button', { name: 'Set new password' }).click()
})
})
}
describe('filestore migration', function () {
if (isExcludedBySharding('CE_CUSTOM_3')) return
startWith({ withDataDir: true, resetData: true, vars: {} })
ensureUserExists({ email: 'user@example.com' })
if (isExcludedBySharding('LOCAL_ONLY')) return
const email = 'user@example.com'
// Branding of env vars changed in 5.x
const sharelatexBrandedVars = {
SHARELATEX_SITE_URL: 'http://sharelatex',
SHARELATEX_MONGO_URL: 'mongodb://mongo/sharelatex',
SHARELATEX_REDIS_HOST: 'redis',
}
let projectName: string
let projectId: string
let waitForCompileRateLimitCoolOff: (fn: () => void) => void
const previousBinaryFiles: (() => void)[] = []
beforeWithReRunOnTestRetry(function () {
function avoid502() {
// The next step will likely restart the instance and any following
// requests will fail with a 502/bad gateway. Avoid this by navigating
// away from the editor, which will reload upon receiving a
// 'forceDisconnect' socket.io message.
cy.visit('/project')
}
function addNewBinaryFileAndCheckPrevious(
universeSelector = 'img[alt="universe.jpg"]'
) {
before(function () {
login(email)
waitForCompileRateLimitCoolOff(() => {
cy.visit(`/project/${projectId}`)
})
previousBinaryFiles.push(prepareFileUploadTest(true))
cy.log('check binary files')
for (const check of previousBinaryFiles) {
check()
}
cy.findByRole('treeitem', { name: 'universe.jpg' }).click()
cy.get(universeSelector)
.should('be.visible')
.and('have.prop', 'naturalWidth')
.should('be.greaterThan', 0)
avoid502()
})
}
// --------------
// Server Pro 1.x
startWith({
pro: true,
resetData: true,
withDataDir: true,
vars: sharelatexBrandedVars,
version: '1.2.4',
mongoVersion: '5.0',
})
let activateURL: string
before(async function () {
const { stdout } = await runGruntTask({
task: 'user:create-admin',
args: ['--email', email],
})
;[activateURL] = stdout.match(
/http:\/\/.+\/user\/password\/set\?passwordResetToken=\S+/
)!
})
before(function () {
activateUserVersion1x(activateURL)
login(email)
projectName = `project-${uuid()}`
login('user@example.com')
createProject(projectName, { type: 'Example project' }).then(
id => (projectId = id)
)
cy.visit('/project')
// Legacy angular based UI uses links instead of buttons
cy.findByRole('link', {
name: /Create First Project|New Project/,
}).click()
cy.findByRole('link', { name: 'Example Project' }).click()
cy.findByPlaceholderText('Project Name').type(projectName)
cy.findByRole('button', { name: 'Create' }).click()
cy.url()
.should('match', /\/project\/[a-fA-F0-9]{24}/)
.then(url => (projectId = url.split('/').pop()!))
let queueReset
;({ waitForCompileRateLimitCoolOff, queueReset } =
prepareWaitForNextCompileSlot())
queueReset()
previousBinaryFiles.push(prepareFileUploadTest(true))
// Create a new binary file
cy.get(`a[tooltip="Upload"]`).click()
const name = `${uuid()}.txt`
// Binary file detection is not sophisticated in version 1.x
const binName = name.replace('.txt', '.bin')
const content = `Test File Content ${name} \x00`
cy.get('input[type=file]')
.first()
.selectFile(
{
contents: Cypress.Buffer.from(content),
fileName: binName,
lastModified: Date.now(),
},
{ force: true }
)
// Rename back to .txt to enable preview
cy.findByText(binName).click()
cy.findByText(binName).dblclick()
cy.focused().type(name + '{del}'.repeat('.bin'.length) + '{enter}')
// Switch back and forth
cy.findByText('universe.jpg').click()
cy.findByText(name).click()
cy.findByText(content)
.parent()
.parent()
.should('have.class', 'text-preview')
previousBinaryFiles.push(() => expectFileExists(name, true, content))
avoid502()
})
// --------------
// Server Pro 2.x
startWith({
pro: true,
withDataDir: true,
vars: sharelatexBrandedVars,
version: '2.7.1',
mongoVersion: '5.0',
})
before(function () {
// Cypress strips the Content-Length header: https://github.com/cypress-io/cypress/issues/16469
// Server Pro 2.x does not gracefully handle a missing value.
cy.intercept(
{
method: 'HEAD',
url: `http://sharelatex/project/${projectId}/file/*`,
times: previousBinaryFiles.length + 1,
},
req => {
req.continue(res => {
res.headers['Content-Length'] = '60'
})
}
)
})
// Server Pro 2.x does not have alt tags on images.
addNewBinaryFileAndCheckPrevious('img')
// ----------------------------------
// Server Pro 3.x + history migration
startWith({
pro: true,
withDataDir: true,
vars: sharelatexBrandedVars,
version: '3.5.13',
mongoVersion: '5.0',
})
addNewBinaryFileAndCheckPrevious() // before history migration
before(async function () {
await runScript({
cwd: 'services/web',
script: 'scripts/history/migrate_history.js',
args: [
'--force-clean',
'--fix-invalid-characters',
'--convert-large-docs-to-file',
],
hasOverleafEnv: false,
})
})
before(async function () {
await runScript({
cwd: 'services/web',
script: 'scripts/history/clean_sl_history_data.js',
hasOverleafEnv: false,
})
})
addNewBinaryFileAndCheckPrevious() // after history migration
// ------------------------------
// Server Pro 4.x + mongo upgrade
startWith({
pro: true,
withDataDir: true,
vars: sharelatexBrandedVars,
version: '4.2.9',
mongoVersion: '5.0',
})
startWith({
pro: true,
withDataDir: true,
vars: sharelatexBrandedVars,
version: '4.2.9',
mongoVersion: '6.0',
})
before(async function () {
await setMongoFeatureCompatibilityVersion('6.0')
})
addNewBinaryFileAndCheckPrevious()
// ------------------------------------------
// Server Pro 5.x + mongo upgrade 6 -> 7 -> 8
startWith({
pro: true,
withDataDir: true,
mongoVersion: '6.0',
})
startWith({
pro: true,
withDataDir: true,
mongoVersion: '7.0',
})
before(async function () {
await setMongoFeatureCompatibilityVersion('7.0')
})
startWith({
pro: true,
withDataDir: true,
// implicit mongo upgrade to 8.0
})
before(async function () {
await setMongoFeatureCompatibilityVersion('8.0')
})
addNewBinaryFileAndCheckPrevious()
// -------------------
// filestore-migration
beforeEach(() => {
login('user@example.com')
login(email)
waitForCompileRateLimitCoolOff(() => {
openProjectById(projectId)
})
@@ -47,9 +267,9 @@ describe('filestore migration', function () {
}
})
it('renders frog jpg', () => {
cy.findByTestId('file-tree').findByText('frog.jpg').click()
cy.get('[alt="frog.jpg"]')
it('renders universe jpg', () => {
cy.findByTestId('file-tree').findByText('universe.jpg').click()
cy.get('[alt="universe.jpg"]')
.should('be.visible')
.and('have.prop', 'naturalWidth')
.should('be.greaterThan', 0)
@@ -57,12 +277,13 @@ describe('filestore migration', function () {
}
describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL not set', function () {
startWith({ withDataDir: true, vars: {} })
startWith({ pro: true, withDataDir: true, vars: {} })
checkFilesAreAccessible()
})
describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=0', function () {
startWith({
pro: true,
withDataDir: true,
vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '0' },
})
@@ -70,6 +291,7 @@ describe('filestore migration', function () {
describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=1', function () {
startWith({
pro: true,
withDataDir: true,
vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '1' },
})
@@ -77,6 +299,7 @@ describe('filestore migration', function () {
describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=2', function () {
startWith({
pro: true,
withDataDir: true,
vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '1' },
})
@@ -84,9 +307,11 @@ describe('filestore migration', function () {
await runScript({
cwd: 'services/history-v1',
script: 'storage/scripts/back_fill_file_hash.mjs',
args: ['--all'],
})
})
startWith({
pro: true,
withDataDir: true,
vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '2' },
})

7
server-ce/test/helpers/config.ts
View File

@@ -6,6 +6,7 @@ export const STARTUP_TIMEOUT =
export function isExcludedBySharding(
shard:
| 'LOCAL_ONLY'
| 'CE_DEFAULT'
| 'CE_CUSTOM_1'
| 'CE_CUSTOM_2'
@@ -29,6 +30,7 @@ export function startWith({
varsFn = () => ({}),
withDataDir = false,
resetData = false,
mongoVersion = '',
}) {
before(async function () {
Object.assign(vars, varsFn())
@@ -38,14 +40,18 @@ export function startWith({
vars,
withDataDir,
resetData,
mongoVersion,
})
if (resetData) {
cy.log('resetting data and sessions')
resetCreatedUsersCache()
resetActivateUserRateLimit()
// no return here, always reconfigure when resetting data
} else if (previousConfigFrontend === cfg) {
cy.log(`already running with ${cfg}`)
return
}
cy.log(`starting with ${cfg}`)
this.timeout(STARTUP_TIMEOUT)
const { previousConfigServer } = await reconfigure({
@@ -54,6 +60,7 @@ export function startWith({
vars,
withDataDir,
resetData,
mongoVersion,
})
if (previousConfigServer !== cfg) {
await Cypress.session.clearAllSavedSessions()

33
server-ce/test/helpers/hostAdminClient.ts
View File

@@ -15,6 +15,7 @@ export async function reconfigure({
vars = {},
withDataDir = false,
resetData = false,
mongoVersion = '',
}): Promise<{ previousConfigServer: string }> {
return await fetchJSON(`${hostAdminURL}/reconfigure`, {
method: 'POST',
@@ -24,6 +25,7 @@ export async function reconfigure({
vars,
withDataDir,
resetData,
mongoVersion,
}),
})
}
@@ -63,10 +65,12 @@ export async function runScript({
cwd,
script,
args = [],
hasOverleafEnv = true,
}: {
cwd: string
script: string
args?: string[]
hasOverleafEnv?: boolean
}) {
return await fetchJSON(`${hostAdminURL}/run/script`, {
method: 'POST',
@@ -74,6 +78,23 @@ export async function runScript({
cwd,
script,
args,
hasOverleafEnv,
}),
})
}
export async function runGruntTask({
task,
args = [],
}: {
task: string
args?: string[]
}) {
return await fetchJSON(`${hostAdminURL}/run/gruntTask`, {
method: 'POST',
body: JSON.stringify({
task,
args,
}),
})
}
@@ -85,6 +106,18 @@ export async function getRedisKeys() {
return stdout.split('\n')
}
export async function setMongoFeatureCompatibilityVersion(
mongoVersion: string
) {
cy.log(`advancing mongo featureCompatibilityVersion to ${mongoVersion}`)
await fetchJSON(`${hostAdminURL}/mongo/setFeatureCompatibilityVersion`, {
method: 'POST',
body: JSON.stringify({
mongoVersion,
}),
})
}
export async function purgeFilestoreData() {
await fetchJSON(`${hostAdminURL}/data/user_files`, {
method: 'DELETE',

2
server-ce/test/helpers/login.ts
View File

@@ -1,6 +1,6 @@
import { runScript } from './hostAdminClient'
const DEFAULT_PASSWORD = 'Passw0rd!'
export const DEFAULT_PASSWORD = 'Passw0rd!'
const createdUsers = new Set<string>()

22
server-ce/test/helpers/project.ts
View File

@@ -217,6 +217,19 @@ export function createNewFile() {
return fileName
}
export function expectFileExists(
name: string,
binary: boolean,
content: string
) {
cy.findByRole('treeitem', { name }).click()
if (binary) {
cy.findByText(content).should('not.have.class', 'cm-line')
} else {
cy.findByText(content).should('have.class', 'cm-line')
}
}
export function prepareFileUploadTest(binary = false) {
const name = `${uuid()}.txt`
const content = `Test File Content ${name}${binary ? ' \x00' : ''}`
@@ -235,14 +248,7 @@ export function prepareFileUploadTest(binary = false) {
// wait for the upload to finish
cy.findByRole('treeitem', { name })
return function check() {
cy.findByRole('treeitem', { name }).click()
if (binary) {
cy.findByText(content).should('not.have.class', 'cm-line')
} else {
cy.findByText(content).should('have.class', 'cm-line')
}
}
return () => expectFileExists(name, binary, content)
}
export function testNewFileUpload() {

111
server-ce/test/host-admin.js
View File

@@ -107,20 +107,62 @@ app.post(
cwd: Joi.string().required(),
script: Joi.string().required(),
args: Joi.array().items(Joi.string()),
hasOverleafEnv: Joi.boolean().required(),
},
},
{ allowUnknown: false }
),
(req, res) => {
const { cwd, script, args } = req.body
const { cwd, script, args, hasOverleafEnv } = req.body
const env = hasOverleafEnv
? 'source /etc/overleaf/env.sh || source /etc/sharelatex/env.sh'
: 'true'
runDockerCompose(
'exec',
[
'--workdir',
`/overleaf/${cwd}`,
'sharelatex',
'bash',
'-c',
`source /etc/container_environment.sh && source /etc/overleaf/env.sh || source /etc/sharelatex/env.sh && cd ${JSON.stringify(cwd)} && node ${JSON.stringify(script)} ${args.map(a => JSON.stringify(a)).join(' ')}`,
`source /etc/container_environment.sh && ${env} && node ${JSON.stringify(script)} ${args.map(a => JSON.stringify(a)).join(' ')}`,
],
(error, stdout, stderr) => {
res.json({
error,
stdout,
stderr,
})
}
)
}
)
app.post(
'/run/gruntTask',
validate(
{
body: {
task: Joi.string().required(),
args: Joi.array().items(Joi.string()),
},
},
{ allowUnknown: false }
),
(req, res) => {
const { task, args } = req.body
runDockerCompose(
'exec',
[
'--workdir',
'/var/www/sharelatex',
'sharelatex',
'bash',
'-c',
`source /etc/container_environment.sh && grunt ${JSON.stringify(task)} ${args.map(a => JSON.stringify(a)).join(' ')}`,
],
(error, stdout, stderr) => {
res.json({
@@ -178,7 +220,13 @@ const allowedVars = Joi.object(
)
)
function setVarsDockerCompose({ pro, vars, version, withDataDir }) {
function setVarsDockerCompose({
pro,
vars,
version,
withDataDir,
mongoVersion,
}) {
const cfg = readDockerComposeOverride()
cfg.services.sharelatex.image = `${pro ? IMAGES.PRO : IMAGES.CE}:${version}`
@@ -198,8 +246,8 @@ function setVarsDockerCompose({ pro, vars, version, withDataDir }) {
const dataDirInContainer =
version === 'latest' || version >= '5.0'
? '/var/lib/overleaf/data'
: '/var/lib/sharelatex/data'
? '/var/lib/overleaf'
: '/var/lib/sharelatex'
cfg.services.sharelatex.volumes = []
if (withDataDir) {
@@ -220,11 +268,19 @@ function setVarsDockerCompose({ pro, vars, version, withDataDir }) {
)
if (!withDataDir) {
cfg.services.sharelatex.volumes.push(
`${PATHS.SANDBOXED_COMPILES_HOST_DIR}:${dataDirInContainer}/compiles`
`${PATHS.SANDBOXED_COMPILES_HOST_DIR}:${dataDirInContainer}/data/compiles`
)
}
}
if (mongoVersion) {
cfg.services.mongo = {
image: `mongo:${mongoVersion}`,
}
} else {
delete cfg.services.mongo
}
writeDockerComposeOverride(cfg)
}
@@ -285,6 +341,7 @@ app.post(
{
body: {
pro: Joi.boolean().required(),
mongoVersion: Joi.string().allow('').optional(),
version: Joi.string().required(),
vars: allowedVars,
withDataDir: Joi.boolean().optional(),
@@ -294,7 +351,8 @@ app.post(
{ allowUnknown: false }
),
(req, res) => {
const { pro, version, vars, withDataDir, resetData } = req.body
const { pro, version, vars, withDataDir, resetData, mongoVersion } =
req.body
maybeResetData(resetData, (error, stdout, stderr) => {
if (error) return res.json({ error, stdout, stderr })
@@ -305,7 +363,7 @@ app.post(
}
try {
setVarsDockerCompose({ pro, version, vars, withDataDir })
setVarsDockerCompose({ pro, version, vars, withDataDir, mongoVersion })
} catch (error) {
return res.json({ error })
}
@@ -323,6 +381,43 @@ app.post(
}
)
app.post(
'/mongo/setFeatureCompatibilityVersion',
validate(
{
body: {
mongoVersion: Joi.string().required(),
},
},
{ allowUnknown: false }
),
(req, res) => {
const { mongoVersion } = req.body
const mongosh = mongoVersion > '5' ? 'mongosh' : 'mongo'
const params = {
setFeatureCompatibilityVersion: mongoVersion,
}
if (mongoVersion >= '7.0') {
// MongoServerError: Once you have upgraded to 7.0, you will not be able to downgrade FCV and binary version without support assistance. Please re-run this command with 'confirm: true' to acknowledge this and continue with the FCV upgrade.
// NOTE: 6.0 does not know about this flag. So conditionally add it.
// MongoServerError: BSON field 'setFeatureCompatibilityVersion.confirm' is an unknown field.
params.confirm = true
}
runDockerCompose(
'exec',
[
'mongo',
mongosh,
'--eval',
`db.adminCommand(${JSON.stringify(params)})`,
],
(error, stdout, stderr) => {
res.json({ error, stdout, stderr })
}
)
}
)
app.get('/redis/keys', (req, res) => {
runDockerCompose(
'exec',