mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-11-15 06:33:55 +01:00
[web] download binary files in clsi from filestore via new endpoints (#27505)
* [history-v1] use String.padStart instead of lodash.padStart * [web] download binary files in clsi from filestore via new endpoints * [server-ce] tests: Cypress.env() is parsing boolean values * [server-ce] tests: run history migration as root GitOrigin-RevId: bdf6c0e542531ccc4b3f13d2ed68ca0d31e580e9
This commit is contained in:
Jakob Ackermann
@@ -75,6 +75,7 @@ services:
|
||||
environment:
|
||||
CYPRESS_SHARD:
|
||||
CYPRESS_BASE_URL: http://sharelatex
|
||||
CYPRESS_FULL_FILESTORE_MIGRATION:
|
||||
SPEC_PATTERN: '**/*.spec.{js,jsx,ts,tsx}'
|
||||
depends_on:
|
||||
sharelatex:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { DEFAULT_PASSWORD, login } from './helpers/login'
|
||||
import { DEFAULT_PASSWORD, ensureUserExists, login } from './helpers/login'
|
||||
import {
|
||||
createProject,
|
||||
expectFileExists,
|
||||
openProjectById,
|
||||
prepareFileUploadTest,
|
||||
@@ -35,7 +36,8 @@ describe('filestore migration', function () {
|
||||
SHARELATEX_MONGO_URL: 'mongodb://mongo/sharelatex',
|
||||
SHARELATEX_REDIS_HOST: 'redis',
|
||||
}
|
||||
let projectName: string
|
||||
const projectName = `project-${uuid()}`
|
||||
let defaultImage: string
|
||||
let projectId: string
|
||||
let waitForCompileRateLimitCoolOff: (fn: () => void) => void
|
||||
const previousBinaryFiles: (() => void)[] = []
|
||||
@@ -49,7 +51,7 @@ describe('filestore migration', function () {
|
||||
}
|
||||
|
||||
function addNewBinaryFileAndCheckPrevious(
|
||||
universeSelector = 'img[alt="universe.jpg"]'
|
||||
universeSelector = `img[alt="${defaultImage}"]`
|
||||
) {
|
||||
before(function () {
|
||||
login(email)
|
||||
@@ -61,7 +63,7 @@ describe('filestore migration', function () {
|
||||
for (const check of previousBinaryFiles) {
|
||||
check()
|
||||
}
|
||||
cy.findByRole('treeitem', { name: 'universe.jpg' }).click()
|
||||
cy.findByRole('treeitem', { name: defaultImage }).click()
|
||||
cy.get(universeSelector)
|
||||
.should('be.visible')
|
||||
.and('have.prop', 'naturalWidth')
|
||||
@@ -71,184 +73,223 @@ describe('filestore migration', function () {
|
||||
})
|
||||
}
|
||||
|
||||
// --------------
|
||||
// Server Pro 1.x
|
||||
startWith({
|
||||
pro: true,
|
||||
resetData: true,
|
||||
withDataDir: true,
|
||||
vars: sharelatexBrandedVars,
|
||||
version: '1.2.4',
|
||||
mongoVersion: '5.0',
|
||||
})
|
||||
|
||||
let activateURL: string
|
||||
before(async function () {
|
||||
const { stdout } = await runGruntTask({
|
||||
task: 'user:create-admin',
|
||||
args: ['--email', email],
|
||||
if (Cypress.env('FULL_FILESTORE_MIGRATION')) {
|
||||
// --------------
|
||||
// Server Pro 1.x
|
||||
startWith({
|
||||
pro: true,
|
||||
resetData: true,
|
||||
withDataDir: true,
|
||||
vars: sharelatexBrandedVars,
|
||||
version: '1.2.4',
|
||||
mongoVersion: '5.0',
|
||||
})
|
||||
;[activateURL] = stdout.match(
|
||||
/http:\/\/.+\/user\/password\/set\?passwordResetToken=\S+/
|
||||
)!
|
||||
})
|
||||
before(function () {
|
||||
activateUserVersion1x(activateURL)
|
||||
login(email)
|
||||
projectName = `project-${uuid()}`
|
||||
cy.visit('/project')
|
||||
defaultImage = 'universe.jpg'
|
||||
|
||||
// Legacy angular based UI uses links instead of buttons
|
||||
cy.findByRole('link', {
|
||||
name: /Create First Project|New Project/,
|
||||
}).click()
|
||||
cy.findByRole('link', { name: 'Example Project' }).click()
|
||||
cy.findByPlaceholderText('Project Name').type(projectName)
|
||||
cy.findByRole('button', { name: 'Create' }).click()
|
||||
cy.url()
|
||||
.should('match', /\/project\/[a-fA-F0-9]{24}/)
|
||||
.then(url => (projectId = url.split('/').pop()!))
|
||||
let queueReset
|
||||
;({ waitForCompileRateLimitCoolOff, queueReset } =
|
||||
prepareWaitForNextCompileSlot())
|
||||
queueReset()
|
||||
let activateURL: string
|
||||
before(async function () {
|
||||
const { stdout } = await runGruntTask({
|
||||
task: 'user:create-admin',
|
||||
args: ['--email', email],
|
||||
})
|
||||
;[activateURL] = stdout.match(
|
||||
/http:\/\/.+\/user\/password\/set\?passwordResetToken=\S+/
|
||||
)!
|
||||
})
|
||||
before(function () {
|
||||
activateUserVersion1x(activateURL)
|
||||
login(email)
|
||||
cy.visit('/project')
|
||||
|
||||
// Create a new binary file
|
||||
cy.get(`a[tooltip="Upload"]`).click()
|
||||
const name = `${uuid()}.txt`
|
||||
// Binary file detection is not sophisticated in version 1.x
|
||||
const binName = name.replace('.txt', '.bin')
|
||||
const content = `Test File Content ${name} \x00`
|
||||
cy.get('input[type=file]')
|
||||
.first()
|
||||
.selectFile(
|
||||
// Legacy angular based UI uses links instead of buttons
|
||||
cy.findByRole('link', {
|
||||
name: /Create First Project|New Project/,
|
||||
}).click()
|
||||
cy.findByRole('link', { name: 'Example Project' }).click()
|
||||
cy.findByPlaceholderText('Project Name').type(projectName)
|
||||
cy.findByRole('button', { name: 'Create' }).click()
|
||||
cy.url()
|
||||
.should('match', /\/project\/[a-fA-F0-9]{24}/)
|
||||
.then(url => (projectId = url.split('/').pop()!))
|
||||
let queueReset
|
||||
;({ waitForCompileRateLimitCoolOff, queueReset } =
|
||||
prepareWaitForNextCompileSlot())
|
||||
queueReset()
|
||||
|
||||
// Create a new binary file
|
||||
cy.get(`a[tooltip="Upload"]`).click()
|
||||
const name = `${uuid()}.txt`
|
||||
// Binary file detection is not sophisticated in version 1.x
|
||||
const binName = name.replace('.txt', '.bin')
|
||||
const content = `Test File Content ${name} \x00`
|
||||
cy.get('input[type=file]')
|
||||
.first()
|
||||
.selectFile(
|
||||
{
|
||||
contents: Cypress.Buffer.from(content),
|
||||
fileName: binName,
|
||||
lastModified: Date.now(),
|
||||
},
|
||||
{ force: true }
|
||||
)
|
||||
// Rename back to .txt to enable preview
|
||||
cy.findByText(binName).click()
|
||||
cy.findByText(binName).dblclick()
|
||||
cy.focused().type(name + '{del}'.repeat('.bin'.length) + '{enter}')
|
||||
// Switch back and forth
|
||||
cy.findByText('universe.jpg').click()
|
||||
cy.findByText(name).click()
|
||||
cy.findByText(content)
|
||||
.parent()
|
||||
.parent()
|
||||
.should('have.class', 'text-preview')
|
||||
|
||||
previousBinaryFiles.push(() => expectFileExists(name, true, content))
|
||||
avoid502()
|
||||
})
|
||||
|
||||
// --------------
|
||||
// Server Pro 2.x
|
||||
startWith({
|
||||
pro: true,
|
||||
withDataDir: true,
|
||||
vars: sharelatexBrandedVars,
|
||||
version: '2.7.1',
|
||||
mongoVersion: '5.0',
|
||||
})
|
||||
before(function () {
|
||||
// Cypress strips the Content-Length header: https://github.com/cypress-io/cypress/issues/16469
|
||||
// Server Pro 2.x does not gracefully handle a missing value.
|
||||
cy.intercept(
|
||||
{
|
||||
contents: Cypress.Buffer.from(content),
|
||||
fileName: binName,
|
||||
lastModified: Date.now(),
|
||||
method: 'HEAD',
|
||||
url: `http://sharelatex/project/${projectId}/file/*`,
|
||||
times: previousBinaryFiles.length + 1,
|
||||
},
|
||||
{ force: true }
|
||||
req => {
|
||||
req.continue(res => {
|
||||
res.headers['Content-Length'] = '60'
|
||||
})
|
||||
}
|
||||
)
|
||||
// Rename back to .txt to enable preview
|
||||
cy.findByText(binName).click()
|
||||
cy.findByText(binName).dblclick()
|
||||
cy.focused().type(name + '{del}'.repeat('.bin'.length) + '{enter}')
|
||||
// Switch back and forth
|
||||
cy.findByText('universe.jpg').click()
|
||||
cy.findByText(name).click()
|
||||
cy.findByText(content)
|
||||
.parent()
|
||||
.parent()
|
||||
.should('have.class', 'text-preview')
|
||||
})
|
||||
// Server Pro 2.x does not have alt tags on images.
|
||||
addNewBinaryFileAndCheckPrevious('img')
|
||||
|
||||
previousBinaryFiles.push(() => expectFileExists(name, true, content))
|
||||
avoid502()
|
||||
})
|
||||
// ----------------------------------
|
||||
// Server Pro 3.x + history migration
|
||||
startWith({
|
||||
pro: true,
|
||||
withDataDir: true,
|
||||
vars: sharelatexBrandedVars,
|
||||
version: '3.5.13',
|
||||
mongoVersion: '5.0',
|
||||
})
|
||||
addNewBinaryFileAndCheckPrevious() // before history migration
|
||||
before(async function () {
|
||||
await runScript({
|
||||
cwd: 'services/web',
|
||||
script: 'scripts/history/migrate_history.js',
|
||||
args: [
|
||||
'--force-clean',
|
||||
'--fix-invalid-characters',
|
||||
'--convert-large-docs-to-file',
|
||||
],
|
||||
hasOverleafEnv: false,
|
||||
user: 'root',
|
||||
})
|
||||
})
|
||||
before(async function () {
|
||||
await runScript({
|
||||
cwd: 'services/web',
|
||||
script: 'scripts/history/clean_sl_history_data.js',
|
||||
hasOverleafEnv: false,
|
||||
})
|
||||
})
|
||||
addNewBinaryFileAndCheckPrevious() // after history migration
|
||||
|
||||
// --------------
|
||||
// Server Pro 2.x
|
||||
startWith({
|
||||
pro: true,
|
||||
withDataDir: true,
|
||||
vars: sharelatexBrandedVars,
|
||||
version: '2.7.1',
|
||||
mongoVersion: '5.0',
|
||||
})
|
||||
before(function () {
|
||||
// Cypress strips the Content-Length header: https://github.com/cypress-io/cypress/issues/16469
|
||||
// Server Pro 2.x does not gracefully handle a missing value.
|
||||
cy.intercept(
|
||||
{
|
||||
method: 'HEAD',
|
||||
url: `http://sharelatex/project/${projectId}/file/*`,
|
||||
times: previousBinaryFiles.length + 1,
|
||||
},
|
||||
req => {
|
||||
req.continue(res => {
|
||||
res.headers['Content-Length'] = '60'
|
||||
})
|
||||
// ------------------------------
|
||||
// Server Pro 4.x + mongo upgrade
|
||||
startWith({
|
||||
pro: true,
|
||||
withDataDir: true,
|
||||
vars: sharelatexBrandedVars,
|
||||
version: '4.2.9',
|
||||
mongoVersion: '5.0',
|
||||
})
|
||||
startWith({
|
||||
pro: true,
|
||||
withDataDir: true,
|
||||
vars: sharelatexBrandedVars,
|
||||
version: '4.2.9',
|
||||
mongoVersion: '6.0',
|
||||
})
|
||||
before(async function () {
|
||||
await setMongoFeatureCompatibilityVersion('6.0')
|
||||
})
|
||||
addNewBinaryFileAndCheckPrevious()
|
||||
|
||||
// ------------------------------------------
|
||||
// Server Pro 5.x + mongo upgrade 6 -> 7 -> 8
|
||||
startWith({
|
||||
pro: true,
|
||||
withDataDir: true,
|
||||
mongoVersion: '6.0',
|
||||
})
|
||||
startWith({
|
||||
pro: true,
|
||||
withDataDir: true,
|
||||
mongoVersion: '7.0',
|
||||
})
|
||||
before(async function () {
|
||||
await setMongoFeatureCompatibilityVersion('7.0')
|
||||
})
|
||||
startWith({
|
||||
pro: true,
|
||||
withDataDir: true,
|
||||
// implicit mongo upgrade to 8.0
|
||||
})
|
||||
before(async function () {
|
||||
await setMongoFeatureCompatibilityVersion('8.0')
|
||||
})
|
||||
} else {
|
||||
// 5.x
|
||||
startWith({ pro: true, withDataDir: true })
|
||||
defaultImage = 'frog.jpg'
|
||||
ensureUserExists({ email })
|
||||
before(function () {
|
||||
login(email)
|
||||
createProject(projectName, { type: 'Example project', open: false }).then(
|
||||
id => (projectId = id)
|
||||
)
|
||||
;({ waitForCompileRateLimitCoolOff } = prepareWaitForNextCompileSlot())
|
||||
})
|
||||
}
|
||||
addNewBinaryFileAndCheckPrevious()
|
||||
|
||||
function ensureStopOnFirstErrorIsActive() {
|
||||
cy.findByRole('button', { name: 'Toggle compile options menu' }).click()
|
||||
cy.findByRole('menuitem', {
|
||||
name: 'Stop on first error',
|
||||
}).then(el => {
|
||||
// NOTE: THIS IS BAD, but the selected option is otherwise not accessible :/
|
||||
if (
|
||||
el.get()[0]?.querySelector('.material-symbol')?.textContent !== 'check'
|
||||
) {
|
||||
cy.findByRole('menuitem', {
|
||||
name: 'Stop on first error',
|
||||
}).click()
|
||||
// Clicking on "Stop on first error" closes the mode. Open it again.
|
||||
cy.findByRole('button', { name: 'Toggle compile options menu' }).click()
|
||||
}
|
||||
)
|
||||
})
|
||||
// Server Pro 2.x does not have alt tags on images.
|
||||
addNewBinaryFileAndCheckPrevious('img')
|
||||
|
||||
// ----------------------------------
|
||||
// Server Pro 3.x + history migration
|
||||
startWith({
|
||||
pro: true,
|
||||
withDataDir: true,
|
||||
vars: sharelatexBrandedVars,
|
||||
version: '3.5.13',
|
||||
mongoVersion: '5.0',
|
||||
})
|
||||
addNewBinaryFileAndCheckPrevious() // before history migration
|
||||
before(async function () {
|
||||
await runScript({
|
||||
cwd: 'services/web',
|
||||
script: 'scripts/history/migrate_history.js',
|
||||
args: [
|
||||
'--force-clean',
|
||||
'--fix-invalid-characters',
|
||||
'--convert-large-docs-to-file',
|
||||
],
|
||||
hasOverleafEnv: false,
|
||||
})
|
||||
})
|
||||
before(async function () {
|
||||
await runScript({
|
||||
cwd: 'services/web',
|
||||
script: 'scripts/history/clean_sl_history_data.js',
|
||||
hasOverleafEnv: false,
|
||||
cy.findByRole('menuitem', {
|
||||
name: 'Stop on first error',
|
||||
}).within(() => {
|
||||
cy.findByText('check').should('be.visible')
|
||||
})
|
||||
})
|
||||
addNewBinaryFileAndCheckPrevious() // after history migration
|
||||
|
||||
// ------------------------------
|
||||
// Server Pro 4.x + mongo upgrade
|
||||
startWith({
|
||||
pro: true,
|
||||
withDataDir: true,
|
||||
vars: sharelatexBrandedVars,
|
||||
version: '4.2.9',
|
||||
mongoVersion: '5.0',
|
||||
})
|
||||
startWith({
|
||||
pro: true,
|
||||
withDataDir: true,
|
||||
vars: sharelatexBrandedVars,
|
||||
version: '4.2.9',
|
||||
mongoVersion: '6.0',
|
||||
})
|
||||
before(async function () {
|
||||
await setMongoFeatureCompatibilityVersion('6.0')
|
||||
})
|
||||
addNewBinaryFileAndCheckPrevious()
|
||||
|
||||
// ------------------------------------------
|
||||
// Server Pro 5.x + mongo upgrade 6 -> 7 -> 8
|
||||
startWith({
|
||||
pro: true,
|
||||
withDataDir: true,
|
||||
mongoVersion: '6.0',
|
||||
})
|
||||
startWith({
|
||||
pro: true,
|
||||
withDataDir: true,
|
||||
mongoVersion: '7.0',
|
||||
})
|
||||
before(async function () {
|
||||
await setMongoFeatureCompatibilityVersion('7.0')
|
||||
})
|
||||
startWith({
|
||||
pro: true,
|
||||
withDataDir: true,
|
||||
// implicit mongo upgrade to 8.0
|
||||
})
|
||||
before(async function () {
|
||||
await setMongoFeatureCompatibilityVersion('8.0')
|
||||
})
|
||||
addNewBinaryFileAndCheckPrevious()
|
||||
cy.findByRole('button', { name: 'Toggle compile options menu' }).click()
|
||||
}
|
||||
|
||||
// -------------------
|
||||
// filestore-migration
|
||||
@@ -257,6 +298,7 @@ describe('filestore migration', function () {
|
||||
waitForCompileRateLimitCoolOff(() => {
|
||||
openProjectById(projectId)
|
||||
})
|
||||
ensureStopOnFirstErrorIsActive()
|
||||
})
|
||||
|
||||
function checkFilesAreAccessible() {
|
||||
@@ -267,13 +309,31 @@ describe('filestore migration', function () {
|
||||
}
|
||||
})
|
||||
|
||||
it('renders universe jpg', () => {
|
||||
cy.findByTestId('file-tree').findByText('universe.jpg').click()
|
||||
cy.get('[alt="universe.jpg"]')
|
||||
it('renders image of example project', () => {
|
||||
cy.findByTestId('file-tree').findByText(defaultImage).click()
|
||||
cy.get(`[alt="${defaultImage}"]`)
|
||||
.should('be.visible')
|
||||
.and('have.prop', 'naturalWidth')
|
||||
.should('be.greaterThan', 0)
|
||||
})
|
||||
|
||||
it('can recompile from scratch', function () {
|
||||
const id = uuid()
|
||||
cy.findByText('\\maketitle').parent().click()
|
||||
cy.findByText('\\maketitle')
|
||||
.parent()
|
||||
.type(`\n\\section{{}Test Section ${id}}`)
|
||||
|
||||
waitForCompileRateLimitCoolOff(() => {
|
||||
cy.findByRole('button', { name: 'Toggle compile options menu' }).click()
|
||||
|
||||
cy.findByRole('menuitem', {
|
||||
name: 'Recompile from scratch',
|
||||
}).trigger('click')
|
||||
})
|
||||
|
||||
cy.get('.pdf-viewer').should('contain.text', `Test Section ${id}`)
|
||||
})
|
||||
}
|
||||
|
||||
describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL not set', function () {
|
||||
|
||||
@@ -65,11 +65,13 @@ export async function runScript({
|
||||
cwd,
|
||||
script,
|
||||
args = [],
|
||||
user = 'www-data',
|
||||
hasOverleafEnv = true,
|
||||
}: {
|
||||
cwd: string
|
||||
script: string
|
||||
args?: string[]
|
||||
user?: string
|
||||
hasOverleafEnv?: boolean
|
||||
}) {
|
||||
return await fetchJSON(`${hostAdminURL}/run/script`, {
|
||||
@@ -78,6 +80,7 @@ export async function runScript({
|
||||
cwd,
|
||||
script,
|
||||
args,
|
||||
user,
|
||||
hasOverleafEnv,
|
||||
}),
|
||||
})
|
||||
|
||||
@@ -107,13 +107,14 @@ app.post(
|
||||
cwd: Joi.string().required(),
|
||||
script: Joi.string().required(),
|
||||
args: Joi.array().items(Joi.string()),
|
||||
user: Joi.string().required(),
|
||||
hasOverleafEnv: Joi.boolean().required(),
|
||||
},
|
||||
},
|
||||
{ allowUnknown: false }
|
||||
),
|
||||
(req, res) => {
|
||||
const { cwd, script, args, hasOverleafEnv } = req.body
|
||||
const { cwd, script, args, user, hasOverleafEnv } = req.body
|
||||
|
||||
const env = hasOverleafEnv
|
||||
? 'source /etc/overleaf/env.sh || source /etc/sharelatex/env.sh'
|
||||
@@ -127,7 +128,7 @@ app.post(
|
||||
'sharelatex',
|
||||
'bash',
|
||||
'-c',
|
||||
`source /etc/container_environment.sh && ${env} && /sbin/setuser www-data node ${JSON.stringify(script)} ${args.map(a => JSON.stringify(a)).join(' ')}`,
|
||||
`source /etc/container_environment.sh && ${env} && /sbin/setuser ${user} node ${script} ${args.map(a => JSON.stringify(a)).join(' ')}`,
|
||||
],
|
||||
(error, stdout, stderr) => {
|
||||
res.json({
|
||||
|
||||
Reference in New Issue
Block a user