mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-11-14 22:23:42 +01:00
Create new module from overleaf/filestore persistors (#1)
* Create new module from overleaf/filestore persistors * Convert persistors to ES6 classes with local settings * Update README.md Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com> * Update README.md Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com> * Update .gitignore Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com> * Switch to AGPL license * Paginate S3 list-object results * Remove S3 client caching * Clean up S3 md5-verification mechanism * Update README for recent changes * Update README.md Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com> * Remove package-lock * Remove comment about FileHandler * Add directory marker to FSPersistor.deleteDirectory * Don't copy opts in GcsPersistor.getObjectStream * Use Date.now instead of getTime * Catch errors in migration persistor * Check that settings.buckets exists * Don't mutate options in ObserverStream constructor * Update src/PersistorHelper.js Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com> * Lint and format fixes Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com>
This commit is contained in:
Simon Detheridge
committed by
GitHub
GitHub
co-authored by
Eric Mc Sween
Eric Mc Sween
parent
b83ee27f83
commit
e92b75a2f8
@@ -0,0 +1,522 @@
|
||||
const sinon = require('sinon')
|
||||
const chai = require('chai')
|
||||
const { expect } = chai
|
||||
const modulePath = '../../src/MigrationPersistor.js'
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
|
||||
const Errors = require('../../src/Errors')
|
||||
|
||||
// Not all methods are tested here, but a method with each type of wrapping has
|
||||
// tests. Specifically, the following wrapping methods are tested here:
|
||||
// getObjectStream: _wrapFallbackMethod
|
||||
// sendStream: forward-to-primary
|
||||
// deleteObject: _wrapMethodOnBothPersistors
|
||||
// copyObject: copyFileWithFallback
|
||||
|
||||
describe('MigrationPersistorTests', function () {
|
||||
const bucket = 'womBucket'
|
||||
const fallbackBucket = 'bucKangaroo'
|
||||
const key = 'monKey'
|
||||
const destKey = 'donKey'
|
||||
const genericError = new Error('guru meditation error')
|
||||
const notFoundError = new Errors.NotFoundError('not found')
|
||||
const size = 33
|
||||
const md5 = 'ffffffff'
|
||||
|
||||
let Settings, Logger, Stream, MigrationPersistor, fileStream, newPersistor
|
||||
|
||||
beforeEach(function () {
|
||||
fileStream = {
|
||||
name: 'fileStream',
|
||||
on: sinon.stub().withArgs('end').yields(),
|
||||
pipe: sinon.stub()
|
||||
}
|
||||
|
||||
newPersistor = function (hasFile) {
|
||||
return {
|
||||
sendFile: sinon.stub().resolves(),
|
||||
sendStream: sinon.stub().resolves(),
|
||||
getObjectStream: hasFile
|
||||
? sinon.stub().resolves(fileStream)
|
||||
: sinon.stub().rejects(notFoundError),
|
||||
deleteDirectory: sinon.stub().resolves(),
|
||||
getObjectSize: hasFile
|
||||
? sinon.stub().resolves(size)
|
||||
: sinon.stub().rejects(notFoundError),
|
||||
deleteObject: sinon.stub().resolves(),
|
||||
copyObject: hasFile
|
||||
? sinon.stub().resolves()
|
||||
: sinon.stub().rejects(notFoundError),
|
||||
checkIfObjectExists: sinon.stub().resolves(hasFile),
|
||||
directorySize: hasFile
|
||||
? sinon.stub().resolves(size)
|
||||
: sinon.stub().rejects(notFoundError),
|
||||
getObjectMd5Hash: hasFile
|
||||
? sinon.stub().resolves(md5)
|
||||
: sinon.stub().rejects(notFoundError)
|
||||
}
|
||||
}
|
||||
|
||||
Settings = {
|
||||
buckets: {
|
||||
[bucket]: fallbackBucket
|
||||
}
|
||||
}
|
||||
|
||||
Stream = {
|
||||
pipeline: sinon.stub().yields(),
|
||||
PassThrough: sinon.stub()
|
||||
}
|
||||
|
||||
Logger = {
|
||||
warn: sinon.stub()
|
||||
}
|
||||
|
||||
MigrationPersistor = SandboxedModule.require(modulePath, {
|
||||
requires: {
|
||||
stream: Stream,
|
||||
'./Errors': Errors,
|
||||
'logger-sharelatex': Logger
|
||||
},
|
||||
globals: { console }
|
||||
})
|
||||
})
|
||||
|
||||
describe('getObjectStream', function () {
|
||||
const options = { wombat: 'potato' }
|
||||
describe('when the primary persistor has the file', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor, response
|
||||
beforeEach(async function () {
|
||||
primaryPersistor = newPersistor(true)
|
||||
fallbackPersistor = newPersistor(false)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
response = await migrationPersistor.getObjectStream(
|
||||
bucket,
|
||||
key,
|
||||
options
|
||||
)
|
||||
})
|
||||
|
||||
it('should return the file stream', function () {
|
||||
expect(response).to.equal(fileStream)
|
||||
})
|
||||
|
||||
it('should fetch the file from the primary persistor, with the correct options', function () {
|
||||
expect(primaryPersistor.getObjectStream).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key,
|
||||
options
|
||||
)
|
||||
})
|
||||
|
||||
it('should not query the fallback persistor', function () {
|
||||
expect(fallbackPersistor.getObjectStream).not.to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the fallback persistor has the file', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor, response
|
||||
beforeEach(async function () {
|
||||
primaryPersistor = newPersistor(false)
|
||||
fallbackPersistor = newPersistor(true)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
response = await migrationPersistor.getObjectStream(
|
||||
bucket,
|
||||
key,
|
||||
options
|
||||
)
|
||||
})
|
||||
|
||||
it('should return the file stream', function () {
|
||||
expect(response).to.be.an.instanceOf(Stream.PassThrough)
|
||||
})
|
||||
|
||||
it('should fetch the file from the primary persistor with the correct options', function () {
|
||||
expect(primaryPersistor.getObjectStream).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key,
|
||||
options
|
||||
)
|
||||
})
|
||||
|
||||
it('should fetch the file from the fallback persistor with the fallback bucket with the correct options', function () {
|
||||
expect(
|
||||
fallbackPersistor.getObjectStream
|
||||
).to.have.been.calledWithExactly(fallbackBucket, key, options)
|
||||
})
|
||||
|
||||
it('should create one read stream', function () {
|
||||
expect(fallbackPersistor.getObjectStream).to.have.been.calledOnce
|
||||
})
|
||||
|
||||
it('should not send the file to the primary', function () {
|
||||
expect(primaryPersistor.sendStream).not.to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the file should be copied to the primary', function () {
|
||||
let primaryPersistor,
|
||||
fallbackPersistor,
|
||||
migrationPersistor,
|
||||
returnedStream
|
||||
beforeEach(async function () {
|
||||
primaryPersistor = newPersistor(false)
|
||||
fallbackPersistor = newPersistor(true)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
Settings.copyOnMiss = true
|
||||
returnedStream = await migrationPersistor.getObjectStream(
|
||||
bucket,
|
||||
key,
|
||||
options
|
||||
)
|
||||
})
|
||||
|
||||
it('should create one read stream', function () {
|
||||
expect(fallbackPersistor.getObjectStream).to.have.been.calledOnce
|
||||
})
|
||||
|
||||
it('should get the md5 hash from the source', function () {
|
||||
expect(fallbackPersistor.getObjectMd5Hash).to.have.been.calledWith(
|
||||
fallbackBucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
|
||||
it('should send a stream to the primary', function () {
|
||||
expect(primaryPersistor.sendStream).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key,
|
||||
sinon.match.instanceOf(Stream.PassThrough),
|
||||
md5
|
||||
)
|
||||
})
|
||||
|
||||
it('should send a stream to the client', function () {
|
||||
expect(returnedStream).to.be.an.instanceOf(Stream.PassThrough)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when neither persistor has the file', function () {
|
||||
it('rejects with a NotFoundError', async function () {
|
||||
const migrationPersistor = new MigrationPersistor(
|
||||
newPersistor(false),
|
||||
newPersistor(false),
|
||||
Settings
|
||||
)
|
||||
return expect(
|
||||
migrationPersistor.getObjectStream(bucket, key)
|
||||
).to.eventually.be.rejected.and.be.an.instanceOf(Errors.NotFoundError)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the primary persistor throws an unexpected error', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor, error
|
||||
beforeEach(async function () {
|
||||
primaryPersistor = newPersistor(false)
|
||||
fallbackPersistor = newPersistor(true)
|
||||
primaryPersistor.getObjectStream = sinon.stub().rejects(genericError)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
try {
|
||||
await migrationPersistor.getObjectStream(bucket, key, options)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects with the error', function () {
|
||||
expect(error).to.equal(genericError)
|
||||
})
|
||||
|
||||
it('does not call the fallback', function () {
|
||||
expect(fallbackPersistor.getObjectStream).not.to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the fallback persistor throws an unexpected error', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor, error
|
||||
beforeEach(async function () {
|
||||
primaryPersistor = newPersistor(false)
|
||||
fallbackPersistor = newPersistor(false)
|
||||
fallbackPersistor.getObjectStream = sinon.stub().rejects(genericError)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
try {
|
||||
await migrationPersistor.getObjectStream(bucket, key, options)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects with the error', function () {
|
||||
expect(error).to.equal(genericError)
|
||||
})
|
||||
|
||||
it('should have called the fallback', function () {
|
||||
expect(fallbackPersistor.getObjectStream).to.have.been.calledWith(
|
||||
fallbackBucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('sendStream', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor
|
||||
beforeEach(function () {
|
||||
primaryPersistor = newPersistor(false)
|
||||
fallbackPersistor = newPersistor(false)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
})
|
||||
|
||||
describe('when it works', function () {
|
||||
beforeEach(async function () {
|
||||
return migrationPersistor.sendStream(bucket, key, fileStream)
|
||||
})
|
||||
|
||||
it('should send the file to the primary persistor', function () {
|
||||
expect(primaryPersistor.sendStream).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key,
|
||||
fileStream
|
||||
)
|
||||
})
|
||||
|
||||
it('should not send the file to the fallback persistor', function () {
|
||||
expect(fallbackPersistor.sendStream).not.to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the primary persistor throws an error', function () {
|
||||
it('returns the error', async function () {
|
||||
primaryPersistor.sendStream.rejects(notFoundError)
|
||||
return expect(
|
||||
migrationPersistor.sendStream(bucket, key, fileStream)
|
||||
).to.eventually.be.rejected.and.be.an.instanceOf(Errors.NotFoundError)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('deleteObject', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor
|
||||
beforeEach(function () {
|
||||
primaryPersistor = newPersistor(false)
|
||||
fallbackPersistor = newPersistor(false)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
})
|
||||
|
||||
describe('when it works', function () {
|
||||
beforeEach(async function () {
|
||||
return migrationPersistor.deleteObject(bucket, key)
|
||||
})
|
||||
|
||||
it('should delete the file from the primary', function () {
|
||||
expect(primaryPersistor.deleteObject).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
|
||||
it('should delete the file from the fallback', function () {
|
||||
expect(fallbackPersistor.deleteObject).to.have.been.calledWithExactly(
|
||||
fallbackBucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the primary persistor throws an error', function () {
|
||||
let error
|
||||
beforeEach(async function () {
|
||||
primaryPersistor.deleteObject.rejects(genericError)
|
||||
try {
|
||||
await migrationPersistor.deleteObject(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should return the error', function () {
|
||||
expect(error).to.equal(genericError)
|
||||
})
|
||||
|
||||
it('should delete the file from the primary', function () {
|
||||
expect(primaryPersistor.deleteObject).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
|
||||
it('should delete the file from the fallback', function () {
|
||||
expect(fallbackPersistor.deleteObject).to.have.been.calledWithExactly(
|
||||
fallbackBucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the fallback persistor throws an error', function () {
|
||||
let error
|
||||
beforeEach(async function () {
|
||||
fallbackPersistor.deleteObject.rejects(genericError)
|
||||
try {
|
||||
await migrationPersistor.deleteObject(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should return the error', function () {
|
||||
expect(error).to.equal(genericError)
|
||||
})
|
||||
|
||||
it('should delete the file from the primary', function () {
|
||||
expect(primaryPersistor.deleteObject).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
|
||||
it('should delete the file from the fallback', function () {
|
||||
expect(fallbackPersistor.deleteObject).to.have.been.calledWithExactly(
|
||||
fallbackBucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('copyObject', function () {
|
||||
describe('when the file exists on the primary', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor
|
||||
beforeEach(async function () {
|
||||
primaryPersistor = newPersistor(true)
|
||||
fallbackPersistor = newPersistor(false)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
return migrationPersistor.copyObject(bucket, key, destKey)
|
||||
})
|
||||
|
||||
it('should call copyObject to copy the file', function () {
|
||||
expect(primaryPersistor.copyObject).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key,
|
||||
destKey
|
||||
)
|
||||
})
|
||||
|
||||
it('should not try to read from the fallback', function () {
|
||||
expect(fallbackPersistor.getObjectStream).not.to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the file does not exist on the primary', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor
|
||||
beforeEach(async function () {
|
||||
primaryPersistor = newPersistor(false)
|
||||
fallbackPersistor = newPersistor(true)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
return migrationPersistor.copyObject(bucket, key, destKey)
|
||||
})
|
||||
|
||||
it('should call copyObject to copy the file', function () {
|
||||
expect(primaryPersistor.copyObject).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key,
|
||||
destKey
|
||||
)
|
||||
})
|
||||
|
||||
it('should fetch the file from the fallback', function () {
|
||||
expect(
|
||||
fallbackPersistor.getObjectStream
|
||||
).not.to.have.been.calledWithExactly(fallbackBucket, key)
|
||||
})
|
||||
|
||||
it('should get the md5 hash from the source', function () {
|
||||
expect(fallbackPersistor.getObjectMd5Hash).to.have.been.calledWith(
|
||||
fallbackBucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
|
||||
it('should send the file to the primary', function () {
|
||||
expect(primaryPersistor.sendStream).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
destKey,
|
||||
sinon.match.instanceOf(Stream.PassThrough),
|
||||
md5
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the file does not exist on the fallback', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor, error
|
||||
beforeEach(async function () {
|
||||
primaryPersistor = newPersistor(false)
|
||||
fallbackPersistor = newPersistor(false)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
try {
|
||||
await migrationPersistor.copyObject(bucket, key, destKey)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should call copyObject to copy the file', function () {
|
||||
expect(primaryPersistor.copyObject).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key,
|
||||
destKey
|
||||
)
|
||||
})
|
||||
|
||||
it('should fetch the file from the fallback', function () {
|
||||
expect(
|
||||
fallbackPersistor.getObjectStream
|
||||
).not.to.have.been.calledWithExactly(fallbackBucket, key)
|
||||
})
|
||||
|
||||
it('should return a not-found error', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.NotFoundError)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user