diff --git a/services/history-v1/storage/scripts/back_fill_file_hash_fix_up.mjs b/services/history-v1/storage/scripts/back_fill_file_hash_fix_up.mjs index 7bab794692..2525ee1d6e 100644 --- a/services/history-v1/storage/scripts/back_fill_file_hash_fix_up.mjs +++ b/services/history-v1/storage/scripts/back_fill_file_hash_fix_up.mjs @@ -9,15 +9,12 @@ import { Blob } from 'overleaf-editor-core' import { BlobStore, getStringLengthOfFile, - GLOBAL_BLOBS, makeBlobForFile, } from '../lib/blob_store/index.js' import { db } from '../lib/mongodb.js' import commandLineArgs from 'command-line-args' import readline from 'node:readline' -import { _blobIsBackedUp, backupBlob } from '../lib/backupBlob.mjs' import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js' -import filestorePersistor from '../lib/persistor.js' import { setTimeout } from 'node:timers/promises' // Silence warning. @@ -52,12 +49,11 @@ ObjectId.cacheHexString = true */ /** - * @return {{FIX_NOT_FOUND: boolean, FIX_HASH_MISMATCH: boolean, FIX_DELETE_PERMISSION: boolean, FIX_MISSING_HASH: boolean, LOGS: string}} + * @return {{FIX_NOT_FOUND: boolean, FIX_HASH_MISMATCH: boolean, FIX_MISSING_HASH: boolean, LOGS: string}} */ function parseArgs() { const args = commandLineArgs([ { name: 'fixNotFound', type: String, defaultValue: 'true' }, - { name: 'fixDeletePermission', type: String, defaultValue: 'true' }, { name: 'fixHashMismatch', type: String, defaultValue: 'true' }, { name: 'fixMissingHash', type: String, defaultValue: 'true' }, { name: 'logs', type: String, defaultValue: '' }, @@ -74,20 +70,13 @@ function parseArgs() { } return { FIX_HASH_MISMATCH: boolVal('fixNotFound'), - FIX_DELETE_PERMISSION: boolVal('fixDeletePermission'), FIX_NOT_FOUND: boolVal('fixHashMismatch'), FIX_MISSING_HASH: boolVal('fixMissingHash'), LOGS: args.logs, } } -const { - FIX_HASH_MISMATCH, - FIX_DELETE_PERMISSION, - FIX_NOT_FOUND, - FIX_MISSING_HASH, - LOGS, -} = parseArgs() +const { FIX_HASH_MISMATCH, FIX_NOT_FOUND, FIX_MISSING_HASH, LOGS } = parseArgs() if (!LOGS) { throw new Error('--logs parameter missing') } @@ -105,6 +94,37 @@ const STREAM_HIGH_WATER_MARK = parseInt( ) const SLEEP_BEFORE_EXIT = parseInt(process.env.SLEEP_BEFORE_EXIT || '1000', 10) +// Filestore endpoint location +const FILESTORE_HOST = process.env.FILESTORE_HOST || '127.0.0.1' +const FILESTORE_PORT = process.env.FILESTORE_PORT || '3009' + +async function fetchFromFilestore(projectId, fileId) { + const url = `http://${FILESTORE_HOST}:${FILESTORE_PORT}/project/${projectId}/file/${fileId}` + const response = await fetch(url) + if (!response.ok) { + if (response.status === 404) { + throw new NotFoundError('file not found in filestore', { + status: response.status, + }) + } + const body = await response.text() + throw new OError('fetchFromFilestore failed', { + projectId, + fileId, + status: response.status, + body, + }) + } + if (!response.body) { + throw new OError('fetchFromFilestore response has no body', { + projectId, + fileId, + status: response.status, + }) + } + return response.body +} + /** @type {ProjectsCollection} */ const projectsCollection = db.collection('projects') /** @type {DeletedProjectsCollection} */ @@ -302,19 +322,16 @@ async function setHashInMongo(projectId, fileId, hash) { * @return {Promise} */ async function importRestoredFilestoreFile(projectId, fileId, historyId) { - const filestoreKey = `${projectId}/${fileId}` const path = `${BUFFER_DIR}/${projectId}_${fileId}` try { let s try { - s = await filestorePersistor.getObjectStream( - USER_FILES_BUCKET_NAME, - filestoreKey - ) + s = await fetchFromFilestore(projectId, fileId) } catch (err) { if (err instanceof NotFoundError) { throw new OError('missing blob, need to restore filestore file', { - filestoreKey, + projectId, + fileId, }) } throw err @@ -325,7 +342,6 @@ async function importRestoredFilestoreFile(projectId, fileId, historyId) { ) const blobStore = new BlobStore(historyId) const blob = await blobStore.putFile(path) - await backupBlob(historyId, blob, path) await setHashInMongo(projectId, fileId, blob.getHash()) } finally { await fs.promises.rm(path, { force: true }) @@ -339,13 +355,9 @@ async function importRestoredFilestoreFile(projectId, fileId, historyId) { * @return {Promise} */ async function bufferFilestoreFileToDisk(projectId, fileId, path) { - const filestoreKey = `${projectId}/${fileId}` try { await Stream.promises.pipeline( - await filestorePersistor.getObjectStream( - USER_FILES_BUCKET_NAME, - filestoreKey - ), + await fetchFromFilestore(projectId, fileId), fs.createWriteStream(path, { highWaterMark: STREAM_HIGH_WATER_MARK }) ) const blob = await makeBlobForFile(path) @@ -356,7 +368,8 @@ async function bufferFilestoreFileToDisk(projectId, fileId, path) { } catch (err) { if (err instanceof NotFoundError) { throw new OError('missing blob, need to restore filestore file', { - filestoreKey, + projectId, + fileId, }) } throw err @@ -389,7 +402,7 @@ async function uploadFilestoreFile(projectId, fileId) { const blob = await bufferFilestoreFileToDisk(projectId, fileId, path) const hash = blob.getHash() try { - await ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) + await ensureBlobExistsForFile(projectId, fileId, hash) } catch (err) { if (!(err instanceof Blob.NotFoundError)) throw err @@ -397,7 +410,7 @@ async function uploadFilestoreFile(projectId, fileId) { const historyId = project.overleaf.history.id.toString() const blobStore = new BlobStore(historyId) await blobStore.putBlob(path, blob) - await ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) + await ensureBlobExistsForFile(projectId, fileId, hash) } } finally { await fs.promises.rm(path, { force: true }) @@ -426,11 +439,7 @@ async function fixHashMismatch(line) { await importRestoredFilestoreFile(projectId, fileId, historyId) return true } - return await ensureBlobExistsForFileAndUploadToAWS( - projectId, - fileId, - computedHash - ) + return await ensureBlobExistsForFile(projectId, fileId, computedHash) } /** @@ -444,30 +453,19 @@ async function hashAlreadyUpdatedInFileTree(projectId, fileId, hash) { return fileRef.hash === hash } -/** - * @param {string} projectId - * @param {string} hash - * @return {Promise} - */ -async function needsBackingUpToAWS(projectId, hash) { - if (GLOBAL_BLOBS.has(hash)) return false - return !(await _blobIsBackedUp(projectId, hash)) -} - /** * @param {string} projectId * @param {string} fileId * @param {string} hash * @return {Promise} */ -async function ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) { +async function ensureBlobExistsForFile(projectId, fileId, hash) { const { project } = await getProject(projectId) const historyId = project.overleaf.history.id.toString() const blobStore = new BlobStore(historyId) if ( (await hashAlreadyUpdatedInFileTree(projectId, fileId, hash)) && - (await blobStore.getBlob(hash)) && - !(await needsBackingUpToAWS(projectId, hash)) + (await blobStore.getBlob(hash)) ) { return false // already processed } @@ -488,7 +486,7 @@ async function ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) { ) if (writtenBlob.getHash() !== hash) { // Double check download, better safe than sorry. - throw new OError('blob corrupted', { writtenBlob }) + throw new OError('blob corrupted', { writtenBlob, hash }) } let blob = await blobStore.getBlob(hash) @@ -497,7 +495,6 @@ async function ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) { // HACK: Skip upload to GCS and finalize putBlob operation directly. await blobStore.backend.insertBlob(historyId, writtenBlob) } - await backupBlob(historyId, writtenBlob, path) } finally { await fs.promises.rm(path, { force: true }) } @@ -505,16 +502,6 @@ async function ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) { return true } -/** - * @param {string} line - * @return {Promise} - */ -async function fixDeletePermission(line) { - let { projectId, fileId, hash } = JSON.parse(line) - if (!hash) hash = await computeFilestoreFileHash(projectId, fileId) - return await ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) -} - /** * @param {string} line * @return {Promise} @@ -526,7 +513,7 @@ async function fixMissingHash(line) { } = await findFile(projectId, fileId) if (hash) { // processed, double check - return await ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) + return await ensureBlobExistsForFile(projectId, fileId, hash) } await uploadFilestoreFile(projectId, fileId) return true @@ -543,11 +530,6 @@ const CASES = { flag: FIX_HASH_MISMATCH, action: fixHashMismatch, }, - 'delete permission': { - match: 'storage.objects.delete', - flag: FIX_DELETE_PERMISSION, - action: fixDeletePermission, - }, 'missing file hash': { match: '"bad file hash"', flag: FIX_MISSING_HASH, diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs index 62b0b1de25..b6cdd4b9bf 100644 --- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs @@ -20,7 +20,7 @@ import { makeProjectKey, } from '../../../../storage/lib/blob_store/index.js' -import express from 'express' +import { mockFilestore } from './support/MockFilestore.mjs' chai.use(chaiExclude) const TIMEOUT = 20 * 1_000 @@ -28,59 +28,6 @@ const TIMEOUT = 20 * 1_000 const projectsCollection = db.collection('projects') const deletedProjectsCollection = db.collection('deletedProjects') -class MockFilestore { - constructor() { - this.host = process.env.FILESTORE_HOST || '127.0.0.1' - this.port = process.env.FILESTORE_PORT || 3009 - // create a server listening on this.host and this.port - this.files = {} - - this.app = express() - - this.app.get('/project/:projectId/file/:fileId', (req, res) => { - const { projectId, fileId } = req.params - const content = this.files[projectId]?.[fileId] - if (!content) return res.status(404).end() - res.status(200).end(content) - }) - } - - start() { - // reset stored files - this.files = {} - // start the server - if (this.serverPromise) { - return this.serverPromise - } else { - this.serverPromise = new Promise((resolve, reject) => { - this.server = this.app.listen(this.port, this.host, err => { - if (err) return reject(err) - resolve() - }) - }) - return this.serverPromise - } - } - - addFile(projectId, fileId, fileContent) { - if (!this.files[projectId]) { - this.files[projectId] = {} - } - this.files[projectId][fileId] = fileContent - } - - deleteObject(projectId, fileId) { - if (this.files[projectId]) { - delete this.files[projectId][fileId] - if (Object.keys(this.files[projectId]).length === 0) { - delete this.files[projectId] - } - } - } -} - -const mockFilestore = new MockFilestore() - /** * @param {ObjectId} objectId * @return {string} diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash_fix_up.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash_fix_up.test.mjs index ceafa24c3a..3aa00d685a 100644 --- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash_fix_up.test.mjs +++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash_fix_up.test.mjs @@ -1,48 +1,24 @@ import fs from 'node:fs' import Crypto from 'node:crypto' -import Stream from 'node:stream' import { promisify } from 'node:util' import { Binary, ObjectId } from 'mongodb' import { Blob } from 'overleaf-editor-core' -import { backedUpBlobs, blobs, db } from '../../../../storage/lib/mongodb.js' +import { db } from '../../../../storage/lib/mongodb.js' import cleanup from './support/cleanup.js' import testProjects from '../api/support/test_projects.js' import { execFile } from 'node:child_process' import chai, { expect } from 'chai' import chaiExclude from 'chai-exclude' -import config from 'config' -import { WritableBuffer } from '@overleaf/stream-utils' -import { - backupPersistor, - projectBlobsBucket, -} from '../../../../storage/lib/backupPersistor.mjs' -import projectKey from '../../../../storage/lib/project_key.js' -import { - BlobStore, - makeProjectKey, -} from '../../../../storage/lib/blob_store/index.js' -import ObjectPersistor from '@overleaf/object-persistor' +import { BlobStore } from '../../../../storage/lib/blob_store/index.js' +import { mockFilestore } from './support/MockFilestore.mjs' chai.use(chaiExclude) const TIMEOUT = 20 * 1_000 -const { deksBucket } = config.get('backupStore') -const { tieringStorageClass } = config.get('backupPersistor') - const projectsCollection = db.collection('projects') const deletedProjectsCollection = db.collection('deletedProjects') -const FILESTORE_PERSISTOR = ObjectPersistor({ - backend: 'gcs', - gcs: { - endpoint: { - apiEndpoint: process.env.GCS_API_ENDPOINT, - projectId: process.env.GCS_PROJECT_ID, - }, - }, -}) - /** * @param {ObjectId} objectId * @return {string} @@ -70,17 +46,6 @@ function binaryForGitBlobHash(gitBlobHash) { return new Binary(Buffer.from(gitBlobHash, 'hex')) } -async function listS3Bucket(bucket, wantStorageClass) { - const client = backupPersistor._getClientForBucket(bucket) - const response = await client.listObjectsV2({ Bucket: bucket }).promise() - - for (const object of response.Contents || []) { - expect(object).to.have.property('StorageClass', wantStorageClass) - } - - return (response.Contents || []).map(item => item.Key || '') -} - function objectIdFromTime(timestamp) { return ObjectId.createFromTime(new Date(timestamp).getTime() / 1000) } @@ -97,7 +62,6 @@ describe('back_fill_file_hash_fix_up script', function () { const historyIdDeleted0 = projectIdDeleted0.toString() const fileIdWithDifferentHashFound = objectIdFromTime('2017-02-01T00:00:00Z') const fileIdInGoodState = objectIdFromTime('2017-02-01T00:01:00Z') - const fileIdBlobExistsInGCS0 = objectIdFromTime('2017-02-01T00:02:00Z') const fileIdWithDifferentHashNotFound0 = objectIdFromTime( '2017-02-01T00:03:00Z' ) @@ -112,9 +76,6 @@ describe('back_fill_file_hash_fix_up script', function () { const fileIdWithDifferentHashRestore = objectIdFromTime( '2017-02-01T00:08:00Z' ) - const fileIdBlobExistsInGCS1 = objectIdFromTime('2017-02-01T00:09:00Z') - const fileIdRestoreFromFilestore0 = objectIdFromTime('2017-02-01T00:10:00Z') - const fileIdRestoreFromFilestore1 = objectIdFromTime('2017-02-01T00:11:00Z') const fileIdMissing2 = objectIdFromTime('2017-02-01T00:12:00Z') const fileIdHashMissing0 = objectIdFromTime('2017-02-01T00:13:00Z') const fileIdHashMissing1 = objectIdFromTime('2017-02-01T00:14:00Z') @@ -125,31 +86,11 @@ describe('back_fill_file_hash_fix_up script', function () { ) const deleteProjectsRecordId0 = new ObjectId() const writtenBlobs = [ - { - projectId: projectId0, - historyId: historyId0, - fileId: fileIdBlobExistsInGCS0, - }, - { - projectId: projectId0, - historyId: historyId0, - fileId: fileIdBlobExistsInGCS1, - }, { projectId: projectId0, historyId: historyId0, fileId: fileIdWithDifferentHashNotFound0, }, - { - projectId: projectId0, - historyId: historyId0, - fileId: fileIdRestoreFromFilestore0, - }, - { - projectId: projectId0, - historyId: historyId0, - fileId: fileIdRestoreFromFilestore1, - }, { projectId: projectId0, historyId: historyId0, @@ -200,17 +141,6 @@ describe('back_fill_file_hash_fix_up script', function () { }, msg: 'failed to process file', }, - { - projectId: projectId0, - fileId: fileIdRestoreFromFilestore0, - err: { message: 'OError: hash mismatch' }, - hash: gitBlobHash(fileIdRestoreFromFilestore0), - entry: { - ctx: { historyId: historyId0.toString() }, - hash: hashDoesNotExistAsBlob, - }, - msg: 'failed to process file', - }, { projectId: projectIdDeleted0, fileId: fileIdWithDifferentHashNotFound1, @@ -236,33 +166,6 @@ describe('back_fill_file_hash_fix_up script', function () { err: { message: 'NotFoundError' }, msg: 'failed to process file', }, - { - projectId: projectId0, - fileId: fileIdBlobExistsInGCS0, - hash: gitBlobHash(fileIdBlobExistsInGCS0), - err: { message: 'storage.objects.delete' }, - msg: 'failed to process file', - }, - { - projectId: projectId0, - fileId: fileIdBlobExistsInGCSCorrupted, - hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted), - err: { message: 'storage.objects.delete' }, - msg: 'failed to process file', - }, - { - projectId: projectId0, - fileId: fileIdBlobExistsInGCS1, - hash: gitBlobHash(fileIdBlobExistsInGCS1), - err: { message: 'storage.objects.delete' }, - msg: 'failed to process file', - }, - { - projectId: projectId0, - fileId: fileIdRestoreFromFilestore1, - err: { message: 'storage.objects.delete' }, - msg: 'failed to process file', - }, { projectId: projectIdDeleted0, fileId: fileIdMissing1, @@ -291,22 +194,23 @@ describe('back_fill_file_hash_fix_up script', function () { reason: 'bad file hash', msg: 'bad file-tree path', }, + { + projectId: projectId0, + _id: fileIdBlobExistsInGCSCorrupted, + reason: 'bad file hash', + msg: 'bad file-tree path', + }, ] if (PRINT_IDS_AND_HASHES_FOR_DEBUGGING) { const fileIds = { fileIdWithDifferentHashFound, fileIdInGoodState, - fileIdBlobExistsInGCS0, - fileIdBlobExistsInGCS1, fileIdWithDifferentHashNotFound0, fileIdWithDifferentHashNotFound1, - fileIdBlobExistsInGCSCorrupted, fileIdMissing0, fileIdMissing1, fileIdMissing2, fileIdWithDifferentHashRestore, - fileIdRestoreFromFilestore0, - fileIdRestoreFromFilestore1, fileIdHashMissing0, fileIdHashMissing1, } @@ -330,38 +234,25 @@ describe('back_fill_file_hash_fix_up script', function () { before(cleanup.everything) before('populate blobs/GCS', async function () { - await FILESTORE_PERSISTOR.sendStream( - USER_FILES_BUCKET_NAME, - `${projectId0}/${fileIdRestoreFromFilestore0}`, - Stream.Readable.from([fileIdRestoreFromFilestore0.toString()]) + await mockFilestore.start() + mockFilestore.addFile( + projectId0, + fileIdHashMissing0, + fileIdHashMissing0.toString() ) - await FILESTORE_PERSISTOR.sendStream( - USER_FILES_BUCKET_NAME, - `${projectId0}/${fileIdRestoreFromFilestore1}`, - Stream.Readable.from([fileIdRestoreFromFilestore1.toString()]) + mockFilestore.addFile( + projectId0, + fileIdHashMissing1, + fileIdHashMissing1.toString() ) - await FILESTORE_PERSISTOR.sendStream( - USER_FILES_BUCKET_NAME, - `${projectId0}/${fileIdHashMissing0}`, - Stream.Readable.from([fileIdHashMissing0.toString()]) - ) - await FILESTORE_PERSISTOR.sendStream( - USER_FILES_BUCKET_NAME, - `${projectId0}/${fileIdHashMissing1}`, - Stream.Readable.from([fileIdHashMissing1.toString()]) + mockFilestore.addFile( + projectId0, + fileIdBlobExistsInGCSCorrupted, + fileIdBlobExistsInGCSCorrupted.toString() ) await new BlobStore(historyId0.toString()).putString( fileIdHashMissing1.toString() // partially processed ) - await new BlobStore(historyId0.toString()).putString( - fileIdBlobExistsInGCS0.toString() - ) - await new BlobStore(historyId0.toString()).putString( - fileIdBlobExistsInGCS1.toString() - ) - await new BlobStore(historyId0.toString()).putString( - fileIdRestoreFromFilestore1.toString() - ) const path = '/tmp/test-blob-corrupted' try { await fs.promises.writeFile(path, contentCorruptedBlob) @@ -426,22 +317,10 @@ describe('back_fill_file_hash_fix_up script', function () { _id: fileIdWithDifferentHashNotFound0, hash: hashDoesNotExistAsBlob, }, - { - _id: fileIdRestoreFromFilestore0, - hash: hashDoesNotExistAsBlob, - }, - { - _id: fileIdRestoreFromFilestore1, - }, - { - _id: fileIdBlobExistsInGCS0, - hash: gitBlobHash(fileIdBlobExistsInGCS0), - }, { _id: fileIdBlobExistsInGCSCorrupted, hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted), }, - { _id: fileIdBlobExistsInGCS1 }, ], folders: [], }, @@ -546,8 +425,8 @@ describe('back_fill_file_hash_fix_up script', function () { }) it('should print stats', function () { expect(stats).to.contain({ - processedLines: 16, - success: 11, + processedLines: 12, + success: 7, alreadyProcessed: 0, fileDeleted: 0, skipped: 0, @@ -558,9 +437,9 @@ describe('back_fill_file_hash_fix_up script', function () { it('should handle re-run on same logs', async function () { ;({ stats } = await runScriptWithLogs()) expect(stats).to.contain({ - processedLines: 16, + processedLines: 12, success: 0, - alreadyProcessed: 8, + alreadyProcessed: 4, fileDeleted: 3, skipped: 0, failed: 3, @@ -663,31 +542,11 @@ describe('back_fill_file_hash_fix_up script', function () { _id: fileIdWithDifferentHashNotFound0, hash: gitBlobHash(fileIdWithDifferentHashNotFound0), }, - // Updated hash - { - _id: fileIdRestoreFromFilestore0, - hash: gitBlobHash(fileIdRestoreFromFilestore0), - }, - // Added hash - { - _id: fileIdRestoreFromFilestore1, - hash: gitBlobHash(fileIdRestoreFromFilestore1), - }, - // No change, blob created - { - _id: fileIdBlobExistsInGCS0, - hash: gitBlobHash(fileIdBlobExistsInGCS0), - }, // No change, flagged { _id: fileIdBlobExistsInGCSCorrupted, hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted), }, - // Added hash - { - _id: fileIdBlobExistsInGCS1, - hash: gitBlobHash(fileIdBlobExistsInGCS1), - }, ], folders: [], }, @@ -696,7 +555,7 @@ describe('back_fill_file_hash_fix_up script', function () { ], overleaf: { history: { id: historyId0 } }, // Incremented when removing file/updating hash - version: 8, + version: 5, }, ]) expect(await deletedProjectsCollection.find({}).toArray()).to.deep.equal([ @@ -745,62 +604,6 @@ describe('back_fill_file_hash_fix_up script', function () { (writtenBlobsByProject.get(projectId) || []).concat([fileId]) ) } - expect( - (await backedUpBlobs.find({}, { sort: { _id: 1 } }).toArray()).map( - entry => { - // blobs are pushed unordered into mongo. Sort the list for consistency. - entry.blobs.sort() - return entry - } - ) - ).to.deep.equal( - Array.from(writtenBlobsByProject.entries()).map( - ([projectId, fileIds]) => { - return { - _id: projectId, - blobs: fileIds - .map(fileId => binaryForGitBlobHash(gitBlobHash(fileId))) - .sort(), - } - } - ) - ) - }) - it('should have backed up all the files', async function () { - expect(tieringStorageClass).to.exist - const objects = await listS3Bucket(projectBlobsBucket, tieringStorageClass) - expect(objects.sort()).to.deep.equal( - writtenBlobs - .map(({ historyId, fileId, hash }) => - makeProjectKey(historyId, hash || gitBlobHash(fileId)) - ) - .sort() - ) - for (let { historyId, fileId } of writtenBlobs) { - const hash = gitBlobHash(fileId.toString()) - const s = await backupPersistor.getObjectStream( - projectBlobsBucket, - makeProjectKey(historyId, hash), - { autoGunzip: true } - ) - const buf = new WritableBuffer() - await Stream.promises.pipeline(s, buf) - expect(gitBlobHashBuffer(buf.getContents())).to.equal(hash) - const id = buf.getContents().toString('utf-8') - expect(id).to.equal(fileId.toString()) - // double check we are not comparing 'undefined' or '[object Object]' above - expect(id).to.match(/^[a-f0-9]{24}$/) - } - const deks = await listS3Bucket(deksBucket, 'STANDARD') - expect(deks.sort()).to.deep.equal( - Array.from( - new Set( - writtenBlobs.map( - ({ historyId }) => projectKey.format(historyId) + '/dek' - ) - ) - ).sort() - ) }) it('should have written the back filled files to history v1', async function () { for (const { historyId, fileId } of writtenBlobs) { diff --git a/services/history-v1/test/acceptance/js/storage/support/MockFilestore.mjs b/services/history-v1/test/acceptance/js/storage/support/MockFilestore.mjs new file mode 100644 index 0000000000..55d0923c34 --- /dev/null +++ b/services/history-v1/test/acceptance/js/storage/support/MockFilestore.mjs @@ -0,0 +1,54 @@ +import express from 'express' + +class MockFilestore { + constructor() { + this.host = process.env.FILESTORE_HOST || '127.0.0.1' + this.port = process.env.FILESTORE_PORT || 3009 + // create a server listening on this.host and this.port + this.files = {} + + this.app = express() + + this.app.get('/project/:projectId/file/:fileId', (req, res) => { + const { projectId, fileId } = req.params + const content = this.files[projectId]?.[fileId] + if (!content) return res.status(404).end() + res.status(200).end(content) + }) + } + + start() { + // reset stored files + this.files = {} + // start the server + if (this.serverPromise) { + return this.serverPromise + } else { + this.serverPromise = new Promise((resolve, reject) => { + this.server = this.app.listen(this.port, this.host, err => { + if (err) return reject(err) + resolve() + }) + }) + return this.serverPromise + } + } + + addFile(projectId, fileId, fileContent) { + if (!this.files[projectId]) { + this.files[projectId] = {} + } + this.files[projectId][fileId] = fileContent + } + + deleteObject(projectId, fileId) { + if (this.files[projectId]) { + delete this.files[projectId][fileId] + if (Object.keys(this.files[projectId]).length === 0) { + delete this.files[projectId] + } + } + } +} + +export const mockFilestore = new MockFilestore()