agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-11-12 13:13:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #28554 from overleaf/msm-force-s3-lib-storage-uploads

Browse Source 
[object-persistor] Use `@aws-sdk/lib-storage` for all uploads

GitOrigin-RevId: ab8e54a7bae843f9e6b05ed9cf936130a36b8c2f
This commit is contained in:
Miguel Serrano
2025-10-08 08:05:55 +00:00
committed by Copybot
parent 4a5b29d166
commit 3a35b8680e
17 changed files with 159 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/history-v1/docker-compose.ci.yml
+17 -4
View File
@@ -41,6 +41,7 @@ services:
POSTGRES_HOST: postgres
AWS_S3_ENDPOINT: https://minio:9000
AWS_S3_PATH_STYLE: 'true'
DELETE_OBJECTS_MD5_FALLBACK: true
AWS_ACCESS_KEY_ID: OVERLEAF_HISTORY_S3_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: OVERLEAF_HISTORY_S3_SECRET_ACCESS_KEY
MINIO_ROOT_USER: MINIO_ROOT_USER
@@ -188,7 +189,10 @@ services:
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:DeleteObject"
"s3:DeleteObject",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts",
"s3:ListBucketMultipartUploads"
],
"Resource": "arn:aws:s3:::overleaf-test-history-chunks/*"
},
@@ -204,7 +208,10 @@ services:
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:DeleteObject"
"s3:DeleteObject",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts",
"s3:ListBucketMultipartUploads"
],
"Resource": "arn:aws:s3:::overleaf-test-history-deks/*"
},
@@ -220,7 +227,10 @@ services:
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:DeleteObject"
"s3:DeleteObject",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts",
"s3:ListBucketMultipartUploads"
],
"Resource": "arn:aws:s3:::overleaf-test-history-global-blobs/*"
},
@@ -236,7 +246,10 @@ services:
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:DeleteObject"
"s3:DeleteObject",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts",
"s3:ListBucketMultipartUploads"
],
"Resource": "arn:aws:s3:::overleaf-test-history-project-blobs/*"
}
services/history-v1/docker-compose.yml
+17 -4
View File
@@ -58,6 +58,7 @@ services:
POSTGRES_HOST: postgres
AWS_S3_ENDPOINT: https://minio:9000
AWS_S3_PATH_STYLE: 'true'
DELETE_OBJECTS_MD5_FALLBACK: true
AWS_ACCESS_KEY_ID: OVERLEAF_HISTORY_S3_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: OVERLEAF_HISTORY_S3_SECRET_ACCESS_KEY
MINIO_ROOT_USER: MINIO_ROOT_USER
@@ -196,7 +197,10 @@ services:
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:DeleteObject"
"s3:DeleteObject",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts",
"s3:ListBucketMultipartUploads"
],
"Resource": "arn:aws:s3:::overleaf-test-history-chunks/*"
},
@@ -212,7 +216,10 @@ services:
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:DeleteObject"
"s3:DeleteObject",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts",
"s3:ListBucketMultipartUploads"
],
"Resource": "arn:aws:s3:::overleaf-test-history-deks/*"
},
@@ -228,7 +235,10 @@ services:
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:DeleteObject"
"s3:DeleteObject",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts",
"s3:ListBucketMultipartUploads"
],
"Resource": "arn:aws:s3:::overleaf-test-history-global-blobs/*"
},
@@ -244,7 +254,10 @@ services:
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:DeleteObject"
"s3:DeleteObject",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts",
"s3:ListBucketMultipartUploads"
],
"Resource": "arn:aws:s3:::overleaf-test-history-project-blobs/*"
}
services/history-v1/storage/lib/backupBlob.mjs
-7
View File
@@ -70,7 +70,6 @@ export async function downloadBlobToDir(historyId, blob, tmpDir) {
* @return {Promise<void>}
*/
export async function uploadBlobToBackup(historyId, blob, path, persistor) {
const md5 = Crypto.createHash('md5')
const filePathCompressed = path + '.gz'
let backupSource
let contentEncoding
@@ -86,7 +85,6 @@ export async function uploadBlobToBackup(historyId, blob, path, persistor) {
async function* (source) {
for await (const chunk of source) {
size += chunk.byteLength
md5.update(chunk)
yield chunk
}
},
@@ -97,10 +95,6 @@ export async function uploadBlobToBackup(historyId, blob, path, persistor) {
} else {
backupSource = path
size = blob.getByteLength()
await Stream.promises.pipeline(
fs.createReadStream(path, { highWaterMark: HIGHWATER_MARK }),
md5
)
}
const key = makeProjectKey(historyId, blob.getHash())
await persistor.sendStream(
@@ -111,7 +105,6 @@ export async function uploadBlobToBackup(historyId, blob, path, persistor) {
contentEncoding,
contentType: 'application/octet-stream',
contentLength: size,
sourceMd5: md5.digest('hex'),
ifNoneMatch: '*',
}
)
services/history-v1/storage/lib/backupPersistor.mjs
-1
View File
@@ -104,7 +104,6 @@ async function getRootKeyEncryptionKeys() {
export const backupPersistor = new PerProjectEncryptedS3Persistor({
...persistorConfig.s3SSEC,
disableMultiPartUpload: true,
dataEncryptionKeyBucketName: deksBucket,
pathToProjectFolder,
getRootKeyEncryptionKeys,
services/history-v1/storage/scripts/backup.mjs
-2
View File
@@ -221,7 +221,6 @@ async function backupChunk(
}
const key = makeChunkKey(historyId, chunkToBackup.startVersion)
logger.debug({ chunkRecord, historyId, projectId, key }, 'backing up chunk')
const md5 = Crypto.createHash('md5').update(chunkBuffer)
await chunkBackupPersistorForProject.sendStream(
chunksBucket,
makeChunkKey(historyId, chunkToBackup.startVersion),
@@ -230,7 +229,6 @@ async function backupChunk(
contentType: 'application/json',
contentEncoding: 'gzip',
contentLength: chunkBuffer.byteLength,
sourceMd5: md5.digest('hex'),
}
)
}
services/history-v1/test/acceptance/js/api/backupVerifier.test.mjs
-2
View File
@@ -131,7 +131,6 @@ async function backupChunk(historyId) {
historyId,
newChunkMetadata.id
)
const md5 = Crypto.createHash('md5').update(chunkBuffer)
await backupPersistor.sendStream(
chunksBucket,
path.join(
@@ -143,7 +142,6 @@ async function backupChunk(historyId) {
contentType: 'application/json',
contentEncoding: 'gzip',
contentLength: chunkBuffer.byteLength,
sourceMd5: md5.digest('hex'),
}
)
}