From 18cff6e1ac957932bf00dade88dd29e580595c2b Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 12 Jan 2026 12:28:16 +0000 Subject: [PATCH] [object-persistor] import ProjectKey helper from history-v1 (#30600) GitOrigin-RevId: c72aa4bf91569904a2072c74d6ed2f3c764d97bb --- .../object-persistor/src/ProjectKey.js | 1 - .../test/unit/ProjectKeyTests.js | 4 +--- services/filestore/app/js/KeyBuilder.js | 2 +- services/filestore/app/js/project_key.js | 20 ------------------- .../history-v1/storage/lib/backupArchiver.mjs | 2 +- .../history-v1/storage/lib/backupDeletion.mjs | 4 ++-- .../history-v1/storage/lib/backupVerifier.mjs | 2 +- .../storage/lib/blob_store/index.js | 2 +- .../history-v1/storage/lib/history_store.js | 2 +- services/history-v1/storage/lib/zip_store.js | 2 +- .../history-v1/storage/scripts/backup.mjs | 2 +- .../history-v1/storage/scripts/recover_zip.js | 2 +- services/history-v1/storage/scripts/show.mjs | 2 +- .../storage/tasks/copy_project_blobs.js | 2 +- .../acceptance/js/api/backupDeletion.test.mjs | 4 ++-- .../acceptance/js/api/backupVerifier.test.mjs | 2 +- .../acceptance/js/storage/backup.test.mjs | 2 +- 17 files changed, 17 insertions(+), 40 deletions(-) rename services/history-v1/storage/lib/project_key.js => libraries/object-persistor/src/ProjectKey.js (90%) rename services/history-v1/test/acceptance/js/storage/project_key.test.js => libraries/object-persistor/test/unit/ProjectKeyTests.js (88%) delete mode 100644 services/filestore/app/js/project_key.js diff --git a/services/history-v1/storage/lib/project_key.js b/libraries/object-persistor/src/ProjectKey.js similarity index 90% rename from services/history-v1/storage/lib/project_key.js rename to libraries/object-persistor/src/ProjectKey.js index cd559860d7..2ea0888ee3 100644 --- a/services/history-v1/storage/lib/project_key.js +++ b/libraries/object-persistor/src/ProjectKey.js @@ -1,4 +1,3 @@ -// Keep in sync with services/filestore/app/js/project_key.js const path = require('node:path') // diff --git a/services/history-v1/test/acceptance/js/storage/project_key.test.js b/libraries/object-persistor/test/unit/ProjectKeyTests.js similarity index 88% rename from services/history-v1/test/acceptance/js/storage/project_key.test.js rename to libraries/object-persistor/test/unit/ProjectKeyTests.js index 63cc5e0c9b..94763ad233 100644 --- a/services/history-v1/test/acceptance/js/storage/project_key.test.js +++ b/libraries/object-persistor/test/unit/ProjectKeyTests.js @@ -1,8 +1,6 @@ -'use strict' - const { expect } = require('chai') -const { format, pad } = require('../../../../storage/lib/project_key') +const { format, pad } = require('../../src/ProjectKey.js') describe('projectKey', function () { it('reverses padded keys', function () { diff --git a/services/filestore/app/js/KeyBuilder.js b/services/filestore/app/js/KeyBuilder.js index 6ab17938c7..c548508280 100644 --- a/services/filestore/app/js/KeyBuilder.js +++ b/services/filestore/app/js/KeyBuilder.js @@ -1,5 +1,5 @@ import settings from '@overleaf/settings' -import * as projectKey from './project_key.js' +import projectKey from '@overleaf/object-persistor/src/ProjectKey.js' export default { getConvertedFolderKey, diff --git a/services/filestore/app/js/project_key.js b/services/filestore/app/js/project_key.js deleted file mode 100644 index 574c0ccd3b..0000000000 --- a/services/filestore/app/js/project_key.js +++ /dev/null @@ -1,20 +0,0 @@ -// Keep in sync with services/history-v1/storage/lib/project_key.js -import path from 'node:path' - -// -// The advice in http://docs.aws.amazon.com/AmazonS3/latest/dev/ -// request-rate-perf-considerations.html is to avoid sequential key prefixes, -// so we reverse the project ID part of the key as they suggest. -// -export function format(projectId) { - const prefix = naiveReverse(pad(projectId)) - return path.join(prefix.slice(0, 3), prefix.slice(3, 6), prefix.slice(6)) -} - -export function pad(number) { - return (number || 0).toString().padStart(9, '0') -} - -function naiveReverse(string) { - return string.split('').reverse().join('') -} diff --git a/services/history-v1/storage/lib/backupArchiver.mjs b/services/history-v1/storage/lib/backupArchiver.mjs index 12ee9d8ab2..02bdbb6fd3 100644 --- a/services/history-v1/storage/lib/backupArchiver.mjs +++ b/services/history-v1/storage/lib/backupArchiver.mjs @@ -1,6 +1,6 @@ // @ts-check import path from 'node:path' -import projectKey from './project_key.js' +import projectKey from '@overleaf/object-persistor/src/ProjectKey.js' import { chunksBucket, backupPersistor, diff --git a/services/history-v1/storage/lib/backupDeletion.mjs b/services/history-v1/storage/lib/backupDeletion.mjs index ef50609753..402cd78c5f 100644 --- a/services/history-v1/storage/lib/backupDeletion.mjs +++ b/services/history-v1/storage/lib/backupDeletion.mjs @@ -1,10 +1,10 @@ // @ts-check -import { callbackify } from 'util' +import { callbackify } from 'node:util' import { ObjectId } from 'mongodb' import config from 'config' import OError from '@overleaf/o-error' import { db } from './mongodb.js' -import projectKey from './project_key.js' +import projectKey from '@overleaf/object-persistor/src/ProjectKey.js' import chunkStore from '../lib/chunk_store/index.js' import { backupPersistor, diff --git a/services/history-v1/storage/lib/backupVerifier.mjs b/services/history-v1/storage/lib/backupVerifier.mjs index 0b66149287..85f13a8c66 100644 --- a/services/history-v1/storage/lib/backupVerifier.mjs +++ b/services/history-v1/storage/lib/backupVerifier.mjs @@ -12,7 +12,7 @@ import blobHash from './blob_hash.js' import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js' import logger from '@overleaf/logger' import path from 'node:path' -import projectKey from './project_key.js' +import projectKey from '@overleaf/object-persistor/src/ProjectKey.js' import streams from './streams.js' import objectPersistor from '@overleaf/object-persistor' import { getEndDateForRPO } from '../../backupVerifier/utils.mjs' diff --git a/services/history-v1/storage/lib/blob_store/index.js b/services/history-v1/storage/lib/blob_store/index.js index 033e288554..ddc030cdc8 100644 --- a/services/history-v1/storage/lib/blob_store/index.js +++ b/services/history-v1/storage/lib/blob_store/index.js @@ -16,7 +16,7 @@ const assert = require('../assert') const blobHash = require('../blob_hash') const mongodb = require('../mongodb') const persistor = require('../persistor') -const projectKey = require('../project_key') +const projectKey = require('@overleaf/object-persistor/src/ProjectKey.js') const streams = require('../streams') const postgresBackend = require('./postgres') const mongoBackend = require('./mongo') diff --git a/services/history-v1/storage/lib/history_store.js b/services/history-v1/storage/lib/history_store.js index e51bdc25c5..9a219ab667 100644 --- a/services/history-v1/storage/lib/history_store.js +++ b/services/history-v1/storage/lib/history_store.js @@ -15,7 +15,7 @@ const logger = require('@overleaf/logger') const assert = require('./assert') const persistor = require('./persistor') -const projectKey = require('./project_key') +const projectKey = require('@overleaf/object-persistor/src/ProjectKey.js') const streams = require('./streams') const Chunk = core.Chunk diff --git a/services/history-v1/storage/lib/zip_store.js b/services/history-v1/storage/lib/zip_store.js index 0741829303..511bc146c1 100644 --- a/services/history-v1/storage/lib/zip_store.js +++ b/services/history-v1/storage/lib/zip_store.js @@ -12,7 +12,7 @@ const assert = require('./assert') const { BlobStore } = require('./blob_store') const persistor = require('./persistor') const ProjectArchive = require('./project_archive') -const projectKey = require('./project_key') +const projectKey = require('@overleaf/object-persistor/src/ProjectKey.js') const temp = require('./temp') const BUCKET = config.get('zipStore.bucket') diff --git a/services/history-v1/storage/scripts/backup.mjs b/services/history-v1/storage/scripts/backup.mjs index 974f6112c4..7a99a5a85f 100644 --- a/services/history-v1/storage/scripts/backup.mjs +++ b/services/history-v1/storage/scripts/backup.mjs @@ -40,7 +40,7 @@ import { backupGenerator } from '../lib/backupGenerator.mjs' import { promises as fs, createWriteStream } from 'node:fs' import os from 'node:os' import path from 'node:path' -import projectKey from '../lib/project_key.js' +import projectKey from '@overleaf/object-persistor/src/ProjectKey.js' import Crypto from 'node:crypto' import Stream from 'node:stream' import { EventEmitter } from 'node:events' diff --git a/services/history-v1/storage/scripts/recover_zip.js b/services/history-v1/storage/scripts/recover_zip.js index 91872c0fbd..a5703916f9 100644 --- a/services/history-v1/storage/scripts/recover_zip.js +++ b/services/history-v1/storage/scripts/recover_zip.js @@ -29,7 +29,7 @@ const { Storage } = require('@google-cloud/storage') const isValidUtf8 = require('utf-8-validate') const core = require('overleaf-editor-core') -const projectKey = require('../lib/project_key') +const projectKey = require('@overleaf/object-persistor/src/ProjectKey.js') const streams = require('../lib/streams') const ProjectArchive = require('../lib/project_archive') diff --git a/services/history-v1/storage/scripts/show.mjs b/services/history-v1/storage/scripts/show.mjs index 51697dc38f..5de63c65bd 100644 --- a/services/history-v1/storage/scripts/show.mjs +++ b/services/history-v1/storage/scripts/show.mjs @@ -23,7 +23,7 @@ import { pipeline } from 'node:stream/promises' import os from 'node:os' import path from 'node:path' import { createHash } from 'node:crypto' -import projectKey from '../lib/project_key.js' +import projectKey from '@overleaf/object-persistor/src/ProjectKey.js' import { createGunzip } from 'node:zlib' import { text } from 'node:stream/consumers' diff --git a/services/history-v1/storage/tasks/copy_project_blobs.js b/services/history-v1/storage/tasks/copy_project_blobs.js index c3511bf95c..952ff690e7 100755 --- a/services/history-v1/storage/tasks/copy_project_blobs.js +++ b/services/history-v1/storage/tasks/copy_project_blobs.js @@ -8,7 +8,7 @@ const fs = require('node:fs') const readline = require('node:readline') const { History } = require('overleaf-editor-core') const { knex, historyStore, persistor } = require('..') -const projectKey = require('../lib/project_key') +const projectKey = require('@overleaf/object-persistor/src/ProjectKey.js') const MAX_POSTGRES_INTEGER = 2147483647 const DEFAULT_BATCH_SIZE = 1000 diff --git a/services/history-v1/test/acceptance/js/api/backupDeletion.test.mjs b/services/history-v1/test/acceptance/js/api/backupDeletion.test.mjs index 1890404c4f..9c576ec714 100644 --- a/services/history-v1/test/acceptance/js/api/backupDeletion.test.mjs +++ b/services/history-v1/test/acceptance/js/api/backupDeletion.test.mjs @@ -13,8 +13,8 @@ import { } from '../../../../storage/lib/backupPersistor.mjs' import { makeProjectKey } from '../../../../storage/lib/blob_store/index.js' import config from 'config' -import Stream from 'stream' -import projectKey from '../../../../storage/lib/project_key.js' +import Stream from 'node:stream' +import projectKey from '@overleaf/object-persistor/src/ProjectKey.js' import { ListObjectsV2Command } from '@aws-sdk/client-s3' /** diff --git a/services/history-v1/test/acceptance/js/api/backupVerifier.test.mjs b/services/history-v1/test/acceptance/js/api/backupVerifier.test.mjs index c025a6dd10..e898620c5b 100644 --- a/services/history-v1/test/acceptance/js/api/backupVerifier.test.mjs +++ b/services/history-v1/test/acceptance/js/api/backupVerifier.test.mjs @@ -29,7 +29,7 @@ import { } from 'overleaf-editor-core' import Crypto from 'node:crypto' import path from 'node:path' -import projectKey from '../../../../storage/lib/project_key.js' +import projectKey from '@overleaf/object-persistor/src/ProjectKey.js' import { historyStore } from '../../../../storage/lib/history_store.js' /** diff --git a/services/history-v1/test/acceptance/js/storage/backup.test.mjs b/services/history-v1/test/acceptance/js/storage/backup.test.mjs index e1dc5c90ba..6c5f18c224 100644 --- a/services/history-v1/test/acceptance/js/storage/backup.test.mjs +++ b/services/history-v1/test/acceptance/js/storage/backup.test.mjs @@ -13,7 +13,7 @@ import { makeProjectKey, } from '../../../../storage/lib/blob_store/index.js' import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js' -import projectKey from '../../../../storage/lib/project_key.js' +import projectKey from '@overleaf/object-persistor/src/ProjectKey.js' import { getBackupStatus } from '../../../../storage/lib/backup_store/index.js' import { text, buffer } from 'node:stream/consumers' import { createGunzip } from 'node:zlib'