agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-11-14 22:23:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #12575 from overleaf/em-fix-chunk-replace-transaction

Browse Source 
Fix history chunk replacement transaction

GitOrigin-RevId: fc0fa8f4a55ae5f402ba87db5e4ac2856fe6627b
This commit is contained in:
Eric Mc Sween
2023-04-19 08:03:10 +00:00
committed by Copybot
parent 924012e21f
commit cff54eae78
49 changed files with 236 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/web/scripts/back_fill_dummy_doc_meta.js
+7 -3
View File
@@ -1,6 +1,10 @@
const { promisify } = require('util')
const { ObjectId, ReadPreference } = require('mongodb')
const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
const { ObjectId } = require('mongodb')
const {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
const sleep = promisify(setTimeout)
const _ = require('lodash')
@@ -64,7 +68,7 @@ async function main(options) {
},
}
const docs = await db.docs
.find(query, { readPreference: ReadPreference.SECONDARY })
.find(query, { readPreference: READ_PREFERENCE_SECONDARY })
.project({ _id: 1, project_id: 1 })
.limit(options.batchSize)
.toArray()
services/web/scripts/back_fill_staff_access.js
+6 -3
View File
@@ -1,5 +1,8 @@
const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
const { ReadPreference } = require('mongodb')
const {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
const UserSessionsManager = require('../app/src/Features/User/UserSessionsManager')
const COMMIT = process.argv.includes('--commit')
@@ -46,7 +49,7 @@ async function main() {
email: 1,
staffAccess: 1,
},
readPreference: ReadPreference.SECONDARY,
readPreference: READ_PREFERENCE_SECONDARY,
}
)
.toArray()
services/web/scripts/count_encrypted_access_tokens.js
+6 -3
View File
@@ -2,8 +2,11 @@ const TEN_MINUTES = 1000 * 60 * 10
process.env.MONGO_SOCKET_TIMEOUT =
process.env.MONGO_SOCKET_TIMEOUT || TEN_MINUTES.toString()
const { ReadPreference } = require('mongodb')
const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
const {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
const _ = require('lodash')
const {
formatTokenUsageStats,
@@ -32,7 +35,7 @@ async function count(collectionName, paths) {
const cursor = collection.find(
{},
{
readPreference: ReadPreference.SECONDARY,
readPreference: READ_PREFERENCE_SECONDARY,
projection,
}
)
services/web/scripts/count_image_files.js
+9 -3
View File
@@ -1,5 +1,8 @@
const { ReadPreference } = require('mongodb')
const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
const {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
const { extname } = require('node:path')
const FILE_TYPES = [
@@ -22,7 +25,10 @@ async function main() {
await waitForDb()
const projects = db.projects.find(
{},
{ projection: { rootFolder: 1 }, readPreference: ReadPreference.SECONDARY }
{
projection: { rootFolder: 1 },
readPreference: READ_PREFERENCE_SECONDARY,
}
)
let projectsProcessed = 0
const result = new Map(FILE_TYPES.map(fileType => [fileType, 0]))
services/web/scripts/delete_orphaned_data_helper.js
+7 -4
View File
@@ -1,5 +1,8 @@
const { ReadPreference } = require('mongodb')
const { db } = require('../app/src/infrastructure/mongodb')
const {
db,
READ_PREFERENCE_PRIMARY,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
const { promiseMapWithLimit } = require('../app/src/util/promises')
async function getDeletedProject(projectId, readPreference) {
@@ -57,14 +60,14 @@ async function checkProjectExistsWithReadPreference(projectId, readPreference) {
async function checkProjectExistsOnPrimary(projectId) {
return await checkProjectExistsWithReadPreference(
projectId,
ReadPreference.PRIMARY
READ_PREFERENCE_PRIMARY
)
}
async function checkProjectExistsOnSecondary(projectId) {
return await checkProjectExistsWithReadPreference(
projectId,
ReadPreference.SECONDARY
READ_PREFERENCE_SECONDARY
)
}
services/web/scripts/delete_orphaned_docs_online_check.js
+9 -4
View File
@@ -1,7 +1,12 @@
const DocstoreManager = require('../app/src/Features/Docstore/DocstoreManager')
const { promisify } = require('util')
const { ObjectId, ReadPreference } = require('mongodb')
const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
const { ObjectId } = require('mongodb')
const {
db,
waitForDb,
READ_PREFERENCE_PRIMARY,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
const { promiseMapWithLimit } = require('../app/src/util/promises')
const { getHardDeletedProjectIds } = require('./delete_orphaned_data_helper')
const sleep = promisify(setTimeout)
@@ -52,7 +57,7 @@ async function main() {
},
}
const docs = await db.docs
.find(query, { readPreference: ReadPreference.SECONDARY })
.find(query, { readPreference: READ_PREFERENCE_SECONDARY })
.project({ project_id: 1 })
.sort({ project_id: 1 })
.limit(BATCH_SIZE)
@@ -97,7 +102,7 @@ async function getProjectDocs(projectId) {
{ project_id: projectId },
{
projection: { _id: 1 },
readPreference: ReadPreference.PRIMARY,
readPreference: READ_PREFERENCE_PRIMARY,
}
)
.toArray()
services/web/scripts/find_malformed_filetrees.js
+9 -3
View File
@@ -1,11 +1,17 @@
const { ReadPreference } = require('mongodb')
const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
const {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
async function main() {
await waitForDb()
const projects = db.projects.find(
{},
{ projection: { rootFolder: 1 }, readPreference: ReadPreference.SECONDARY }
{
projection: { rootFolder: 1 },
readPreference: READ_PREFERENCE_SECONDARY,
}
)
let projectsProcessed = 0
for await (const project of projects) {
services/web/scripts/helpers/batchedUpdate.js
+7 -3
View File
@@ -1,5 +1,9 @@
const { ReadPreference, ObjectId } = require('mongodb')
const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
const { ObjectId } = require('mongodb')
const {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../../app/src/infrastructure/mongodb')
const ONE_MONTH_IN_MS = 1000 * 60 * 60 * 24 * 31
let ID_EDGE_PAST
@@ -129,7 +133,7 @@ async function batchedUpdate(
refreshGlobalOptionsForBatchedUpdate(batchedUpdateOptions)
findOptions = findOptions || {}
findOptions.readPreference = ReadPreference.SECONDARY
findOptions.readPreference = READ_PREFERENCE_SECONDARY
projection = projection || { _id: 1 }
let nextBatch
services/web/scripts/history/downgrade_project.js
+7 -3
View File
@@ -5,8 +5,12 @@ process.env.MONGO_SOCKET_TIMEOUT =
const PROJECT_ID = process.env.PROJECT_ID
const { ReadPreference, ObjectId } = require('mongodb')
const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
const { ObjectId } = require('mongodb')
const {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../../app/src/infrastructure/mongodb')
const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler')
console.log({
@@ -43,7 +47,7 @@ async function shouldPreserveHistory(project) {
{ preserveHistory: { $eq: true } },
],
},
{ readPreference: ReadPreference.SECONDARY }
{ readPreference: READ_PREFERENCE_SECONDARY }
)
}
services/web/scripts/history/unset_allow_downgrade.js
+7 -3
View File
@@ -1,6 +1,10 @@
const { promisify } = require('util')
const { ObjectId, ReadPreference } = require('mongodb')
const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
const { ObjectId } = require('mongodb')
const {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../../app/src/infrastructure/mongodb')
const sleep = promisify(setTimeout)
const _ = require('lodash')
@@ -63,7 +67,7 @@ async function main(options) {
'overleaf.history.allowDowngrade': true,
}
const projects = await db.projects
.find(query, { readPreference: ReadPreference.SECONDARY })
.find(query, { readPreference: READ_PREFERENCE_SECONDARY })
.project({ _id: 1 })
.limit(options.batchSize)
.toArray()
services/web/scripts/history/upgrade_none_without_conversion_if_no_sl_history.js
+6 -4
View File
@@ -14,8 +14,10 @@ process.env.BATCH_SIZE = BATCH_SIZE
process.env.MONGO_SOCKET_TIMEOUT =
parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
const { ReadPreference } = require('mongodb')
const { db } = require('../../app/src/infrastructure/mongodb')
const {
db,
READ_PREFERENCE_SECONDARY,
} = require('../../app/src/infrastructure/mongodb')
const { promiseMapWithLimit } = require('../../app/src/util/promises')
const { batchedUpdate } = require('../helpers/batchedUpdate')
const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler')
@@ -172,7 +174,7 @@ async function anyDocHistoryExists(project) {
{ project_id: { $eq: project._id } },
{
projection: { _id: 1 },
readPreference: ReadPreference.SECONDARY,
readPreference: READ_PREFERENCE_SECONDARY,
}
)
}
@@ -182,7 +184,7 @@ async function anyDocHistoryIndexExists(project) {
{ project_id: { $eq: project._id } },
{
projection: { _id: 1 },
readPreference: ReadPreference.SECONDARY,
readPreference: READ_PREFERENCE_SECONDARY,
}
)
}
services/web/scripts/history/upgrade_project.js
+7 -3
View File
@@ -1,5 +1,9 @@
const { ReadPreference, ObjectId } = require('mongodb')
const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
const { ObjectId } = require('mongodb')
const {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../../app/src/infrastructure/mongodb')
const {
upgradeProject,
} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
@@ -20,7 +24,7 @@ async function main() {
}
const options = {
projection,
readPreference: ReadPreference.SECONDARY,
readPreference: READ_PREFERENCE_SECONDARY,
}
const project = await db.projects.findOne(query, options)
if (project) {
services/web/scripts/history/upgrade_v1_with_conversion_if_sl_history.js
+9 -5
View File
@@ -16,8 +16,12 @@ process.env.MONGO_SOCKET_TIMEOUT =
const PROJECT_ID = process.env.PROJECT_ID
const { ReadPreference, ObjectId } = require('mongodb')
const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
const { ObjectId } = require('mongodb')
const {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../../app/src/infrastructure/mongodb')
const { promiseMapWithLimit } = require('../../app/src/util/promises')
const { batchedUpdate } = require('../helpers/batchedUpdate')
const ProjectHistoryController = require('../../modules/history-migration/app/src/ProjectHistoryController')
@@ -165,7 +169,7 @@ async function shouldPreserveHistory(project) {
{ preserveHistory: { $eq: true } },
],
},
{ readPreference: ReadPreference.SECONDARY }
{ readPreference: READ_PREFERENCE_SECONDARY }
)
}
@@ -174,7 +178,7 @@ async function anyDocHistoryExists(project) {
{ project_id: { $eq: project._id } },
{
projection: { _id: 1 },
readPreference: ReadPreference.SECONDARY,
readPreference: READ_PREFERENCE_SECONDARY,
}
)
}
@@ -184,7 +188,7 @@ async function anyDocHistoryIndexExists(project) {
{ project_id: { $eq: project._id } },
{
projection: { _id: 1 },
readPreference: ReadPreference.SECONDARY,
readPreference: READ_PREFERENCE_SECONDARY,
}
)
}
services/web/scripts/history/upgrade_v1_without_conversion_if_created_after_fph_enabled.js
+7 -4
View File
@@ -10,8 +10,11 @@ process.env.BATCH_SIZE = BATCH_SIZE
process.env.MONGO_SOCKET_TIMEOUT =
parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
const { ReadPreference, ObjectId } = require('mongodb')
const { db } = require('../../app/src/infrastructure/mongodb')
const { ObjectId } = require('mongodb')
const {
db,
READ_PREFERENCE_SECONDARY,
} = require('../../app/src/infrastructure/mongodb')
const { promiseMapWithLimit } = require('../../app/src/util/promises')
const { batchedUpdate } = require('../helpers/batchedUpdate')
@@ -111,7 +114,7 @@ async function anyDocHistoryExists(project) {
{ project_id: { $eq: project._id } },
{
projection: { _id: 1 },
readPreference: ReadPreference.SECONDARY,
readPreference: READ_PREFERENCE_SECONDARY,
}
)
}
@@ -121,7 +124,7 @@ async function anyDocHistoryIndexExists(project) {
{ project_id: { $eq: project._id } },
{
projection: { _id: 1 },
readPreference: ReadPreference.SECONDARY,
readPreference: READ_PREFERENCE_SECONDARY,
}
)
}
services/web/scripts/history/upgrade_v1_without_conversion_if_no_sl_history.js
+7 -5
View File
@@ -12,8 +12,10 @@ process.env.BATCH_SIZE = BATCH_SIZE
process.env.MONGO_SOCKET_TIMEOUT =
parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
const { ReadPreference } = require('mongodb')
const { db } = require('../../app/src/infrastructure/mongodb')
const {
db,
READ_PREFERENCE_SECONDARY,
} = require('../../app/src/infrastructure/mongodb')
const { promiseMapWithLimit } = require('../../app/src/util/promises')
const { batchedUpdate } = require('../helpers/batchedUpdate')
@@ -99,7 +101,7 @@ async function shouldPreserveHistory(project) {
{ preserveHistory: { $eq: true } },
],
},
{ readPreference: ReadPreference.SECONDARY }
{ readPreference: READ_PREFERENCE_SECONDARY }
)
}
@@ -108,7 +110,7 @@ async function anyDocHistoryExists(project) {
{ project_id: { $eq: project._id } },
{
projection: { _id: 1 },
readPreference: ReadPreference.SECONDARY,
readPreference: READ_PREFERENCE_SECONDARY,
}
)
}
@@ -118,7 +120,7 @@ async function anyDocHistoryIndexExists(project) {
{ project_id: { $eq: project._id } },
{
projection: { _id: 1 },
readPreference: ReadPreference.SECONDARY,
readPreference: READ_PREFERENCE_SECONDARY,
}
)
}
services/web/scripts/oauth/remove_client.js
+7 -4
View File
@@ -1,6 +1,9 @@
const minimist = require('minimist')
const { ReadPreference } = require('mongodb')
const { waitForDb, db } = require('../../app/src/infrastructure/mongodb')
const {
waitForDb,
db,
READ_PREFERENCE_SECONDARY,
} = require('../../app/src/infrastructure/mongodb')
async function main() {
const opts = parseArgs()
@@ -49,7 +52,7 @@ async function countAccessTokens(applicationId) {
{
oauthApplication_id: applicationId,
},
{ readPreference: ReadPreference.secondary }
{ readPreference: READ_PREFERENCE_SECONDARY }
)
}
@@ -58,7 +61,7 @@ async function countAuthorizationCodes(applicationId) {
{
oauthApplication_id: applicationId,
},
{ readPreference: ReadPreference.secondary }
{ readPreference: READ_PREFERENCE_SECONDARY }
)
}
services/web/scripts/regenerate_duplicate_referral_ids.js
+6 -4
View File
@@ -4,8 +4,10 @@ const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
// persist fallback in order to keep batchedUpdate in-sync
process.env.BATCH_SIZE = BATCH_SIZE
const { ReadPreference } = require('mongodb')
const { db } = require('../app/src/infrastructure/mongodb')
const {
db,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
const { promiseMapWithLimit } = require('../app/src/util/promises')
const TokenGenerator = require('../app/src/Features/TokenGenerator/TokenGenerator')
const { batchedUpdate } = require('./helpers/batchedUpdate')
@@ -24,7 +26,7 @@ async function rewriteDuplicates(duplicateReferralIds) {
{ referal_id: referralId },
{
projection: { _id: 1 },
readPreference: ReadPreference.SECONDARY,
readPreference: READ_PREFERENCE_SECONDARY,
}
)
.toArray()
@@ -74,7 +76,7 @@ async function processBatch(users) {
.find(
{ referal_id: { $in: uniqueReferalIdsInBatch } },
{
readPreference: ReadPreference.SECONDARY,
readPreference: READ_PREFERENCE_SECONDARY,
projection: { _id: true },
}
)