Merge pull request #3226 from overleaf/jpa-mongodb-native-scripts

[misc] migrate active scripts to the native mongo driver

GitOrigin-RevId: f3b441bc53754dc2f83ca24c3e57d2cc978dd7fe

services/web/scripts/delete-orphaned-docs/delete-orphaned-docs.js

@@ -4,7 +4,11 @@ const fs = require('fs')
 const minimist = require('minimist')
 const readline = require('readline')
 
-const { db, ObjectId } = require('../../app/src/infrastructure/mongojs')
+const {
+  db,
+  ObjectId,
+  waitForDb
+} = require('../../app/src/infrastructure/mongodb')
 const DocstoreManager = require('../../app/src/Features/Docstore/DocstoreManager')
   .promises
 
@@ -64,6 +68,8 @@ rl.on('close', async () => {
 
   console.log(`Loaded Data for ${docCount} docs in ${projectCount} Projects`)
 
+  await waitForDb()
+
   for (const projectId of Object.keys(orphanedDocs)) {
     await deleteOrphanedDocs(projectId, orphanedDocs[projectId])
   }
@@ -107,33 +113,15 @@ async function projectIdExists(projectId) {
 }
 
 async function findProject(projectId) {
-  return new Promise((resolve, reject) => {
-    db.projects.findOne(
-      { _id: ObjectId(projectId) },
-      { _id: 1 },
-      (err, project) => {
-        if (err) {
-          reject(err)
-        } else {
-          resolve(project)
-        }
-      }
-    )
-  })
+  return db.projects.findOne(
+    { _id: ObjectId(projectId) },
+    { projection: { _id: 1 } }
+  )
 }
 
 async function findDeletedProject(projectId) {
-  return new Promise((resolve, reject) => {
-    db.deletedProjects.findOne(
-      { 'project._id': ObjectId(projectId) },
-      { _id: 1 },
-      (err, project) => {
-        if (err) {
-          reject(err)
-        } else {
-          resolve(project)
-        }
-      }
-    )
-  })
+  return db.deletedProjects.findOne(
+    { 'project._id': ObjectId(projectId) },
+    { projection: { _id: 1 } }
+  )
 }

services/web/scripts/helpers/batchedUpdate.js

@@ -1,6 +1,5 @@
-const { promisify } = require('util')
 const { ReadPreference, ObjectId } = require('mongodb')
-const { getNativeDb } = require('../../app/src/infrastructure/Mongoose')
+const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
 
 const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 1000
 let BATCH_LAST_ID
@@ -13,7 +12,8 @@ async function getNextBatch(collection, query, maxId, projection) {
     query['_id'] = { $gt: maxId }
   }
   const entries = await collection
-    .find(query, projection)
+    .find(query)
+    .project(projection)
     .sort({ _id: 1 })
     .limit(BATCH_SIZE)
     .setReadPreference(ReadPreference.SECONDARY)
@@ -29,15 +29,8 @@ async function performUpdate(collection, nextBatch, update) {
 }
 
 async function batchedUpdate(collectionName, query, update, projection) {
-  // Apparently the mongo driver returns the connection too early.
-  // Some secondary connections are not ready as it returns, leading to
-  //  failing cursor actions with a readPreference set to 'secondary'.
-  // TODO(das7pad): revisit/remove this delay after the mongo-driver update.
-  const CONNECT_DELAY = parseInt(process.env.CONNECT_DELAY, 10) || 10000
-  await Promise.all([getNativeDb(), promisify(setTimeout)(CONNECT_DELAY)])
-
-  const db = await getNativeDb()
-  const collection = db.collection(collectionName)
+  await waitForDb()
+  const collection = db[collectionName]
 
   projection = projection || { _id: 1 }
   let nextBatch

services/web/scripts/invalidate_tokens.js

@@ -1,6 +1,5 @@
-const { db, ObjectId } = require('../app/src/infrastructure/mongojs')
+const { db, ObjectId, waitForDb } = require('../app/src/infrastructure/mongodb')
 const minimist = require('minimist')
-const _ = require('lodash')
 const argv = minimist(process.argv.slice(2))
 const commit = argv.commit !== undefined
 const projectIds = argv._.map(x => {
@@ -11,57 +10,44 @@ if (!commit) {
   console.log('Doing dry run without --commit')
 }
 console.log('checking', projectIds.length, 'projects')
-db.projects.find(
-  {
-    _id: { $in: projectIds }
-  },
-  (err, affectedProjects) => {
-    if (err) {
-      throw err
-    }
-    console.log('Found ' + affectedProjects.length + ' affected projects')
-    affectedProjects.forEach(x => {
-      console.log(
-        JSON.stringify(
-          _.pick(x, [
-            '_id',
-            'owner_ref',
-            'tokenAccessReadOnly_refs',
-            'tokenAccessReadAndWrite_refs'
-          ])
-        )
-      )
-    })
-    if (!commit) {
-      console.log('dry run, not updating')
-      process.exit(0)
-    } else {
-      db.projects.update(
-        {
-          _id: {
-            $in: affectedProjects.map(x => {
-              return x._id
-            })
-          }
-        },
+waitForDb().then(async () => {
+  const affectedProjects = await db.projects
+    .find(
+      { _id: { $in: projectIds } },
+      {
+        projection: {
+          _id: 1,
+          owner_ref: 1,
+          tokenAccessReadOnly_refs: 1,
+          tokenAccessReadAndWrite_refs: 1
+        }
+      }
+    )
+    .toArray()
+  console.log('Found ' + affectedProjects.length + ' affected projects')
+  affectedProjects.forEach(project => {
+    console.log(JSON.stringify(project))
+  })
+  if (!commit) {
+    console.log('dry run, not updating')
+    process.exit(0)
+  } else {
+    try {
+      const result = await db.projects.updateMany(
+        { _id: { $in: affectedProjects.map(project => project._id) } },
         {
           $set: {
             publicAccesLevel: 'private', // note the spelling in the db is publicAccesLevel (with one 's')
             tokenAccessReadOnly_refs: [],
             tokenAccessReadAndWrite_refs: []
           }
-        },
-        {
-          multi: true
-        },
-        (err, result) => {
-          console.log('err', err, 'result', result)
-          db.close()
-          setTimeout(() => {
-            process.exit(0)
-          }, 5000)
         }
       )
+      console.log('result', JSON.stringify(result))
+      process.exit(0)
+    } catch (err) {
+      console.error('err', err)
+      process.exit(1)
     }
   }
-)
+})

services/web/scripts/refresh_features.js

@@ -1,4 +1,4 @@
-const { db } = require('../app/src/infrastructure/mongojs')
+const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
 const minimist = require('minimist')
 const _ = require('lodash')
 const async = require('async')
@@ -85,10 +85,12 @@ const checkAndUpdateUsers = (users, callback) =>
 
 const loopForUsers = (skip, callback) => {
   db.users
-    .find({}, { features: 1, lastLoggedIn: 1 })
+    .find({})
+    .project({ features: 1, lastLoggedIn: 1 })
     .sort('_id')
     .skip(skip)
-    .limit(FETCH_LIMIT, (error, users) => {
+    .limit(FETCH_LIMIT)
+    .toArray((error, users) => {
       if (error) {
         return callback(error)
       }
@@ -141,5 +143,7 @@ const setup = () => {
   }
 }
 
-setup()
-run()
+waitForDb().then(() => {
+  setup()
+  run()
+})

services/web/scripts/ukamf/check-certs.js

@@ -13,10 +13,11 @@
 const { Certificate } = require('@fidm/x509')
 const UKAMFDB = require('./ukamf-db')
 const V1Api = require(`../../app/src/Features/V1/V1Api`).promises
-const { db } = require('../../app/src/infrastructure/mongojs')
+const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
 const moment = require('moment')
 
-main()
+waitForDb()
+  .then(main)
   .catch(err => {
     console.error(err.stack)
   })
@@ -93,17 +94,7 @@ async function checkCert(ukamfDB, providerId) {
 }
 
 async function getActiveProviderIds() {
-  return new Promise((resolve, reject) => {
-    db.users.distinct(
-      'samlIdentifiers.providerId',
-      { 'samlIdentifiers.externalUserId': { $exists: true } },
-      (err, providerIds) => {
-        if (err) {
-          reject(err)
-        } else {
-          resolve(providerIds)
-        }
-      }
-    )
+  return db.users.distinct('samlIdentifiers.providerId', {
+    'samlIdentifiers.externalUserId': { $exists: true }
   })
 }