From 715b0df4a1bd7e53ecf8a691543c46cbcba52cbb Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 5 Feb 2015 16:36:41 +0000 Subject: [PATCH] prototype for expanding packs --- .../app/coffee/MongoManager.coffee | 97 +++++++++++++++---- services/track-changes/package.json | 3 +- 2 files changed, 81 insertions(+), 19 deletions(-) diff --git a/services/track-changes/app/coffee/MongoManager.coffee b/services/track-changes/app/coffee/MongoManager.coffee index f2f664dfd6..da7cc08001 100644 --- a/services/track-changes/app/coffee/MongoManager.coffee +++ b/services/track-changes/app/coffee/MongoManager.coffee @@ -1,7 +1,9 @@ {db, ObjectId} = require "./mongojs" async = require "async" +_ = require "underscore" module.exports = MongoManager = + # only used in this module getLastCompressedUpdate: (doc_id, callback = (error, update) ->) -> db.docHistory .find(doc_id: ObjectId(doc_id.toString())) @@ -11,9 +13,11 @@ module.exports = MongoManager = return callback(error) if error? return callback null, compressedUpdates[0] or null + # only used in this module deleteCompressedUpdate: (id, callback = (error) ->) -> db.docHistory.remove({ _id: ObjectId(id.toString()) }, callback) + # used in UpdatesManager popLastCompressedUpdate: (doc_id, callback = (error, update) ->) -> MongoManager.getLastCompressedUpdate doc_id, (error, update) -> return callback(error) if error? @@ -24,6 +28,7 @@ module.exports = MongoManager = else callback null, null + # used in UpdatesManager insertCompressedUpdates: (project_id, doc_id, updates, permanent, callback = (error) ->) -> jobs = [] for update in updates @@ -45,8 +50,78 @@ module.exports = MongoManager = hours = 60 * minutes days = 24 * hours update.expiresAt = new Date(Date.now() + 7 * days) + # may need to roll over a pack here if we are inserting packs db.docHistory.insert update, callback + _findResults: (param, query, limit, callback) -> + sort = {} + sort[param] = -1; + cursor = db.docHistory + .find( query ) + .sort( sort ) + + if limit? + cursor.limit(limit) + + rangeQuery = query[param] + extraQuery = _.clone(query) + if rangeQuery?['$gte']? + extraQuery[param] = {'$lt' : rangeQuery['$gte']} + else if rangeQuery?['$gt'] + extraQuery[param] = {'$lte' : rangeQuery['$gt']} + + filterFn = (item) -> + #console.log 'filter', item, rangeQuery + return false if rangeQuery?['$gte']? && item[param] < rangeQuery['$gte'] + return false if rangeQuery?['$lte']? && item[param] > rangeQuery['$lte'] + return false if rangeQuery?['$lt']? && item[param] >= rangeQuery['$lt'] + return false if rangeQuery?['$gt']? && item[param] <= rangeQuery['$gt'] + #console.log 'accepted' + return true + + # need to support limit here + + cursor.toArray (err, updates) -> + console.log 'query=', query, 'UPDATES=', updates + all = MongoManager._unpackResults(updates).filter(filterFn) + if all.length == 0 + # need an extra result set + console.log 'extraQuery', extraQuery + extra = db.docHistory + .find(extraQuery) + .sort(sort) + .limit(1) + extra.toArray (err, updates2) -> + all2 = MongoManager._unpackResults(updates2).filter(filterFn) + console.log 'got extra', all2 + callback err, all2 + return + if err? + callback err, updates + else + callback err, all + + _unpackResults: (updates) -> + result = [] + # iterate over the updates + # if it's a pack, expand it into ops and insert it into the array at that point + updates.forEach (item) -> + if item.pack? + all = MongoManager._explodePackToOps item + result = result.concat all + else + result.push item + return result + + _explodePackToOps: (packObj) -> + doc_id = packObj.doc_id + project_id = packObj.project_id + result = packObj.pack.map (item) -> + item.doc_id = doc_id + item.project_id = project_id + item + return result.reverse() + getDocUpdates:(doc_id, options = {}, callback = (error, updates) ->) -> query = doc_id: ObjectId(doc_id.toString()) @@ -57,14 +132,7 @@ module.exports = MongoManager = query["v"] ||= {} query["v"]["$lte"] = options.to - cursor = db.docHistory - .find( query ) - .sort( v: -1 ) - - if options.limit? - cursor.limit(options.limit) - - cursor.toArray callback + MongoManager._findResults('v', query, options.limit, callback) getProjectUpdates: (project_id, options = {}, callback = (error, updates) ->) -> query = @@ -73,14 +141,7 @@ module.exports = MongoManager = if options.before? query["meta.end_ts"] = { $lt: options.before } - cursor = db.docHistory - .find( query ) - .sort( "meta.end_ts": -1 ) - - if options.limit? - cursor.limit(options.limit) - - cursor.toArray callback + MongoManager._findResults('meta.end_ts', query, options.limit, callback) backportProjectId: (project_id, doc_id, callback = (error) ->) -> db.docHistory.update { @@ -109,9 +170,9 @@ module.exports = MongoManager = }, callback ensureIndices: () -> - # For finding all updates that go into a diff for a doc + # For finding all updates that go into a diff for a doc (getLastCompressedUpdate, getDocUpdates v > from && v < to) db.docHistory.ensureIndex { doc_id: 1, v: 1 }, { background: true } - # For finding all updates that affect a project + # For finding all updates that affect a project (getProjectUpdates meta.end_ts < before db.docHistory.ensureIndex { project_id: 1, "meta.end_ts": 1 }, { background: true } # For finding updates that don't yet have a project_id and need it inserting db.docHistory.ensureIndex { doc_id: 1, project_id: 1 }, { background: true } diff --git a/services/track-changes/package.json b/services/track-changes/package.json index 4b03ee789a..e6f006d739 100644 --- a/services/track-changes/package.json +++ b/services/track-changes/package.json @@ -15,7 +15,8 @@ "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.0.0", "request": "~2.33.0", "redis-sharelatex": "~0.0.4", - "redis": "~0.10.1" + "redis": "~0.10.1", + "underscore": "~1.7.0" }, "devDependencies": { "chai": "~1.9.0",