diff --git a/server-ce/Gruntfile.coffee b/server-ce/Gruntfile.coffee index d9c5114557..61be652c2d 100644 --- a/server-ce/Gruntfile.coffee +++ b/server-ce/Gruntfile.coffee @@ -55,6 +55,8 @@ module.exports = (grunt) -> grunt.loadNpmTasks 'grunt-execute' grunt.loadNpmTasks 'grunt-available-tasks' grunt.loadNpmTasks 'grunt-concurrent' + grunt.loadNpmTasks "grunt-contrib-coffee" + execute = {} for service in SERVICES @@ -70,6 +72,18 @@ module.exports = (grunt) -> options: limit: SERVICES.length logConcurrentOutput: true + coffee: + migrate: + expand: true, + flatten: false, + cwd: './', + src: ['./migrations/*.coffee'], + dest: './', + ext: '.js' + options: + bare:true + + availabletasks: tasks: @@ -147,6 +161,10 @@ module.exports = (grunt) -> grunt.registerTask "build:upstart_scripts", "Create upstart scripts for each service", () -> Helpers.buildUpstartScripts() + + grunt.registerTask 'migrate', 'run migrations', ['coffee:migrate'] + + Helpers = installService: (service, callback = (error) ->) -> Helpers.cloneGitRepo service, (error) -> diff --git a/server-ce/migrations/1422460849371-doc-lines.coffee b/server-ce/migrations/1422460849371-doc-lines.coffee new file mode 100644 index 0000000000..1d370e986f --- /dev/null +++ b/server-ce/migrations/1422460849371-doc-lines.coffee @@ -0,0 +1,113 @@ + +fs = require("fs") +mongojs = require("mongojs") +ObjectId = mongojs.ObjectId +db = mongojs('sharelatex', ['projects', 'docs']) +_ = require("underscore") +async = require("async") + +local_db_path = "/tmp/process-db" + +Datastore = require('nedb') +processDb = new Datastore({ filename: local_db_path}) + + +printProgress = -> + processDb.count {processed:false}, (err, todo)-> + processDb.count {}, (err, total)-> + console.log "#{todo}/#{total} processed" + setTimeout printProgress, 1000 + + +writeProjectIdsToDisk = (callback)-> + db.projects.find {}, {_id:1}, (err,ids)-> + console.log "total found projects in mongo #{ids.length}" + ids = _.map ids, (id)-> return id._id.toString() + jobs = _.map ids, (id)-> + return (cb)-> + processDb.findOne {project_id:id}, (err, doc)-> + if doc? + return cb() + processDb.insert {project_id:id, processed:false}, cb + async.series jobs, (err)-> + processDb.count {processed:false}, (err, count)-> + console.log "projects to process: #{count}" + callback() + +getNextProjectToProccess = (callback)-> + processDb.findOne {processed:false}, (err, doc)-> + callback err, doc.project_id + +markProjectAsProcessed = (project_id, callback)-> + processDb.update project_id:project_id, {$set:{processed:true}}, {}, callback + + +getAllDocs = (project_id, callback = (error, docs) ->) -> + db.projects.findOne _id:ObjectId(project_id), (error, project) -> + return callback(error) if error? + return callback new Errors.NotFoundError("No such project: #{project_id}") if !project? + findAllDocsInProject project, (error, docs) -> + return callback(error) if error? + return callback null, docs + +findAllDocsInProject = (project, callback = (error, docs) ->) -> + callback null, _findAllDocsInFolder project.rootFolder[0] + +_findDocInFolder = (folder = {}, doc_id, currentPath) -> + for doc, i in folder.docs or [] + if doc?._id? and doc._id.toString() == doc_id.toString() + return { + doc: doc + mongoPath: "#{currentPath}.docs.#{i}" + } + + for childFolder, i in folder.folders or [] + result = _findDocInFolder childFolder, doc_id, "#{currentPath}.folders.#{i}" + return result if result? + + return null + +_findAllDocsInFolder = (folder = {}) -> + docs = folder.docs or [] + for childFolder in folder.folders or [] + docs = docs.concat _findAllDocsInFolder childFolder + return docs + +insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)-> + update = {} + update["_id"] = ObjectId(doc_id) + update["lines"] = lines + update["project_id"] = ObjectId(project_id) + update["rev"] = oldRev + db.docs.insert _id: ObjectId(doc_id), callback + +saveDocsIntoMongo = (project_id, docs, callback)-> + jobs = _.map docs, (doc)-> + (cb)-> + insertDocIntoDocCollection project_id, doc._id, project_id.lines, doc.rev, cb + async.series jobs, callback + + +processNext = -> + processDb.count {processed:false}, (err, total)-> + if total == 0 + console.log "DONE" + process.exit() + else + getNextProjectToProccess (err, project_id)-> + getAllDocs project_id, (err, docs)-> + saveDocsIntoMongo project_id, docs, -> + markProjectAsProcessed project_id, -> + processNext() + +processDb.loadDatabase -> + writeProjectIdsToDisk -> + processNext() + +exports.up = (next)-> + + next() + + +exports.down = (next)-> + next() diff --git a/server-ce/package.json b/server-ce/package.json index 2e922d5b14..fdb5894911 100644 --- a/server-ce/package.json +++ b/server-ce/package.json @@ -4,8 +4,11 @@ "description": "An online collaborative LaTeX editor", "dependencies": { "async": "^0.9.0", + "mongojs": "^0.18.1", + "nedb": "^1.1.1", "rimraf": "~2.2.6", - "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git" + "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git", + "underscore": "^1.7.0" }, "devDependencies": { "grunt": "~0.4.2", @@ -14,6 +17,7 @@ "grunt-execute": "~0.1.5", "grunt-available-tasks": "~0.4.1", "grunt-concurrent": "~0.4.3", + "grunt-contrib-coffee": "~0.10.1", "semver": "~2.2.1", "knox": "~0.8.9" }