Merge branch 'master' into node-4.2

services/web/.gitignore

@@ -40,6 +40,7 @@ app.js
 app/js/*
 test/UnitTests/js/*
 test/smoke/js/*
+test/acceptance/js/*
 cookies.txt
 requestQueueWorker.js
 TpdsWorker.js

services/web/Gruntfile.coffee

@@ -71,6 +71,14 @@ module.exports = (grunt) ->
 				dest: 'test/UnitTests/js/',
 				ext: '.js'
 
+			acceptance_tests: 
+				expand: true,
+				flatten: false,
+				cwd: 'test/acceptance/coffee',
+				src: ['**/*.coffee'],
+				dest: 'test/acceptance/js/',
+				ext: '.js'
+
 		less:
 			app:
 				files:
@@ -119,6 +127,7 @@ module.exports = (grunt) ->
 		clean:
 			app: ["app/js"]
 			unit_tests: ["test/UnitTests/js"]
+			acceptance_tests: ["test/acceptance/js"]
 
 		mochaTest:
 			unit:
@@ -131,6 +140,12 @@ module.exports = (grunt) ->
 				options:
 					reporter: grunt.option('reporter') or 'spec'
 					grep: grunt.option("grep")
+			acceptance:
+				src: ["test/acceptance/js/#{grunt.option('feature') or '**'}/*.js"]
+				options:
+					timeout: 10000
+					reporter: grunt.option('reporter') or 'spec'
+					grep: grunt.option("grep")
 
 		"git-rev-parse":
 			version:
@@ -184,6 +199,7 @@ module.exports = (grunt) ->
 		            	]
 		            	"Test tasks": [
 		            		"test:unit"
+		            		"test:acceptance"
 		            	]
 		            	"Run tasks": [
 		            		"run"
@@ -290,6 +306,7 @@ module.exports = (grunt) ->
 	grunt.registerTask 'compile:css', 'Compile the less files to css', ['less']
 	grunt.registerTask 'compile:minify', 'Concat and minify the client side js', ['requirejs', "file_append"]
 	grunt.registerTask 'compile:unit_tests', 'Compile the unit tests', ['clean:unit_tests', 'coffee:unit_tests']
+	grunt.registerTask 'compile:acceptance_tests', 'Compile the acceptance tests', ['clean:acceptance_tests', 'coffee:acceptance_tests']
 	grunt.registerTask 'compile:smoke_tests', 'Compile the smoke tests', ['coffee:smoke_tests']
 	grunt.registerTask 'compile:tests', 'Compile all the tests', ['compile:smoke_tests', 'compile:unit_tests']
 	grunt.registerTask 'compile', 'Compiles everything need to run web-sharelatex', ['compile:server', 'compile:client', 'compile:css']
@@ -297,6 +314,7 @@ module.exports = (grunt) ->
 	grunt.registerTask 'install', "Compile everything when installing as an npm module", ['compile']
 
 	grunt.registerTask 'test:unit', 'Run the unit tests (use --grep=<regex> or --feature=<feature> for individual tests)', ['compile:server', 'compile:modules:server', 'compile:unit_tests', 'compile:modules:unit_tests', 'mochaTest:unit'].concat(moduleUnitTestTasks)
+	grunt.registerTask 'test:acceptance', 'Run the acceptance tests (use --grep=<regex> or --feature=<feature> for individual tests)', ['compile:acceptance_tests', 'mochaTest:acceptance']
 	grunt.registerTask 'test:smoke', 'Run the smoke tests', ['compile:smoke_tests', 'mochaTest:smoke']
 	
 	grunt.registerTask 'test:modules:unit', 'Run the unit tests for the modules', ['compile:modules:server', 'compile:modules:unit_tests'].concat(moduleUnitTestTasks)

services/web/app.coffee

@@ -10,7 +10,6 @@ metrics = require("metrics-sharelatex")
 metrics.initialize("web")
 metrics.memory.monitor(logger)
 Server = require("./app/js/infrastructure/Server")
-Errors = require "./app/js/errors"
 
 argv = require("optimist")
 	.options("user", {alias : "u", description : "Run the server with permissions of the specified user"})
@@ -18,18 +17,6 @@ argv = require("optimist")
 	.usage("Usage: $0")
 	.argv
 
-Server.app.use (error, req, res, next) ->
-	if error?.code is 'EBADCSRFTOKEN'
-		logger.log err: error,url:req.url, method:req.method, user:req?.sesson?.user, "invalid csrf"
-		res.sendStatus(403)
-		return
-	logger.error err: error, url:req.url, method:req.method, user:req?.sesson?.user, "error passed to top level next middlewear"
-	res.statusCode = error.status or 500
-	if res.statusCode == 500
-		res.end("Oops, something went wrong with your request, sorry. If this continues, please contact us at #{Settings.adminEmail}")
-	else
-		res.end()
-
 if Settings.catchErrors
 	process.removeAllListeners "uncaughtException"
 	process.on "uncaughtException", (error) ->

services/web/app/coffee/Features/Authentication/AuthenticationController.coffee

@@ -44,50 +44,27 @@ module.exports = AuthenticationController =
 						text: req.i18n.translate("email_or_password_wrong_try_again"),
 						type: 'error'
 
-	getAuthToken: (req, res, next = (error) ->) ->
-		AuthenticationController.getLoggedInUserId req, (error, user_id) ->
-			return next(error) if error?
-			AuthenticationManager.getAuthToken user_id, (error, auth_token) ->
-				return next(error) if error?
-				res.send(auth_token)
-
 	getLoggedInUserId: (req, callback = (error, user_id) ->) ->
 		if req?.session?.user?._id?
 			callback null, req.session.user._id.toString()
 		else
 			callback null, null
 
-	getLoggedInUser: (req, options = {allow_auth_token: false}, callback = (error, user) ->) ->
-		if typeof(options) == "function"
-			callback = options
-			options = {allow_auth_token: false}
-
+	getLoggedInUser: (req, callback = (error, user) ->) ->
 		if req.session?.user?._id?
 			query = req.session.user._id
-		else if req.query?.auth_token? and options.allow_auth_token
-			query = { auth_token: req.query.auth_token }
 		else
 			return callback null, null
 
 		UserGetter.getUser query, callback
 
-	requireLogin: (options = {allow_auth_token: false, load_from_db: false}) ->
+	requireLogin: () ->
 		doRequest = (req, res, next = (error) ->) ->
-			load_from_db = options.load_from_db
-			if req.query?.auth_token? and options.allow_auth_token
-				load_from_db = true
-			if load_from_db
-				AuthenticationController.getLoggedInUser req, { allow_auth_token: options.allow_auth_token }, (error, user) ->
-					return next(error) if error?
-					return AuthenticationController._redirectToLoginOrRegisterPage(req, res) if !user?
-					req.user = user
-					return next()
+			if !req.session.user?
+				AuthenticationController._redirectToLoginOrRegisterPage(req, res) 
 			else
-				if !req.session.user?
-					AuthenticationController._redirectToLoginOrRegisterPage(req, res) 
-				else
-					req.user = req.session.user
-					return next()
+				req.user = req.session.user
+				return next()
 
 		return doRequest
 

services/web/app/coffee/Features/Authentication/AuthenticationManager.coffee

@@ -35,19 +35,3 @@ module.exports = AuthenticationManager =
 					$unset: password: true
 				}, callback)
 
-	getAuthToken: (user_id, callback = (error, auth_token) ->) ->
-		db.users.findOne { _id: ObjectId(user_id.toString()) }, { auth_token : true }, (error, user) =>
-			return callback(error) if error?
-			return callback(new Error("user could not be found: #{user_id}")) if !user?
-			if user.auth_token?
-				callback null, user.auth_token
-			else
-				@_createSecureToken (error, auth_token) ->
-					db.users.update { _id: ObjectId(user_id.toString()) }, { $set : auth_token: auth_token }, (error) ->
-						return callback(error) if error?
-						callback null, auth_token
-		
-	_createSecureToken: (callback = (error, token) ->) ->
-		crypto.randomBytes 48, (error, buffer) ->
-			return callback(error) if error?
-			callback null, buffer.toString("hex")

services/web/app/coffee/Features/Authorization/AuthorizationManager.coffee

@@ -0,0 +1,73 @@
+CollaboratorsHandler = require("../Collaborators/CollaboratorsHandler")
+Project = require("../../models/Project").Project
+User = require("../../models/User").User
+PrivilegeLevels = require("./PrivilegeLevels")
+PublicAccessLevels = require("./PublicAccessLevels")
+Errors = require("../Errors/Errors")
+
+module.exports = AuthorizationManager =
+	# Get the privilege level that the user has for the project
+	# Returns:
+	#	* privilegeLevel: "owner", "readAndWrite", of "readOnly" if the user has
+	#	  access. false if the user does not have access
+	#   * becausePublic: true if the access level is only because the project is public.
+	getPrivilegeLevelForProject: (user_id, project_id, callback = (error, privilegeLevel, becausePublic) ->) ->
+		getPublicAccessLevel = () ->
+			Project.findOne { _id: project_id }, { publicAccesLevel: 1 }, (error, project) ->
+				return callback(error) if error?
+				if !project?
+					return callback new Errors.NotFoundError("no project found with id #{project_id}")
+				if project.publicAccesLevel == PublicAccessLevels.READ_ONLY
+					return callback null, PrivilegeLevels.READ_ONLY, true
+				else if project.publicAccesLevel == PublicAccessLevels.READ_AND_WRITE
+					return callback null, PrivilegeLevels.READ_AND_WRITE, true
+				else
+					return callback null, PrivilegeLevels.NONE, false
+		
+		if !user_id?
+			getPublicAccessLevel()
+		else
+			CollaboratorsHandler.getMemberIdPrivilegeLevel user_id, project_id, (error, privilegeLevel) ->
+				return callback(error) if error?
+				if privilegeLevel? and privilegeLevel != PrivilegeLevels.NONE
+					# The user has direct access
+					callback null, privilegeLevel, false
+				else
+					AuthorizationManager.isUserSiteAdmin user_id, (error, isAdmin) ->
+						return callback(error) if error?
+						if isAdmin
+							callback null, PrivilegeLevels.OWNER, false
+						else
+							getPublicAccessLevel()
+
+	canUserReadProject: (user_id, project_id, callback = (error, canRead) ->) ->
+		AuthorizationManager.getPrivilegeLevelForProject user_id, project_id, (error, privilegeLevel) ->
+			return callback(error) if error?
+			return callback null, (privilegeLevel in [PrivilegeLevels.OWNER, PrivilegeLevels.READ_AND_WRITE, PrivilegeLevels.READ_ONLY])
+		
+	canUserWriteProjectContent: (user_id, project_id, callback = (error, canWriteContent) ->) ->
+		AuthorizationManager.getPrivilegeLevelForProject user_id, project_id, (error, privilegeLevel) ->
+			return callback(error) if error?
+			return callback null, (privilegeLevel in [PrivilegeLevels.OWNER, PrivilegeLevels.READ_AND_WRITE])
+		
+	canUserWriteProjectSettings: (user_id, project_id, callback = (error, canWriteSettings) ->) ->
+		AuthorizationManager.getPrivilegeLevelForProject user_id, project_id, (error, privilegeLevel, becausePublic) ->
+			return callback(error) if error?
+			if privilegeLevel == PrivilegeLevels.OWNER
+				return callback null, true
+			else if privilegeLevel == PrivilegeLevels.READ_AND_WRITE and !becausePublic
+				return callback null, true
+			else
+				return callback null, false
+	
+	canUserAdminProject: (user_id, project_id, callback = (error, canAdmin) ->) ->
+		AuthorizationManager.getPrivilegeLevelForProject user_id, project_id, (error, privilegeLevel) ->
+			return callback(error) if error?
+			return callback null, (privilegeLevel == PrivilegeLevels.OWNER)
+	
+	isUserSiteAdmin: (user_id, callback = (error, isAdmin) ->) ->
+		if !user_id?
+			return callback null, false
+		User.findOne { _id: user_id }, { isAdmin: 1 }, (error, user) ->
+			return callback(error) if error?
+			return callback null, (user?.isAdmin == true)

services/web/app/coffee/Features/Authorization/AuthorizationMiddlewear.coffee

@@ -0,0 +1,111 @@
+AuthorizationManager = require("./AuthorizationManager")
+async = require "async"
+logger = require "logger-sharelatex"
+ObjectId = require("mongojs").ObjectId
+Errors = require "../Errors/Errors"
+
+module.exports = AuthorizationMiddlewear =
+	ensureUserCanReadMultipleProjects: (req, res, next) ->
+		project_ids = (req.query.project_ids or "").split(",")
+		AuthorizationMiddlewear._getUserId req, (error, user_id) ->
+			return next(error) if error?
+			# Remove the projects we have access to. Note rejectSeries doesn't use
+			# errors in callbacks
+			async.rejectSeries project_ids, (project_id, cb) ->
+				AuthorizationManager.canUserReadProject user_id, project_id, (error, canRead) ->
+					return next(error) if error?
+					cb(canRead)
+			, (unauthorized_project_ids) ->
+				if unauthorized_project_ids.length > 0
+					AuthorizationMiddlewear.redirectToRestricted req, res, next
+				else
+					next()
+		
+	ensureUserCanReadProject: (req, res, next) ->
+		AuthorizationMiddlewear._getUserAndProjectId req, (error, user_id, project_id) ->
+			return next(error) if error?
+			AuthorizationManager.canUserReadProject user_id, project_id, (error, canRead) ->
+				return next(error) if error?
+				if canRead
+					logger.log {user_id, project_id}, "allowing user read access to project"
+					next()
+				else
+					logger.log {user_id, project_id}, "denying user read access to project"
+					AuthorizationMiddlewear.redirectToRestricted req, res, next
+		
+	ensureUserCanWriteProjectSettings: (req, res, next) ->
+		AuthorizationMiddlewear._getUserAndProjectId req, (error, user_id, project_id) ->
+			return next(error) if error?
+			AuthorizationManager.canUserWriteProjectSettings user_id, project_id, (error, canWrite) ->
+				return next(error) if error?
+				if canWrite
+					logger.log {user_id, project_id}, "allowing user write access to project settings"
+					next()
+				else
+					logger.log {user_id, project_id}, "denying user write access to project settings"
+					AuthorizationMiddlewear.redirectToRestricted req, res, next
+		
+	ensureUserCanWriteProjectContent: (req, res, next) ->
+		AuthorizationMiddlewear._getUserAndProjectId req, (error, user_id, project_id) ->
+			return next(error) if error?
+			AuthorizationManager.canUserWriteProjectContent user_id, project_id, (error, canWrite) ->
+				return next(error) if error?
+				if canWrite
+					logger.log {user_id, project_id}, "allowing user write access to project content"
+					next()
+				else
+					logger.log {user_id, project_id}, "denying user write access to project settings"
+					AuthorizationMiddlewear.redirectToRestricted req, res, next
+		
+	ensureUserCanAdminProject: (req, res, next) ->
+		AuthorizationMiddlewear._getUserAndProjectId req, (error, user_id, project_id) ->
+			return next(error) if error?
+			AuthorizationManager.canUserAdminProject user_id, project_id, (error, canAdmin) ->
+				return next(error) if error?
+				if canAdmin
+					logger.log {user_id, project_id}, "allowing user admin access to project"
+					next()
+				else
+					logger.log {user_id, project_id}, "denying user admin access to project"
+					AuthorizationMiddlewear.redirectToRestricted req, res, next
+		
+	ensureUserIsSiteAdmin: (req, res, next) ->
+		AuthorizationMiddlewear._getUserId req, (error, user_id) ->
+			return next(error) if error?
+			AuthorizationManager.isUserSiteAdmin user_id, (error, isAdmin) ->
+				return next(error) if error?
+				if isAdmin
+					logger.log {user_id}, "allowing user admin access to site"
+					next()
+				else
+					logger.log {user_id}, "denying user admin access to site"
+					AuthorizationMiddlewear.redirectToRestricted req, res, next
+
+	_getUserAndProjectId: (req, callback = (error, user_id, project_id) ->) ->
+		project_id = req.params?.project_id or req.params?.Project_id
+		if !project_id?
+			return callback(new Error("Expected project_id in request parameters"))
+		if !ObjectId.isValid(project_id)
+			return callback(new Errors.NotFoundError("invalid project_id: #{project_id}"))
+		AuthorizationMiddlewear._getUserId req, (error, user_id) ->
+			return callback(error) if error?
+			callback(null, user_id, project_id)
+	
+	_getUserId: (req, callback = (error, user_id) ->) ->
+		if req.session?.user?._id?
+			user_id = req.session.user._id
+		else
+			user_id = null
+		callback null, user_id
+		
+	redirectToRestricted: (req, res, next) ->
+		res.redirect "/restricted"
+	
+	restricted : (req, res, next)->
+		if req.session.user?
+			res.render 'user/restricted',
+				title:'restricted'
+		else
+			logger.log "user not logged in and trying to access #{req.url}, being redirected to login"
+			res.redirect '/register'
+		

services/web/app/coffee/Features/Authorization/PrivilegeLevels.coffee

@@ -0,0 +1,5 @@
+module.exports =
+	NONE: false
+	READ_ONLY: "readOnly"
+	READ_AND_WRITE: "readAndWrite"
+	OWNER: "owner"

services/web/app/coffee/Features/Authorization/PublicAccessLevels.coffee

@@ -0,0 +1,4 @@
+module.exports =
+	READ_ONLY: "readOnly"
+	READ_AND_WRITE: "readAndWrite"
+	PRIVATE: "private"

services/web/app/coffee/Features/Collaborators/CollaboratorsController.coffee

@@ -7,15 +7,6 @@ UserGetter = require "../User/UserGetter"
 mimelib = require("mimelib")
 
 module.exports = CollaboratorsController =
-	getCollaborators: (req, res, next = (error) ->) ->
-		ProjectGetter.getProject req.params.Project_id, { owner_ref: true, collaberator_refs: true, readOnly_refs: true}, (error, project) ->
-			return next(error) if error?
-			ProjectGetter.populateProjectWithUsers project, (error, project) ->
-				return next(error) if error?
-				CollaboratorsController._formatCollaborators project, (error, collaborators) ->
-					return next(error) if error?
-					res.send(JSON.stringify(collaborators))
-			
 	addUserToProject: (req, res, next) ->
 		project_id = req.params.Project_id
 		LimitationsManager.canAddXCollaborators project_id, 1, (error, allowed) =>
@@ -59,29 +50,3 @@ module.exports = CollaboratorsController =
 			EditorRealTimeController.emitToRoom(project_id, 'userRemovedFromProject', user_id)
 			callback()
 
-	_formatCollaborators: (project, callback = (error, collaborators) ->) ->
-		collaborators = []
-
-		pushCollaborator = (user, permissions, owner) ->
-			collaborators.push {
-				id: user._id.toString()
-				first_name: user.first_name
-				last_name: user.last_name
-				email: user.email
-				permissions: permissions
-				owner: owner
-			}
-			
-		if project.owner_ref?
-			pushCollaborator(project.owner_ref, ["read", "write", "admin"], true)
-
-		if project.collaberator_refs? and project.collaberator_refs.length > 0
-			for user in project.collaberator_refs
-				pushCollaborator(user, ["read", "write"], false)
-
-		if project.readOnly_refs? and project.readOnly_refs.length > 0
-			for user in project.readOnly_refs
-				pushCollaborator(user, ["read"], false)
-
-		callback null, collaborators
-			

services/web/app/coffee/Features/Collaborators/CollaboratorsHandler.coffee

@@ -1,13 +1,77 @@
 UserCreator = require('../User/UserCreator')
 Project = require("../../models/Project").Project
-ProjectEntityHandler = require("../Project/ProjectEntityHandler")
 mimelib = require("mimelib")
 logger = require('logger-sharelatex')
 UserGetter = require "../User/UserGetter"
 ContactManager = require "../Contacts/ContactManager"
 CollaboratorsEmailHandler = require "./CollaboratorsEmailHandler"
+async = require "async"
+PrivilegeLevels = require "../Authorization/PrivilegeLevels"
+Errors = require "../Errors/Errors"
 
 module.exports = CollaboratorsHandler =
+	getMemberIdsWithPrivilegeLevels: (project_id, callback = (error, members) ->) ->
+		Project.findOne { _id: project_id }, { owner_ref: 1, collaberator_refs: 1, readOnly_refs: 1 }, (error, project) ->
+			return callback(error) if error?
+			return callback new Errors.NotFoundError("no project found with id #{project_id}") if !project?
+			members = []
+			members.push { id: project.owner_ref.toString(), privilegeLevel: PrivilegeLevels.OWNER }
+			for member_id in project.readOnly_refs or []
+				members.push { id: member_id.toString(), privilegeLevel: PrivilegeLevels.READ_ONLY }
+			for member_id in project.collaberator_refs or []
+				members.push { id: member_id.toString(), privilegeLevel: PrivilegeLevels.READ_AND_WRITE }
+			return callback null, members
+	
+	getMemberIds: (project_id, callback = (error, member_ids) ->) ->
+		CollaboratorsHandler.getMemberIdsWithPrivilegeLevels project_id, (error, members) ->
+			return callback(error) if error?
+			return callback null, members.map (m) -> m.id
+	
+	getMembersWithPrivilegeLevels: (project_id, callback = (error, members) ->) ->
+		CollaboratorsHandler.getMemberIdsWithPrivilegeLevels project_id, (error, members = []) ->
+			return callback(error) if error?
+			async.mapLimit members, 3,
+				(member, cb) ->
+					UserGetter.getUser member.id, (error, user) ->
+						return cb(error) if error?
+						return cb(null, { user: user, privilegeLevel: member.privilegeLevel })
+				callback
+	
+	getMemberIdPrivilegeLevel: (user_id, project_id, callback = (error, privilegeLevel) ->) ->
+		# In future if the schema changes and getting all member ids is more expensive (multiple documents)
+		# then optimise this.
+		CollaboratorsHandler.getMemberIdsWithPrivilegeLevels project_id, (error, members = []) ->
+			return callback(error) if error?
+			for member in members
+				if member.id == user_id?.toString()
+					return callback null, member.privilegeLevel
+			return callback null, PrivilegeLevels.NONE
+	
+	getMemberCount: (project_id, callback = (error, count) ->) ->
+		CollaboratorsHandler.getMemberIdsWithPrivilegeLevels project_id, (error, members) ->
+			return callback(error) if error?
+			return callback null, (members or []).length
+		
+	getCollaboratorCount: (project_id, callback = (error, count) ->) ->
+		CollaboratorsHandler.getMemberCount project_id, (error, count) ->
+			return callback(error) if error?
+			return callback null, count - 1 # Don't count project owner
+
+	isUserMemberOfProject: (user_id, project_id, callback = (error, isMember, privilegeLevel) ->) ->
+		CollaboratorsHandler.getMemberIdsWithPrivilegeLevels project_id, (error, members = []) ->
+			return callback(error) if error?
+			for member in members
+				if member.id.toString() == user_id.toString()
+					return callback null, true, member.privilegeLevel
+			return callback null, false, null
+			
+	getProjectsUserIsCollaboratorOf: (user_id, fields, callback = (error, readAndWriteProjects, readOnlyProjects) ->) ->
+		Project.find {collaberator_refs:user_id}, fields, (err, readAndWriteProjects)=>
+			return callback(err) if err?
+			Project.find {readOnly_refs:user_id}, fields, (err, readOnlyProjects)=>
+				return callback(err) if err?
+				callback(null, readAndWriteProjects, readOnlyProjects)
+		
 	removeUserFromProject: (project_id, user_id, callback = (error) ->)->
 		logger.log user_id: user_id, project_id: project_id, "removing user"
 		conditions = _id:project_id
@@ -38,10 +102,10 @@ module.exports = CollaboratorsHandler =
 			if existing_users.indexOf(user_id.toString()) > -1
 				return callback null # User already in Project
 				
-			if privilegeLevel == 'readAndWrite'
+			if privilegeLevel == PrivilegeLevels.READ_AND_WRITE
 				level = {"collaberator_refs":user_id}
 				logger.log {privileges: "readAndWrite", user_id, project_id}, "adding user"
-			else if privilegeLevel == 'readOnly'
+			else if privilegeLevel == PrivilegeLevels.READ_ONLY
 				level = {"readOnly_refs":user_id}
 				logger.log {privileges: "readOnly", user_id, project_id}, "adding user"
 			else
@@ -57,6 +121,7 @@ module.exports = CollaboratorsHandler =
 			Project.update { _id: project_id }, { $addToSet: level }, (error) ->
 				return callback(error) if error?
 				# Flush to TPDS in background to add files to collaborator's Dropbox
+				ProjectEntityHandler = require("../Project/ProjectEntityHandler")
 				ProjectEntityHandler.flushProjectToThirdPartyDataStore project_id, (error) ->
 					if error?
 						logger.error {err: error, project_id, user_id}, "error flushing to TPDS after adding collaborator"

services/web/app/coffee/Features/Collaborators/CollaboratorsRouter.coffee

@@ -1,11 +1,10 @@
 CollaboratorsController = require('./CollaboratorsController')
-SecurityManager = require('../../managers/SecurityManager')
 AuthenticationController = require('../Authentication/AuthenticationController')
+AuthorizationMiddlewear = require('../Authorization/AuthorizationMiddlewear')
 
 module.exports =
 	apply: (webRouter, apiRouter) ->
 		webRouter.post '/project/:Project_id/leave', AuthenticationController.requireLogin(), CollaboratorsController.removeSelfFromProject
-		apiRouter.get  '/project/:Project_id/collaborators', SecurityManager.requestCanAccessProject(allow_auth_token: true), CollaboratorsController.getCollaborators
 
-		webRouter.post   '/project/:Project_id/users', SecurityManager.requestIsOwner, CollaboratorsController.addUserToProject
-		webRouter.delete '/project/:Project_id/users/:user_id', SecurityManager.requestIsOwner, CollaboratorsController.removeUserFromProject
+		webRouter.post   '/project/:Project_id/users', AuthorizationMiddlewear.ensureUserCanAdminProject, CollaboratorsController.addUserToProject
+		webRouter.delete '/project/:Project_id/users/:user_id', AuthorizationMiddlewear.ensureUserCanAdminProject, CollaboratorsController.removeUserFromProject

services/web/app/coffee/Features/Compile/ClsiCookieManager.coffee

@@ -0,0 +1,64 @@
+Settings = require "settings-sharelatex"
+request = require('request')
+redis = require("redis-sharelatex")
+rclient = redis.createClient(Settings.redis.web)
+Cookie = require('cookie')
+logger = require "logger-sharelatex"
+
+buildKey = (project_id)->
+	return "clsiserver:#{project_id}"
+
+clsiCookiesEnabled = Settings.clsiCookie?.key? and Settings.clsiCookie.key.length != 0
+
+
+module.exports = ClsiCookieManager =
+
+	_getServerId : (project_id, callback = (err, serverId)->)->
+		rclient.get buildKey(project_id), (err, serverId)->
+			if err?
+				return callback(err)
+			if serverId?
+				return callback(null, serverId)
+			else
+				return ClsiCookieManager._populateServerIdViaRequest project_id, callback
+
+
+	_populateServerIdViaRequest :(project_id, callback = (err, serverId)->)->
+		url = "#{Settings.apis.clsi.url}/project/#{project_id}/status"
+		request.get url, (err, res, body)->
+			if err?
+				logger.err err:err, project_id:project_id, "error getting initial server id for project"
+				return callback(err)
+			ClsiCookieManager.setServerId project_id, res, (err, serverId)->
+				if err?
+					logger.err err:err, project_id:project_id, "error setting server id via populate request"
+				callback(err, serverId)
+
+	_parseServerIdFromResponse : (response)->
+		cookies = Cookie.parse(response.headers["set-cookie"]?[0] or "")
+		return cookies?[Settings.clsiCookie.key]
+
+	setServerId: (project_id, response, callback = (err, serverId)->)->
+		if !clsiCookiesEnabled
+			return callback()
+		serverId = ClsiCookieManager._parseServerIdFromResponse(response)
+		multi = rclient.multi()
+		multi.set buildKey(project_id), serverId
+		multi.expire buildKey(project_id), Settings.clsiCookie.ttl
+		multi.exec (err)->
+			callback(err, serverId)
+
+
+	getCookieJar: (project_id, callback = (err, jar)->)->
+		if !clsiCookiesEnabled
+			return callback(null, request.jar())
+		ClsiCookieManager._getServerId project_id, (err, serverId)=>
+			if err?
+				logger.err err:err, project_id:project_id, "error getting server id"
+				return callback(err)
+			serverCookie = request.cookie("#{Settings.clsiCookie.key}=#{serverId}")
+			jar = request.jar()
+			jar.setCookie serverCookie, Settings.apis.clsi.url
+			callback(null, jar)
+
+

services/web/app/coffee/Features/Compile/ClsiManager.coffee

@@ -5,39 +5,63 @@ request = require('request')
 Project = require("../../models/Project").Project
 ProjectEntityHandler = require("../Project/ProjectEntityHandler")
 logger = require "logger-sharelatex"
-url = require("url")
+Url = require("url")
+ClsiCookieManager = require("./ClsiCookieManager")
+
 
 module.exports = ClsiManager =
+
 	sendRequest: (project_id, options = {}, callback = (error, success) ->) ->
 		ClsiManager._buildRequest project_id, options, (error, req) ->
 			return callback(error) if error?
 			logger.log project_id: project_id, "sending compile to CLSI"
 			ClsiManager._postToClsi project_id, req, options.compileGroup, (error, response) ->
-				return callback(error) if error?
-				logger.log project_id: project_id, response: response, "received compile response from CLSI"
-				callback(
-					null
-					response?.compile?.status
-					ClsiManager._parseOutputFiles(project_id, response?.compile?.outputFiles)
-				)
+				if error?
+					logger.err err:error, project_id:project_id, "error sending request to clsi"
+					return callback(error)
+				logger.log project_id: project_id, outputFilesLength: response?.outputFiles?.length, status: response?.status, "received compile response from CLSI"
+				ClsiCookieManager._getServerId project_id, (err, clsiServerId)->
+					if err?
+						logger.err err:err, project_id:project_id, "error getting server id"
+						return callback(err)
+					outputFiles = ClsiManager._parseOutputFiles(project_id, response?.compile?.outputFiles)
+					callback(null, response?.compile?.status, outputFiles, clsiServerId)
 
 	deleteAuxFiles: (project_id, options, callback = (error) ->) ->
 		compilerUrl = @_getCompilerUrl(options?.compileGroup)
-		request.del "#{compilerUrl}/project/#{project_id}", callback
+		opts =
+			url:"#{compilerUrl}/project/#{project_id}"
+			method:"DELETE"
+		ClsiManager._makeRequest project_id, opts, callback
+
+
+	_makeRequest: (project_id, opts, callback)->
+		ClsiCookieManager.getCookieJar project_id, (err, jar)->
+			if err?
+				logger.err err:err, "error getting cookie jar for clsi request"
+				return callback(err)
+			opts.jar = jar
+			request opts, (err, response, body)->
+				if err?
+					logger.err err:err, project_id:project_id, url:opts?.url, "error making request to clsi"
+					return callback(err)
+				ClsiCookieManager.setServerId project_id, response, (err)->
+					if err?
+						logger.warn err:err, project_id:project_id, "error setting server id"
+						
+					return callback err, response, body
+
 
 	_getCompilerUrl: (compileGroup) ->
-		if compileGroup == "priority"
-			return Settings.apis.clsi_priority.url
-		else
-			return Settings.apis.clsi.url
+		return Settings.apis.clsi.url
 
 	_postToClsi: (project_id, req, compileGroup, callback = (error, response) ->) ->
-		compilerUrl = @_getCompilerUrl(compileGroup)
-		request.post {
+		compilerUrl = Settings.apis.clsi.url
+		opts = 
 			url:  "#{compilerUrl}/project/#{project_id}/compile"
 			json: req
-			jar:  false
-		}, (error, response, body) ->
+			method: "POST"
+		ClsiManager._makeRequest project_id, opts, (error, response, body) ->
 			return callback(error) if error?
 			if 200 <= response.statusCode < 300
 				callback null, body
@@ -51,8 +75,10 @@ module.exports = ClsiManager =
 	_parseOutputFiles: (project_id, rawOutputFiles = []) ->
 		outputFiles = []
 		for file in rawOutputFiles
+			path = Url.parse(file.url).path
+			path = path.replace("/project/#{project_id}/output/", "")
 			outputFiles.push
-				path: url.parse(file.url).path.replace("/project/#{project_id}/output/", "")
+				path: path
 				type: file.type
 				build: file.build
 		return outputFiles
@@ -86,6 +112,9 @@ module.exports = ClsiManager =
 							rootResourcePathOverride = path
 
 					rootResourcePath = rootResourcePathOverride if rootResourcePathOverride?
+					if !rootResourcePath?
+						logger.warn {project_id}, "no root document found, setting to main.tex"
+						rootResourcePath = "main.tex"
 
 					for path, file of files
 						path = path.replace(/^\//, "") # Remove leading /
@@ -94,19 +123,16 @@ module.exports = ClsiManager =
 							url:      "#{Settings.apis.filestore.url}/project/#{project._id}/file/#{file._id}"
 							modified: file.created?.getTime()
 
-					if !rootResourcePath?
-						callback new Error("no root document exists")
-					else
-						callback null, {
-							compile:
-								options:
-									compiler: project.compiler
-									timeout: options.timeout
-									imageName: project.imageName
-									draft: !!options.draft
-								rootResourcePath: rootResourcePath
-								resources: resources
-						}
+					callback null, {
+						compile:
+							options:
+								compiler: project.compiler
+								timeout: options.timeout
+								imageName: project.imageName
+								draft: !!options.draft
+							rootResourcePath: rootResourcePath
+							resources: resources
+					}
 
 	wordCount: (project_id, file, options, callback = (error, response) ->) ->
 		ClsiManager._buildRequest project_id, options, (error, req) ->
@@ -115,9 +141,10 @@ module.exports = ClsiManager =
 			wordcount_url = "#{compilerUrl}/project/#{project_id}/wordcount?file=#{encodeURIComponent(filename)}"
 			if req.compile.options.imageName?
 				wordcount_url += "&image=#{encodeURIComponent(req.compile.options.imageName)}"
-			request.get {
+			opts =
 				url: wordcount_url
-			}, (error, response, body) ->
+				method: "GET"
+			ClsiManager._makeRequest project_id, opts, (error, response, body) ->
 				return callback(error) if error?
 				if 200 <= response.statusCode < 300
 					callback null, body

services/web/app/coffee/Features/Compile/CompileController.coffee

@@ -8,6 +8,8 @@ Settings = require "settings-sharelatex"
 AuthenticationController = require "../Authentication/AuthenticationController"
 UserGetter = require "../User/UserGetter"
 RateLimiter = require("../../infrastructure/RateLimiter")
+ClsiCookieManager = require("./ClsiCookieManager")
+Path = require("path")
 
 module.exports = CompileController =
 	compile: (req, res, next = (error) ->) ->
@@ -28,27 +30,23 @@ module.exports = CompileController =
 			if req.body?.draft
 				options.draft = req.body.draft
 			logger.log {options, project_id}, "got compile request"
-			CompileManager.compile project_id, user_id, options, (error, status, outputFiles, output, limits) ->
+			CompileManager.compile project_id, user_id, options, (error, status, outputFiles, clsiServerId, limits) ->
 				return next(error) if error?
 				res.contentType("application/json")
-				res.send 200, JSON.stringify {
+				res.status(200).send JSON.stringify {
 					status: status
 					outputFiles: outputFiles
 					compileGroup: limits?.compileGroup
+					clsiServerId:clsiServerId
 				}
 
 	downloadPdf: (req, res, next = (error) ->)->
-	
 		Metrics.inc "pdf-downloads"
 		project_id = req.params.Project_id
 		isPdfjsPartialDownload = req.query?.pdfng
-
-
-
 		rateLimit = (callback)->
 			if isPdfjsPartialDownload
 				callback null, true
-	
 			else
 				rateLimitOpts =
 					endpointName: "full-pdf-download"
@@ -93,10 +91,36 @@ module.exports = CompileController =
 
 	getFileFromClsi: (req, res, next = (error) ->) ->
 		project_id = req.params.Project_id
-		CompileController.proxyToClsi(project_id, "/project/#{project_id}/output/#{req.params.file}", req, res, next)
+		build = req.params.build
+		if build?
+			url = "/project/#{project_id}/build/#{build}/output/#{req.params.file}"
+		else
+			url = "/project/#{project_id}/output/#{req.params.file}"
+		CompileController.proxyToClsi(project_id, url, req, res, next)
 
-	proxySync: (req, res, next = (error) ->) ->
-		CompileController.proxyToClsi(req.params.Project_id, req.url, req, res, next)
+	proxySyncPdf: (req, res, next = (error) ->) ->
+		project_id = req.params.Project_id
+		{page, h, v} = req.query
+		if not page?.match(/^\d+$/)
+			return next(new Error("invalid page parameter"))
+		if not h?.match(/^\d+\.\d+$/)
+			return next(new Error("invalid h parameter"))
+		if not v?.match(/^\d+\.\d+$/)
+			return next(new Error("invalid v parameter"))
+		destination = {url: "/project/#{project_id}/sync/pdf", qs: {page, h, v}}
+		CompileController.proxyToClsi(project_id, destination, req, res, next)
+
+	proxySyncCode: (req, res, next = (error) ->) ->
+		project_id = req.params.Project_id
+		{file, line, column} = req.query
+		if not file? or Path.resolve("/", file) isnt "/#{file}"
+			return next(new Error("invalid file parameter"))
+		if not line?.match(/^\d+$/)
+			return next(new Error("invalid line parameter"))
+		if not column?.match(/^\d+$/)
+			return next(new Error("invalid column parameter"))
+		destination = {url:"/project/#{project_id}/sync/code", qs: {file, line, column}}
+		CompileController.proxyToClsi(project_id, destination, req, res, next)
 
 	proxyToClsi: (project_id, url, req, res, next = (error) ->) ->
 		if req.query?.compileGroup
@@ -107,30 +131,39 @@ module.exports = CompileController =
 				CompileController.proxyToClsiWithLimits(project_id, url, limits, req, res, next)
 
 	proxyToClsiWithLimits: (project_id, url, limits, req, res, next = (error) ->) ->
-		if limits.compileGroup == "priority"
-			compilerUrl = Settings.apis.clsi_priority.url
-		else
-			compilerUrl = Settings.apis.clsi.url
-		url = "#{compilerUrl}#{url}"
-		logger.log url: url, "proxying to CLSI"
-		oneMinute = 60 * 1000
-		# the base request
-		options = { url: url, method: req.method,	timeout: oneMinute }
-		# if we have a build parameter, pass it through to the clsi
-		if req.query?.pdfng && req.query?.build? # only for new pdf viewer
-			options.qs = {}
-			options.qs.build = req.query.build
-		# if we are byte serving pdfs, pass through If-* and Range headers
-		# do not send any others, there's a proxying loop if Host: is passed!
-		if req.query?.pdfng
-			newHeaders = {}
-			for h, v of req.headers
-				newHeaders[h] = req.headers[h] if h.match /^(If-|Range)/i
-			options.headers = newHeaders
-		proxy = request(options)
-		proxy.pipe(res)
-		proxy.on "error", (error) ->
-			logger.warn err: error, url: url, "CLSI proxy error"
+		ClsiCookieManager.getCookieJar project_id, (err, jar)->
+			if err?
+				logger.err err:err, "error getting cookie jar for clsi request"
+				return callback(err)
+			# expand any url parameter passed in as {url:..., qs:...}
+			if typeof url is "object"
+				{url, qs} = url
+			if limits.compileGroup == "priority"
+				compilerUrl = Settings.apis.clsi_priority.url
+			else
+				compilerUrl = Settings.apis.clsi.url
+			url = "#{compilerUrl}#{url}"
+			logger.log url: url, "proxying to CLSI"
+			oneMinute = 60 * 1000
+			# the base request
+			options = { url: url, method: req.method, timeout: oneMinute, jar : jar }
+			# add any provided query string
+			options.qs = qs if qs?
+			# if we have a build parameter, pass it through to the clsi
+			if req.query?.pdfng && req.query?.build? # only for new pdf viewer
+				options.qs ?= {}
+				options.qs.build = req.query.build
+			# if we are byte serving pdfs, pass through If-* and Range headers
+			# do not send any others, there's a proxying loop if Host: is passed!
+			if req.query?.pdfng
+				newHeaders = {}
+				for h, v of req.headers
+					newHeaders[h] = req.headers[h] if h.match /^(If-|Range)/i
+				options.headers = newHeaders
+			proxy = request(options)
+			proxy.pipe(res)
+			proxy.on "error", (error) ->
+				logger.warn err: error, url: url, "CLSI proxy error"
 
 	wordCount: (req, res, next) ->
 		project_id = req.params.Project_id

services/web/app/coffee/Features/Compile/CompileManager.coffee

@@ -1,8 +1,6 @@
 Settings = require('settings-sharelatex')
-
 redis = require("redis-sharelatex")
 rclient = redis.createClient(Settings.redis.web)
-
 DocumentUpdaterHandler = require "../DocumentUpdater/DocumentUpdaterHandler"
 Project = require("../../models/Project").Project
 ProjectRootDocManager = require "../Project/ProjectRootDocManager"
@@ -13,6 +11,8 @@ logger = require("logger-sharelatex")
 rateLimiter = require("../../infrastructure/RateLimiter")
 
 module.exports = CompileManager =
+
+
 	compile: (project_id, user_id, options = {}, _callback = (error) ->) ->
 		timer = new Metrics.Timer("editor.compile")
 		callback = (args...) ->
@@ -26,7 +26,8 @@ module.exports = CompileManager =
 			CompileManager._checkIfRecentlyCompiled project_id, user_id, (error, recentlyCompiled) ->
 				return callback(error) if error?
 				if recentlyCompiled
-					return callback new Error("project was recently compiled so not continuing")
+					logger.warn {project_id, user_id}, "project was recently compiled so not continuing"
+					return callback null, "too-recently-compiled", []
 				
 				CompileManager._ensureRootDocumentIsSet project_id, (error) ->
 					return callback(error) if error?
@@ -36,10 +37,10 @@ module.exports = CompileManager =
 							return callback(error) if error?
 							for key, value of limits
 								options[key] = value
-							ClsiManager.sendRequest project_id, options, (error, status, outputFiles, output) ->
+							ClsiManager.sendRequest project_id, options, (error, status, outputFiles, clsiServerId) ->
 								return callback(error) if error?
 								logger.log files: outputFiles, "output files"
-								callback(null, status, outputFiles, output, limits)
+								callback(null, status, outputFiles, clsiServerId, limits)
 								
 	deleteAuxFiles: (project_id, callback = (error) ->) ->
 		CompileManager.getProjectCompileLimits project_id, (error, limits) ->

services/web/app/coffee/Features/Documents/DocumentController.coffee

@@ -1,5 +1,4 @@
 ProjectEntityHandler = require "../Project/ProjectEntityHandler"
-Errors = require "../../errors"
 logger = require("logger-sharelatex")
 
 module.exports =

services/web/app/coffee/Features/Editor/EditorController.coffee

@@ -164,12 +164,17 @@ module.exports = EditorController =
 
 	moveEntity: (project_id, entity_id, folder_id, entityType, callback)->
 		Metrics.inc "editor.move-entity"
-		ProjectEntityHandler.moveEntity project_id, entity_id, folder_id, entityType, =>
+		LockManager.getLock project_id, (err)->
 			if err?
-				logger.err err:err, project_id:project_id, entity_id:entity_id, folder_id:folder_id, "error moving entity"
+				logger.err err:err, project_id:project_id, "could not get lock for move entity"
 				return callback(err)
-			EditorRealTimeController.emitToRoom project_id, 'reciveEntityMove', entity_id, folder_id
-			callback?()
+			ProjectEntityHandler.moveEntity project_id, entity_id, folder_id, entityType, =>
+				if err?
+					logger.err err:err, project_id:project_id, entity_id:entity_id, folder_id:folder_id, "error moving entity"
+					return callback(err)
+				LockManager.releaseLock project_id, ->
+					EditorRealTimeController.emitToRoom project_id, 'reciveEntityMove', entity_id, folder_id
+					callback?()
 
 	renameProject: (project_id, newName, callback = (err) ->) ->
 		ProjectDetailsHandler.renameProject project_id, newName, ->

services/web/app/coffee/Features/Editor/EditorHttpController.coffee

@@ -5,14 +5,18 @@ EditorRealTimeController = require "./EditorRealTimeController"
 EditorController = require "./EditorController"
 ProjectGetter = require('../Project/ProjectGetter')
 UserGetter = require('../User/UserGetter')
-AuthorizationManager = require("../Security/AuthorizationManager")
+AuthorizationManager = require("../Authorization/AuthorizationManager")
 ProjectEditorHandler = require('../Project/ProjectEditorHandler')
 Metrics = require('../../infrastructure/Metrics')
+CollaboratorsHandler = require("../Collaborators/CollaboratorsHandler")
+PrivilegeLevels = require "../Authorization/PrivilegeLevels"
 
 module.exports = EditorHttpController =
 	joinProject: (req, res, next) ->
 		project_id = req.params.Project_id
 		user_id = req.query.user_id
+		if user_id == "anonymous-user"
+			user_id = null
 		logger.log {user_id, project_id}, "join project request"
 		Metrics.inc "editor.join-project"
 		EditorHttpController._buildJoinProjectView project_id, user_id, (error, project, privilegeLevel) ->
@@ -29,17 +33,17 @@ module.exports = EditorHttpController =
 		ProjectGetter.getProjectWithoutDocLines project_id, (error, project) ->
 			return callback(error) if error?
 			return callback(new Error("not found")) if !project?
-			ProjectGetter.populateProjectWithUsers project, (error, project) ->
+			CollaboratorsHandler.getMembersWithPrivilegeLevels project, (error, members) ->
 				return callback(error) if error?
 				UserGetter.getUser user_id, { isAdmin: true }, (error, user) ->
 					return callback(error) if error?
-					AuthorizationManager.getPrivilegeLevelForProject project, user, (error, canAccess, privilegeLevel) ->
+					AuthorizationManager.getPrivilegeLevelForProject user_id, project_id, (error, privilegeLevel) ->
 						return callback(error) if error?
-						if !canAccess
+						if !privilegeLevel? or privilegeLevel == PrivilegeLevels.NONE
 							callback null, null, false
 						else
 							callback(null,
-								ProjectEditorHandler.buildProjectModelView(project),
+								ProjectEditorHandler.buildProjectModelView(project, members),
 								privilegeLevel
 							)
 

services/web/app/coffee/Features/Editor/EditorRouter.coffee

@@ -1,20 +1,20 @@
 EditorHttpController = require('./EditorHttpController')
-SecurityManager = require('../../managers/SecurityManager')
 AuthenticationController = require "../Authentication/AuthenticationController"
+AuthorizationMiddlewear = require('../Authorization/AuthorizationMiddlewear')
 
 module.exports =
 	apply: (webRouter, apiRouter) ->
-		webRouter.post   '/project/:Project_id/doc', SecurityManager.requestCanModifyProject, EditorHttpController.addDoc
-		webRouter.post   '/project/:Project_id/folder', SecurityManager.requestCanModifyProject, EditorHttpController.addFolder
+		webRouter.post   '/project/:Project_id/doc', AuthorizationMiddlewear.ensureUserCanWriteProjectContent, EditorHttpController.addDoc
+		webRouter.post   '/project/:Project_id/folder', AuthorizationMiddlewear.ensureUserCanWriteProjectContent, EditorHttpController.addFolder
 
-		webRouter.post   '/project/:Project_id/:entity_type/:entity_id/rename', SecurityManager.requestCanModifyProject, EditorHttpController.renameEntity
-		webRouter.post   '/project/:Project_id/:entity_type/:entity_id/move', SecurityManager.requestCanModifyProject, EditorHttpController.moveEntity
+		webRouter.post   '/project/:Project_id/:entity_type/:entity_id/rename', AuthorizationMiddlewear.ensureUserCanWriteProjectContent, EditorHttpController.renameEntity
+		webRouter.post   '/project/:Project_id/:entity_type/:entity_id/move', AuthorizationMiddlewear.ensureUserCanWriteProjectContent, EditorHttpController.moveEntity
 
-		webRouter.delete '/project/:Project_id/file/:entity_id', SecurityManager.requestCanModifyProject, EditorHttpController.deleteFile
-		webRouter.delete '/project/:Project_id/doc/:entity_id', SecurityManager.requestCanModifyProject, EditorHttpController.deleteDoc
-		webRouter.delete '/project/:Project_id/folder/:entity_id', SecurityManager.requestCanModifyProject, EditorHttpController.deleteFolder
+		webRouter.delete '/project/:Project_id/file/:entity_id', AuthorizationMiddlewear.ensureUserCanWriteProjectContent, EditorHttpController.deleteFile
+		webRouter.delete '/project/:Project_id/doc/:entity_id', AuthorizationMiddlewear.ensureUserCanWriteProjectContent, EditorHttpController.deleteDoc
+		webRouter.delete '/project/:Project_id/folder/:entity_id', AuthorizationMiddlewear.ensureUserCanWriteProjectContent, EditorHttpController.deleteFolder
 
-		webRouter.post   '/project/:Project_id/doc/:doc_id/restore', SecurityManager.requestCanModifyProject, EditorHttpController.restoreDoc
+		webRouter.post   '/project/:Project_id/doc/:doc_id/restore', AuthorizationMiddlewear.ensureUserCanWriteProjectContent, EditorHttpController.restoreDoc
 
 		# Called by the real-time API to load up the current project state.
 		# This is a post request because it's more than just a getting of data. We take actions

services/web/app/coffee/Features/Email/EmailSender.coffee

@@ -3,6 +3,8 @@ metrics = require('../../infrastructure/Metrics')
 Settings = require('settings-sharelatex')
 nodemailer = require("nodemailer")
 sesTransport = require('nodemailer-ses-transport')
+sgTransport = require('nodemailer-sendgrid-transport')
+
 _ = require("underscore")
 
 if Settings.email? and Settings.email.fromAddress?
@@ -19,6 +21,9 @@ client =
 if Settings?.email?.parameters?.AWSAccessKeyID?
 	logger.log "using aws ses for email"
 	nm_client = nodemailer.createTransport(sesTransport(Settings.email.parameters))
+else if Settings?.email?.parameters?.sendgridApiKey?
+	logger.log "using sendgrid for email"
+	nm_client = nodemailer.createTransport(sgTransport({auth:{api_key:Settings?.email?.parameters?.sendgridApiKey}}))
 else if Settings?.email?.parameters?
 	smtp = _.pick(Settings?.email?.parameters, "host", "port", "secure", "auth")
 

services/web/app/coffee/Features/Errors/ErrorController.coffee

@@ -1,5 +1,25 @@
+Errors = require "./Errors"
+logger = require "logger-sharelatex"
+
 module.exports = ErrorController =
 	notFound: (req, res)->
-		res.statusCode = 404
+		res.status(404)
 		res.render 'general/404',
-			title: "page_not_found"
+			title: "page_not_found"
+
+	serverError: (req, res)->
+		res.status(500)
+		res.render 'general/500',
+			title: "Server Error"
+	
+	handleError: (error, req, res, next) ->
+		if error?.code is 'EBADCSRFTOKEN'
+			logger.warn err: error,url:req.url, method:req.method, user:req?.sesson?.user, "invalid csrf"
+			res.sendStatus(403)
+			return
+		if error instanceof Errors.NotFoundError
+			logger.warn {err: error, url: req.url}, "not found error"
+			ErrorController.notFound req, res
+		else
+			logger.error err: error, url:req.url, method:req.method, user:req?.sesson?.user, "error passed to top level next middlewear"
+			ErrorController.serverError req, res

services/web/app/coffee/Features/Errors/Errors.coffee

@@ -6,5 +6,4 @@ NotFoundError = (message) ->
 NotFoundError.prototype.__proto__ = Error.prototype
 
 module.exports = Errors =
-	NotFoundError: NotFoundError
-
+	NotFoundError: NotFoundError

services/web/app/coffee/Features/FileStore/FileStoreHandler.coffee

@@ -42,7 +42,14 @@ module.exports = FileStoreHandler =
 			method : "get"
 			uri: "#{@_buildUrl(project_id, file_id)}#{queryString}"
 			timeout:fiveMinsInMs
+			headers: {}
+		if query? and query['range']?
+			rangeText = query['range']
+			if rangeText && rangeText.match? && rangeText.match(/\d+-\d+/)
+				opts.headers['range'] = "bytes=#{query['range']}"
 		readStream = request(opts)
+		readStream.on "error", (err) ->
+			logger.err {err, project_id, file_id, query}, "error in file stream"
 		callback(null, readStream)
 
 	deleteFile: (project_id, file_id, callback)->

services/web/app/coffee/Features/Project/ProjectController.coffee

@@ -5,7 +5,6 @@ projectDuplicator = require("./ProjectDuplicator")
 projectCreationHandler = require("./ProjectCreationHandler")
 editorController = require("../Editor/EditorController")
 metrics = require('../../infrastructure/Metrics')
-Project = require('../../models/Project').Project
 User = require('../../models/User').User
 TagsHandler = require("../Tags/TagsHandler")
 SubscriptionLocator = require("../Subscription/SubscriptionLocator")
@@ -13,10 +12,12 @@ NotificationsHandler = require("../Notifications/NotificationsHandler")
 LimitationsManager = require("../Subscription/LimitationsManager")
 _ = require("underscore")
 Settings = require("settings-sharelatex")
-SecurityManager = require("../../managers/SecurityManager")
+AuthorizationManager = require("../Authorization/AuthorizationManager")
 fs = require "fs"
 InactiveProjectManager = require("../InactiveData/InactiveProjectManager")
 ProjectUpdateHandler = require("./ProjectUpdateHandler")
+ProjectGetter = require("./ProjectGetter")
+PrivilegeLevels = require("../Authorization/PrivilegeLevels")
 
 module.exports = ProjectController =
 
@@ -41,6 +42,14 @@ module.exports = ProjectController =
 			jobs.push (callback) ->
 				editorController.setRootDoc project_id, req.body.rootDocId, callback
 
+		async.series jobs, (error) ->
+			return next(error) if error?
+			res.sendStatus(204)
+			
+	updateProjectAdminSettings: (req, res, next) ->
+		project_id = req.params.Project_id
+		
+		jobs = []
 		if req.body.publicAccessLevel?
 			jobs.push (callback) ->
 				editorController.setPublicAccessLevel project_id, req.body.publicAccessLevel, callback
@@ -129,7 +138,7 @@ module.exports = ProjectController =
 			notifications: (cb)->
 				NotificationsHandler.getUserNotifications user_id, cb
 			projects: (cb)->
-				Project.findAllUsersProjects user_id, 'name lastUpdated publicAccesLevel archived owner_ref', cb
+				ProjectGetter.findAllUsersProjects user_id, 'name lastUpdated publicAccesLevel archived owner_ref', cb
 			hasSubscription: (cb)->
 				LimitationsManager.userHasSubscriptionOrIsGroupMember req.session.user, cb
 			user: (cb) ->
@@ -179,23 +188,23 @@ module.exports = ProjectController =
 			anonymous = false
 		else
 			anonymous = true
-			user_id = 'openUser'
+			user_id = null
 
 		project_id = req.params.Project_id
 		logger.log project_id:project_id, "loading editor"
 
 		async.parallel {
 			project: (cb)->
-				Project.findPopulatedById project_id, cb
+				ProjectGetter.getProject project_id, { name: 1, lastUpdated: 1}, cb
 			user: (cb)->
-				if user_id == 'openUser'
+				if !user_id?
 					cb null, defaultSettingsForAnonymousUser(user_id)
 				else
 					User.findById user_id, (err, user)->
 						logger.log project_id:project_id, user_id:user_id, "got user"
 						cb err, user
 			subscription: (cb)->
-				if user_id == 'openUser'
+				if !user_id?
 					return cb()
 				SubscriptionLocator.getUsersSubscription user_id, cb
 			activate: (cb)->
@@ -216,8 +225,9 @@ module.exports = ProjectController =
 			daysSinceLastUpdated =  (new Date() - project.lastUpdated) /86400000
 			logger.log project_id:project_id, daysSinceLastUpdated:daysSinceLastUpdated, "got db results for loading editor"
 
-			SecurityManager.userCanAccessProject user, project, (canAccess, privilegeLevel)->
-				if !canAccess
+			AuthorizationManager.getPrivilegeLevelForProject user_id, project_id, (error, privilegeLevel)->
+				return next(error) if error?
+				if !privilegeLevel? or privilegeLevel == PrivilegeLevels.NONE
 					return res.sendStatus 401
 
 				if subscription? and subscription.freeTrial? and subscription.freeTrial.expiresAt?
@@ -228,10 +238,9 @@ module.exports = ProjectController =
 					title:  project.name
 					priority_title: true
 					bodyClasses: ["editor"]
-					project : project
 					project_id : project._id
 					user : {
-						id    : user.id
+						id    : user_id
 						email : user.email
 						first_name : user.first_name
 						last_name  : user.last_name
@@ -239,6 +248,8 @@ module.exports = ProjectController =
 						subscription :
 							freeTrial: {allowed: allowedFreeTrial}
 						featureSwitches: user.featureSwitches
+						features: user.features
+						refProviders: user.refProviders
 					}
 					userSettings: {
 						mode  : user.ace.mode

services/web/app/coffee/Features/Project/ProjectDeleter.coffee

@@ -5,6 +5,7 @@ documentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandler')
 tagsHandler = require("../Tags/TagsHandler")
 async = require("async")
 FileStoreHandler = require("../FileStore/FileStoreHandler")
+CollaboratorsHandler = require("../Collaborators/CollaboratorsHandler")
 
 module.exports = ProjectDeleter =
 
@@ -44,16 +45,10 @@ module.exports = ProjectDeleter =
 					(cb)->
 						documentUpdaterHandler.flushProjectToMongoAndDelete project_id, cb
 					(cb)->
-						tagsHandler.removeProjectFromAllTags project.owner_ref, project_id, (err)->
+						CollaboratorsHandler.getMemberIds project_id, (error, member_ids = []) ->
+							for member_id in member_ids
+								tagsHandler.removeProjectFromAllTags member_id, project_id, (err)->
 						cb() #doesn't matter if this fails or the order it happens in
-					(cb)->
-						project.collaberator_refs.forEach (collaberator_ref)->
-							tagsHandler.removeProjectFromAllTags collaberator_ref, project_id, ->
-						cb()
-					(cb)->
-						project.readOnly_refs.forEach (readOnly_ref)->
-							tagsHandler.removeProjectFromAllTags readOnly_ref, project_id, ->
-						cb()
 					(cb)->
 						Project.update {_id:project_id}, { $set: { archived: true }}, cb
 				], (err)->

services/web/app/coffee/Features/Project/ProjectDetailsHandler.coffee

@@ -4,6 +4,7 @@ Project = require('../../models/Project').Project
 logger = require("logger-sharelatex")
 tpdsUpdateSender = require '../ThirdPartyDataStore/TpdsUpdateSender'
 _ = require("underscore")
+PublicAccessLevels = require("../Authorization/PublicAccessLevels")
 
 module.exports = 
 
@@ -49,6 +50,6 @@ module.exports =
 
 	setPublicAccessLevel : (project_id, newAccessLevel, callback = ->)->
 		logger.log project_id: project_id, level: newAccessLevel, "set public access level"
-		if project_id? && newAccessLevel? and _.include ['readOnly', 'readAndWrite', 'private'], newAccessLevel
+		if project_id? && newAccessLevel? and _.include [PublicAccessLevels.READ_ONLY, PublicAccessLevels.READ_AND_WRITE, PublicAccessLevels.PRIVATE], newAccessLevel
 			Project.update {_id:project_id},{publicAccesLevel:newAccessLevel}, (err)->
 				callback()

services/web/app/coffee/Features/Project/ProjectDuplicator.coffee

@@ -75,7 +75,7 @@ module.exports = ProjectDuplicator =
 				return callback(err)
 			{originalProject, newProject, originalRootDoc, docContentsArray} = results
 
-			originalRootDoc = originalRootDoc[0]
+			originalRootDoc = originalRootDoc?[0]
 
 			docContents = {}
 			for docContent in docContentsArray

services/web/app/coffee/Features/Project/ProjectEditorHandler.coffee

@@ -1,11 +1,7 @@
 _ = require("underscore")
 
 module.exports = ProjectEditorHandler =
-	buildProjectModelView: (project, options) ->
-		options ||= {}
-		if !options.includeUsers?
-			options.includeUsers = true
-			
+	buildProjectModelView: (project, members) ->
 		result =
 			_id        : project._id
 			name       : project.name
@@ -18,40 +14,27 @@ module.exports = ProjectEditorHandler =
 			spellCheckLanguage: project.spellCheckLanguage
 			deletedByExternalDataSource : project.deletedByExternalDataSource || false
 			deletedDocs: project.deletedDocs
+			members:     []
+		
+		owner = null
+		for member in members
+			if member.privilegeLevel == "owner"
+				owner = member.user
+			else
+				result.members.push @buildUserModelView member.user, member.privilegeLevel
+		if owner?
+			result.owner = @buildUserModelView owner, "owner"
 
-		if options.includeUsers
-			result.features =
-				collaborators: -1 # Infinite
-				versioning: false
-				dropbox:false
-				compileTimeout: 60
-				compileGroup:"standard"
-				templates: false
-				references: false
+		result.features = _.defaults(owner?.features or {}, {
+			collaborators: -1 # Infinite
+			versioning: false
+			dropbox:false
+			compileTimeout: 60
+			compileGroup:"standard"
+			templates: false
+			references: false
+		})
 
-			if project.owner_ref.features?
-				if project.owner_ref.features.collaborators?
-					result.features.collaborators = project.owner_ref.features.collaborators
-				if project.owner_ref.features.versioning?
-					result.features.versioning = project.owner_ref.features.versioning
-				if project.owner_ref.features.dropbox?
-					result.features.dropbox = project.owner_ref.features.dropbox
-				if project.owner_ref.features.compileTimeout?
-					result.features.compileTimeout = project.owner_ref.features.compileTimeout
-				if project.owner_ref.features.compileGroup?
-					result.features.compileGroup = project.owner_ref.features.compileGroup
-				if project.owner_ref.features.templates?
-					result.features.templates = project.owner_ref.features.templates
-				if project.owner_ref.features.references?
-					result.features.references = project.owner_ref.features.references
-
-					
-			result.owner = @buildUserModelView project.owner_ref, "owner"
-			result.members = []
-			for ref in project.readOnly_refs
-				result.members.push @buildUserModelView ref, "readOnly"
-			for ref in project.collaberator_refs
-				result.members.push @buildUserModelView ref, "readAndWrite"
 		return result
 
 	buildUserModelView: (user, privileges) ->
@@ -63,12 +46,12 @@ module.exports = ProjectEditorHandler =
 		signUpDate : user.signUpDate
 
 	buildFolderModelView: (folder) ->
-		fileRefs = _.filter folder.fileRefs, (file)-> file?
+		fileRefs = _.filter (folder.fileRefs or []), (file)-> file?
 		_id        : folder._id
 		name       : folder.name
-		folders    : @buildFolderModelView childFolder for childFolder in folder.folders
+		folders    : @buildFolderModelView childFolder for childFolder in (folder.folders or [])
 		fileRefs   : @buildFileModelView file for file in fileRefs
-		docs       : @buildDocModelView doc for doc in folder.docs
+		docs       : @buildDocModelView doc for doc in (folder.docs or [])
 
 	buildFileModelView: (file) ->
 		_id  : file._id

services/web/app/coffee/Features/Project/ProjectEntityHandler.coffee

@@ -4,7 +4,7 @@ Doc = require('../../models/Doc').Doc
 Folder = require('../../models/Folder').Folder
 File = require('../../models/File').File
 FileStoreHandler = require("../FileStore/FileStoreHandler")
-Errors = require "../../errors"
+Errors = require "../Errors/Errors"
 tpdsUpdateSender = require('../ThirdPartyDataStore/TpdsUpdateSender')
 projectLocator = require('./ProjectLocator')
 path = require "path"
@@ -22,7 +22,9 @@ module.exports = ProjectEntityHandler =
 		folders = {}
 		processFolder = (basePath, folder) ->
 			folders[basePath] = folder
-			processFolder path.join(basePath, childFolder.name), childFolder for childFolder in folder.folders
+			for childFolder in (folder.folders or [])
+				if childFolder.name?
+					processFolder path.join(basePath, childFolder.name), childFolder
 
 		ProjectGetter.getProjectWithoutDocLines project_id, (err, project) ->
 			return callback(err) if err?
@@ -43,11 +45,11 @@ module.exports = ProjectEntityHandler =
 			for docContent in docContentsArray
 				docContents[docContent._id] = docContent
 
-			ProjectEntityHandler.getAllFolders project_id, (error, folders) ->
+			ProjectEntityHandler.getAllFolders project_id, (error, folders = {}) ->
 				return callback(error) if error?
 				docs = {}
 				for folderPath, folder of folders
-					for doc in folder.docs
+					for doc in (folder.docs or [])
 						content = docContents[doc._id.toString()]
 						if content?
 							docs[path.join(folderPath, doc.name)] = {
@@ -61,11 +63,11 @@ module.exports = ProjectEntityHandler =
 
 	getAllFiles: (project_id, callback) ->
 		logger.log project_id:project_id, "getting all files for project"
-		@getAllFolders project_id, (err, folders) ->
+		@getAllFolders project_id, (err, folders = {}) ->
 			return callback(err) if err?
 			files = {}
 			for folderPath, folder of folders
-				for file in folder.fileRefs
+				for file in (folder.fileRefs or [])
 					if file?
 						files[path.join(folderPath, file.name)] = file
 			callback null, files
@@ -344,15 +346,18 @@ module.exports = ProjectEntityHandler =
 					return callback(error) if error?
 					self._removeElementFromMongoArray Project, project_id, path.mongo, (err)->
 						return callback(err) if err?
-						ProjectEntityHandler._putElement project, destinationFolder_id, entity, entityType, (err, result)->
+						# We've updated the project structure by removing the element, so must refresh it.
+						ProjectGetter.getProject project_id, {rootFolder:true, name:true}, (err, project)=>
 							return callback(err) if err?
-							opts = 
-								project_id:project_id
-								project_name:project.name
-								startPath:path.fileSystem
-								endPath:result.path.fileSystem,
-								rev:entity.rev
-							tpdsUpdateSender.moveEntity opts, callback
+							ProjectEntityHandler._putElement project, destinationFolder_id, entity, entityType, (err, result)->
+								return callback(err) if err?
+								opts = 
+									project_id:project_id
+									project_name:project.name
+									startPath:path.fileSystem
+									endPath:result.path.fileSystem,
+									rev:entity.rev
+								tpdsUpdateSender.moveEntity opts, callback
 
 	deleteEntity: (project_id, entity_id, entityType, callback = (error) ->)->
 		self = @
@@ -501,7 +506,7 @@ module.exports = ProjectEntityHandler =
 				elementType = "fileRefs"
 			return elementType
 
-		if !element?
+		if !element? or !element._id?
 			e = new Error("no element passed to be inserted")
 			logger.err project_id:project._id, folder_id:folder_id, element:element, type:type, "failed trying to insert element as it was null"
 			return callback(e)
@@ -520,13 +525,13 @@ module.exports = ProjectEntityHandler =
 				newPath =
 					fileSystem: "#{path.fileSystem}/#{element.name}"
 					mongo: path.mongo
-				logger.log project_id: project._id, element_id: element._id, fileType: type, folder_id: folder_id, "adding element to project"
 				id = element._id+''
 				element._id = require('mongoose').Types.ObjectId(id)
 				conditions = _id:project._id
 				mongopath = "#{path.mongo}.#{type}"
 				update = "$push":{}
 				update["$push"][mongopath] = element
+				logger.log project_id: project._id, element_id: element._id, fileType: type, folder_id: folder_id, mongopath:mongopath, "adding element to project"
 				Project.update conditions, update, {}, (err)->
 					if err?
 						logger.err err: err, project_id: project._id, 'error saving in putElement project'

services/web/app/coffee/Features/Project/ProjectGetter.coffee

@@ -2,11 +2,9 @@ mongojs = require("../../infrastructure/mongojs")
 db = mongojs.db
 ObjectId = mongojs.ObjectId
 async = require "async"
-Errors = require("../../errors")
+Project = require("../../models/Project").Project
 logger = require("logger-sharelatex")
 
-
-
 module.exports = ProjectGetter =
 	EXCLUDE_DEPTH: 8
 
@@ -51,42 +49,11 @@ module.exports = ProjectGetter =
 				return callback(err)
 			callback(null, project?[0])
 
-	populateProjectWithUsers: (project, callback=(error, project) ->) ->
-		# eventually this should be in a UserGetter.getUser module
-		getUser = (user_id, callback=(error, user) ->) ->
-			unless user_id instanceof ObjectId
-				user_id = ObjectId(user_id)
-			db.users.find _id: user_id, (error, users = []) ->
-				callback error, users[0]
-
-		jobs = []
-		jobs.push (callback) ->
-			getUser project.owner_ref, (error, user) ->
+	
+	findAllUsersProjects: (user_id, fields, callback = (error, ownedProjects, readAndWriteProjects, readOnlyProjects) ->) ->
+		CollaboratorsHandler = require "../Collaborators/CollaboratorsHandler"
+		Project.find {owner_ref: user_id}, fields, (error, projects) ->
+			return callback(error) if error?
+			CollaboratorsHandler.getProjectsUserIsCollaboratorOf user_id, fields, (error, readAndWriteProjects, readOnlyProjects) ->
 				return callback(error) if error?
-				if user?
-					project.owner_ref = user
-				callback null, project
-
-		readOnly_refs = project.readOnly_refs
-		project.readOnly_refs = []
-		for readOnly_ref in readOnly_refs
-			do (readOnly_ref) ->
-				jobs.push (callback) ->
-					getUser readOnly_ref, (error, user) ->
-						return callback(error) if error?
-						if user?
-							project.readOnly_refs.push user
-						callback null, project
-						
-		collaberator_refs = project.collaberator_refs
-		project.collaberator_refs = []
-		for collaberator_ref in collaberator_refs
-			do (collaberator_ref) ->
-				jobs.push (callback) ->
-					getUser collaberator_ref, (error, user) ->
-						return callback(error) if error?
-						if user?
-							project.collaberator_refs.push user
-						callback null, project
-
-		async.parallelLimit jobs, 3, (error) -> callback error, project
+				callback null, projects, readAndWriteProjects, readOnlyProjects

services/web/app/coffee/Features/Project/ProjectLocator.coffee

@@ -1,9 +1,10 @@
 Project = require('../../models/Project').Project
 ProjectGetter = require("./ProjectGetter")
-Errors = require "../../errors"
+Errors = require "../Errors/Errors"
 _ = require('underscore')
 logger = require('logger-sharelatex')
 async = require('async')
+ProjectGetter = require "./ProjectGetter"
 
 module.exports = ProjectLocator =
 	findElement: (options, _callback = (err, element, path, parentFolder)->)->
@@ -60,7 +61,13 @@ module.exports = ProjectLocator =
 	findRootDoc : (opts, callback)->
 		getRootDoc = (project)=>
 			if project.rootDoc_id?
-				@findElement {project:project, element_id:project.rootDoc_id, type:"docs"}, callback
+				@findElement {project:project, element_id:project.rootDoc_id, type:"docs"}, (error, args...) ->
+					if error?
+						if error instanceof Errors.NotFoundError
+							return callback null, null
+						else
+							return callback error
+					return callback null, args...
 			else
 				callback null, null
 		{project, project_id} = opts
@@ -131,7 +138,8 @@ module.exports = ProjectLocator =
 			async.waterfall jobs, callback
 
 	findUsersProjectByName: (user_id, projectName, callback)->
-		Project.findAllUsersProjects user_id, 'name archived', (err, projects, collabertions=[])->
+		ProjectGetter.findAllUsersProjects user_id, 'name archived', (err, projects, collabertions=[])->
+			return callback(error) if error?
 			projects = projects.concat(collabertions)
 			projectName = projectName.toLowerCase()
 			project = _.find projects, (project)->

services/web/app/coffee/Features/Referal/ReferalAllocator.coffee

@@ -35,7 +35,7 @@ module.exports = ReferalAllocator =
 		query = _id: user_id
 		User.findOne query, (error, user) ->
 			return callback(error) if error
-			return callback(new Error("user not found")) if !user?
+			return callback(new Error("user not found #{user_id} for assignBonus")) if !user?
 			logger.log user_id: user_id, refered_user_count: user.refered_user_count, "assigning bonus"
 			if user.refered_user_count? and user.refered_user_count > 0
 				newFeatures = ReferalAllocator._calculateFeatures(user)

services/web/app/coffee/Features/References/ReferencesHandler.coffee

@@ -1,7 +1,8 @@
 logger = require("logger-sharelatex")
 request = require("request")
 settings = require("settings-sharelatex")
-Project = require("../../models/Project").Project
+ProjectGetter = require "../Project/ProjectGetter"
+UserGetter = require "../User/UserGetter"
 DocumentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandler')
 U = require('underscore')
 Async = require('async')
@@ -15,42 +16,56 @@ module.exports = ReferencesHandler =
 	_buildDocUrl: (projectId, docId) ->
 		"#{settings.apis.docstore.url}/project/#{projectId}/doc/#{docId}/raw"
 
+	_buildFileUrl: (projectId, fileId) ->
+		"#{settings.apis.filestore.url}/project/#{projectId}/file/#{fileId}"
+
+	_findBibFileIds: (project) ->
+		ids = []
+		_process = (folder) ->
+			(folder.fileRefs or []).forEach (file) ->
+				if file?.name?.match(/^.*\.bib$/)
+					ids.push(file._id)
+			(folder.folders or []).forEach (folder) ->
+				_process(folder)
+		(project.rootFolder or []).forEach (rootFolder) ->
+			_process(rootFolder)
+		return ids
+
 	_findBibDocIds: (project) ->
 		ids = []
-
 		_process = (folder) ->
-			folder.docs.forEach (doc) ->
+			(folder.docs or []).forEach (doc) ->
 				if doc?.name?.match(/^.*\.bib$/)
 					ids.push(doc._id)
-			folder.folders.forEach (folder) ->
+			(folder.folders or []).forEach (folder) ->
 				_process(folder)
-
-		project.rootFolder.forEach (rootFolder) ->
+		(project.rootFolder or []).forEach (rootFolder) ->
 			_process(rootFolder)
-
 		return ids
 
 	_isFullIndex: (project, callback = (err, result) ->) ->
-		owner = project.owner_ref
-		callback(null, owner.features.references == true)
+		UserGetter.getUser project.owner_ref, { features: true }, (err, owner) ->
+			return callback(err) if err?
+			callback(null, owner?.features?.references == true)
 
 	indexAll: (projectId, callback=(err, data)->) ->
-		Project.findPopulatedById projectId, (err, project) ->
+		ProjectGetter.getProject projectId, {rootFolder: true, owner_ref: 1}, (err, project) ->
 			if err
 				logger.err {err, projectId}, "error finding project"
 				return callback(err)
 			logger.log {projectId}, "indexing all bib files in project"
 			docIds = ReferencesHandler._findBibDocIds(project)
-			ReferencesHandler._doIndexOperation(projectId, project, docIds, callback)
+			fileIds = ReferencesHandler._findBibFileIds(project)
+			ReferencesHandler._doIndexOperation(projectId, project, docIds, fileIds, callback)
 
 	index: (projectId, docIds, callback=(err, data)->) ->
-		Project.findPopulatedById projectId, (err, project) ->
+		ProjectGetter.getProject projectId, {rootFolder: true, owner_ref: 1}, (err, project) ->
 			if err
 				logger.err {err, projectId}, "error finding project"
 				return callback(err)
-			ReferencesHandler._doIndexOperation(projectId, project, docIds, callback)
+			ReferencesHandler._doIndexOperation(projectId, project, docIds, [], callback)
 
-	_doIndexOperation: (projectId, project, docIds, callback) ->
+	_doIndexOperation: (projectId, project, docIds, fileIds, callback) ->
 		ReferencesHandler._isFullIndex project, (err, isFullIndex) ->
 			if err
 				logger.err {err, projectId}, "error checking whether to do full index"
@@ -65,11 +80,14 @@ module.exports = ReferencesHandler =
 						return callback(err)
 					bibDocUrls = docIds.map (docId) ->
 						ReferencesHandler._buildDocUrl projectId, docId
+					bibFileUrls = fileIds.map (fileId) ->
+						ReferencesHandler._buildFileUrl projectId, fileId
+					allUrls = bibDocUrls.concat(bibFileUrls)
 					logger.log {projectId, isFullIndex, docIds, bibDocUrls}, "sending request to references service"
 					request.post {
 						url: "#{settings.apis.references.url}/project/#{projectId}/index"
 						json:
-							docUrls: bibDocUrls
+							docUrls: allUrls
 							fullIndex: isFullIndex
 					}, (err, res, data) ->
 						if err

services/web/app/coffee/Features/Security/AuthorizationManager.coffee

@@ -1,38 +0,0 @@
-SecurityManager = require '../../managers/SecurityManager'
-
-module.exports = AuthorizationManager =
-	getPrivilegeLevelForProject: (
-		project, user,
-		callback = (error, canAccess, privilegeLevel)->
-	) ->
-		# This is not tested because eventually this function should be brought into
-		# this module.
-		SecurityManager.userCanAccessProject user, project, (canAccess, privilegeLevel) ->
-			if canAccess
-				callback null, true, privilegeLevel
-			else
-				callback null, false
-	
-	setPrivilegeLevelOnClient: (client, privilegeLevel) ->
-		client.set("privilege_level", privilegeLevel)
-
-	ensureClientCanViewProject: (client, callback = (error, project_id)->) ->
-		@ensureClientHasPrivilegeLevelForProject client, ["owner", "readAndWrite", "readOnly"], callback
-
-	ensureClientCanEditProject: (client, callback = (error, project_id)->) ->
-		@ensureClientHasPrivilegeLevelForProject client, ["owner", "readAndWrite"], callback
-
-	ensureClientCanAdminProject: (client, callback = (error, project_id)->) ->
-		@ensureClientHasPrivilegeLevelForProject client, ["owner"], callback
-	
-	ensureClientHasPrivilegeLevelForProject: (client, levels, callback = (error, project_id)->) ->
-		client.get "privilege_level", (error, level) ->
-			return callback(error) if error?
-			if level?
-				client.get "project_id", (error, project_id) ->
-					return callback(error) if error?
-					if project_id?
-						if levels.indexOf(level) > -1
-							callback null, project_id
-		
-

services/web/app/coffee/Features/Spelling/SpellingController.coffee

@@ -12,4 +12,5 @@ module.exports = SpellingController =
 		request(url: Settings.apis.spelling.url + url, method: req.method, headers: req.headers, json: req.body, timeout:TEN_SECONDS)
 		.on "error", (error) ->
 			logger.error err: error, "Spelling API error"
+			res.status(500).end()
 		.pipe(res)

services/web/app/coffee/Features/Subscription/LimitationsManager.coffee

@@ -3,6 +3,7 @@ Project = require("../../models/Project").Project
 User = require("../../models/User").User
 SubscriptionLocator = require("./SubscriptionLocator")
 Settings = require("settings-sharelatex")
+CollaboratorsHandler = require("../Collaborators/CollaboratorsHandler")
 
 module.exports =
 
@@ -13,16 +14,11 @@ module.exports =
 				callback null, owner.features.collaborators
 			else
 				callback null, Settings.defaultPlanCode.collaborators
-	
-	currentNumberOfCollaboratorsInProject: (project_id, callback) ->
-		Project.findById project_id, 'collaberator_refs readOnly_refs', (error, project) ->
-			return callback(error) if error?
-			callback null, (project.collaberator_refs.length + project.readOnly_refs.length)
 
 	canAddXCollaborators: (project_id, x_collaborators, callback = (error, allowed)->) ->
 		@allowedNumberOfCollaboratorsInProject project_id, (error, allowed_number) =>
 			return callback(error) if error?
-			@currentNumberOfCollaboratorsInProject project_id, (error, current_number) =>
+			CollaboratorsHandler.getCollaboratorCount project_id, (error, current_number) =>
 				return callback(error) if error?
 				if current_number + x_collaborators <= allowed_number or allowed_number < 0
 					callback null, true

services/web/app/coffee/Features/Subscription/SubscriptionController.coffee

@@ -1,4 +1,4 @@
-SecurityManager     = require '../../managers/SecurityManager'
+AuthenticationController = require '../Authentication/AuthenticationController'
 SubscriptionHandler  = require './SubscriptionHandler'
 PlansLocator = require("./PlansLocator")
 SubscriptionViewModelBuilder = require('./SubscriptionViewModelBuilder')
@@ -31,7 +31,7 @@ module.exports = SubscriptionController =
 
 	#get to show the recurly.js page
 	paymentPage: (req, res, next) ->
-		SecurityManager.getCurrentUser req, (error, user) =>
+		AuthenticationController.getLoggedInUser req, (error, user) =>
 			return next(error) if error?
 			plan = PlansLocator.findLocalPlanInSettings(req.query.planCode)
 			LimitationsManager.userHasSubscription user, (err, hasSubscription)->
@@ -48,7 +48,7 @@ module.exports = SubscriptionController =
 							subscription:
 								plan_code : req.query.planCode
 								currency: currency
-							account_code: user.id
+							account_code: user._id
 						}, (error, signature) ->
 							return next(error) if error?
 							res.render "subscriptions/new",
@@ -64,28 +64,16 @@ module.exports = SubscriptionController =
 								showCouponField: req.query.scf
 								showVatField: req.query.svf
 								couponCode:      req.query.cc or ""
-								subscriptionFormOptions: JSON.stringify
-									acceptedCards: ['discover', 'mastercard', 'visa']
-									target      : "#subscribeForm"
-									signature   : signature
-									planCode    : req.query.planCode
-									successURL  : "#{Settings.siteUrl}/user/subscription/create?_csrf=#{req.session._csrf}"
-									accountCode : user.id
-									enableCoupons: true
-									acceptPaypal: true
-									account     :
-										firstName : user.first_name
-										lastName  : user.last_name
-										email     : user.email
+
 
 
 	userSubscriptionPage: (req, res, next) ->
-		SecurityManager.getCurrentUser req, (error, user) =>
+		AuthenticationController.getLoggedInUser req, (error, user) =>
 			return next(error) if error?
 			LimitationsManager.userHasSubscriptionOrIsGroupMember user, (err, hasSubOrIsGroupMember, subscription)->
 				groupLicenceInviteUrl = SubscriptionDomainHandler.getDomainLicencePage(user)
 				if subscription?.customAccount
-					logger.log user: user, "redirecting to plans"
+					logger.log user: user, "redirecting to custom account page"
 					res.redirect "/user/subscription/custom_account"
 				else if groupLicenceInviteUrl? and !hasSubOrIsGroupMember
 					logger.log user:user, "redirecting to group subscription invite page"		
@@ -109,22 +97,26 @@ module.exports = SubscriptionController =
 
 
 	userCustomSubscriptionPage: (req, res, next)->
-		SecurityManager.getCurrentUser req, (error, user) ->
+		AuthenticationController.getLoggedInUser req, (error, user) ->
 			LimitationsManager.userHasSubscriptionOrIsGroupMember user, (err, hasSubOrIsGroupMember, subscription)->
+				if !subscription?
+					err = new Error("subscription null for custom account, user:#{user?._id}")
+					logger.warn err:err, "subscription is null for custom accounts page"
+					return next(err)
 				res.render "subscriptions/custom_account",
 					title: "your_subscription"
 					subscription: subscription
 
 
 	editBillingDetailsPage: (req, res, next) ->
-		SecurityManager.getCurrentUser req, (error, user) ->
+		AuthenticationController.getLoggedInUser req, (error, user) ->
 			return next(error) if error?
 			LimitationsManager.userHasSubscription user, (err, hasSubscription)->
 				if !hasSubscription
 					res.redirect "/user/subscription"
 				else
 					RecurlyWrapper.sign {
-						account_code: user.id
+						account_code: user._id
 					}, (error, signature) ->
 						return next(error) if error?
 						res.render "subscriptions/edit-billing-details",
@@ -135,10 +127,10 @@ module.exports = SubscriptionController =
 							signature  : signature
 							successURL : "#{Settings.siteUrl}/user/subscription/update"
 							user       :
-								id : user.id
+								id : user._id
 
 	createSubscription: (req, res, next)->
-		SecurityManager.getCurrentUser req, (error, user) ->
+		AuthenticationController.getLoggedInUser req, (error, user) ->
 			return callback(error) if error?
 			recurly_token_id = req.body.recurly_token_id
 			subscriptionDetails = req.body.subscriptionDetails
@@ -150,14 +142,14 @@ module.exports = SubscriptionController =
 				res.sendStatus 201
 
 	successful_subscription: (req, res)->
-		SecurityManager.getCurrentUser req, (error, user) =>
+		AuthenticationController.getLoggedInUser req, (error, user) =>
 			SubscriptionViewModelBuilder.buildUsersSubscriptionViewModel user, (error, subscription) ->
 				res.render "subscriptions/successful_subscription",
 					title: "thank_you"
 					subscription:subscription
 
 	cancelSubscription: (req, res, next) ->
-		SecurityManager.getCurrentUser req, (error, user) ->
+		AuthenticationController.getLoggedInUser req, (error, user) ->
 			logger.log user_id:user._id, "canceling subscription"
 			return next(error) if error?
 			SubscriptionHandler.cancelSubscription user, (err)->
@@ -166,7 +158,7 @@ module.exports = SubscriptionController =
 				res.redirect "/user/subscription"
  
 	updateSubscription: (req, res)->
-		SecurityManager.getCurrentUser req, (error, user) ->
+		AuthenticationController.getLoggedInUser req, (error, user) ->
 			return next(error) if error?
 			planCode = req.body.plan_code
 			logger.log planCode: planCode, user_id:user._id, "updating subscription"
@@ -176,7 +168,7 @@ module.exports = SubscriptionController =
 				res.redirect "/user/subscription"
 
 	reactivateSubscription: (req, res)->
-		SecurityManager.getCurrentUser req, (error, user) ->
+		AuthenticationController.getLoggedInUser req, (error, user) ->
 			logger.log user_id:user._id, "reactivating subscription"
 			return next(error) if error?
 			SubscriptionHandler.reactivateSubscription user, (err)->
@@ -195,7 +187,7 @@ module.exports = SubscriptionController =
 			res.sendStatus 200
 
 	renderUpgradeToAnnualPlanPage: (req, res)->
-		SecurityManager.getCurrentUser req, (error, user) ->
+		AuthenticationController.getLoggedInUser req, (error, user) ->
 			LimitationsManager.userHasSubscription user, (err, hasSubscription, subscription)->
 				planCode = subscription?.planCode.toLowerCase()
 				if planCode?.indexOf("annual") != -1
@@ -212,7 +204,7 @@ module.exports = SubscriptionController =
 					planName: planName
 
 	processUpgradeToAnnualPlan: (req, res)->
-		SecurityManager.getCurrentUser req, (error, user) ->
+		AuthenticationController.getLoggedInUser req, (error, user) ->
 			{planName} = req.body
 			coupon_code = Settings.coupon_codes.upgradeToAnnualPromo[planName]
 			annualPlanName = "#{planName}-annual"
@@ -225,7 +217,7 @@ module.exports = SubscriptionController =
 					res.sendStatus 200
 
 	extendTrial: (req, res)->
-		SecurityManager.getCurrentUser req, (error, user) ->
+		AuthenticationController.getLoggedInUser req, (error, user) ->
 			LimitationsManager.userHasSubscription user, (err, hasSubscription, subscription)->
 				SubscriptionHandler.extendTrial subscription, 14, (err)->
 					if err?

services/web/app/coffee/Features/Subscription/SubscriptionGroupController.coffee

@@ -4,6 +4,7 @@ SubscriptionLocator = require("./SubscriptionLocator")
 ErrorsController = require("../Errors/ErrorController")
 SubscriptionDomainHandler = require("./SubscriptionDomainHandler")
 _ = require("underscore")
+async = require("async")
 
 module.exports =
 
@@ -53,19 +54,26 @@ module.exports =
 					subscription: subscription
 
 	renderGroupInvitePage: (req, res)->
-		subscription_id = req.params.subscription_id
+		group_subscription_id = req.params.subscription_id
 		user_id = req.session.user._id
-		licence = SubscriptionDomainHandler.findDomainLicenceBySubscriptionId(subscription_id)
+		licence = SubscriptionDomainHandler.findDomainLicenceBySubscriptionId(group_subscription_id)
 		if !licence?
 			return ErrorsController.notFound(req, res)
-		SubscriptionGroupHandler.isUserPartOfGroup user_id, licence.subscription_id, (err, partOfGroup)->
+		jobs = 
+			partOfGroup: (cb)->
+				SubscriptionGroupHandler.isUserPartOfGroup user_id, licence.group_subscription_id, cb
+			subscription: (cb)->
+				SubscriptionLocator.getUsersSubscription user_id, cb
+		async.series jobs, (err, results)->
+			{partOfGroup, subscription} = results
 			if partOfGroup
 				return res.redirect("/user/subscription/custom_account")
 			else
 				res.render "subscriptions/group/invite",
 					title: "Group Invitation"
-					subscription_id:subscription_id
+					group_subscription_id:group_subscription_id
 					licenceName:licence.name
+					has_personal_subscription: subscription?
 
 	beginJoinGroup: (req, res)->
 		subscription_id = req.params.subscription_id

services/web/app/coffee/Features/Subscription/SubscriptionLocator.coffee

@@ -22,4 +22,7 @@ module.exports =
 		Subscription.findOne _id:subscription_id, callback
 
 	getSubscriptionByMemberIdAndId: (user_id, subscription_id, callback)->
-		Subscription.findOne member_ids: user_id, _id:subscription_id, {_id:1}, callback
+		Subscription.findOne {member_ids: user_id, _id:subscription_id}, {_id:1}, callback
+
+	getGroupSubscriptionMemberOf: (user_id, callback)->
+		Subscription.findOne {member_ids: user_id}, {_id:1, planCode:1}, callback

services/web/app/coffee/Features/Subscription/SubscriptionUpdater.coffee

@@ -11,19 +11,18 @@ ReferalAllocator = require("../Referal/ReferalAllocator")
 
 oneMonthInSeconds = 60 * 60 * 24 * 30
 
-module.exports =
+module.exports = SubscriptionUpdater =
 
 	syncSubscription: (recurlySubscription, adminUser_id, callback) ->
-		self = @
 		logger.log adminUser_id:adminUser_id, recurlySubscription:recurlySubscription, "syncSubscription, creating new if subscription does not exist"
 		SubscriptionLocator.getUsersSubscription adminUser_id, (err, subscription)->
 			if subscription?
 				logger.log  adminUser_id:adminUser_id, recurlySubscription:recurlySubscription, "subscription does exist"
-				self._updateSubscription recurlySubscription, subscription, callback
+				SubscriptionUpdater._updateSubscriptionFromRecurly recurlySubscription, subscription, callback
 			else
 				logger.log  adminUser_id:adminUser_id, recurlySubscription:recurlySubscription, "subscription does not exist, creating a new one"
-				self._createNewSubscription adminUser_id, (err, subscription)->
-					self._updateSubscription recurlySubscription, subscription, callback
+				SubscriptionUpdater._createNewSubscription adminUser_id, (err, subscription)->
+					SubscriptionUpdater._updateSubscriptionFromRecurly recurlySubscription, subscription, callback
 
 	addUserToGroup: (adminUser_id, user_id, callback)->
 		logger.log adminUser_id:adminUser_id, user_id:user_id, "adding user into mongo subscription"
@@ -46,7 +45,8 @@ module.exports =
 			if err?
 				logger.err err:err, searchOps:searchOps, removeOperation:removeOperation, "error removing user from group"
 				return callback(err)
-			UserFeaturesUpdater.updateFeatures user_id, Settings.defaultPlanCode, callback
+			SubscriptionUpdater._setUsersMinimumFeatures user_id, callback
+
 
 	_createNewSubscription: (adminUser_id, callback)->
 		logger.log adminUser_id:adminUser_id, "creating new subscription"
@@ -55,7 +55,7 @@ module.exports =
 		subscription.save (err)->
 			callback err, subscription
 
-	_updateSubscription: (recurlySubscription, subscription, callback)->
+	_updateSubscriptionFromRecurly: (recurlySubscription, subscription, callback)->
 		logger.log recurlySubscription:recurlySubscription, subscription:subscription, "updaing subscription"
 		plan = PlansLocator.findLocalPlanInSettings(recurlySubscription.plan.plan_code)
 		if recurlySubscription.state == "expired"
@@ -71,11 +71,34 @@ module.exports =
 			subscription.groupPlan = true
 			subscription.membersLimit = plan.membersLimit
 		subscription.save ->
-			allIds = _.union subscription.members_id, [subscription.admin_id]
+			allIds = _.union subscription.member_ids, [subscription.admin_id]
 			jobs = allIds.map (user_id)->
 				return (cb)->
-					UserFeaturesUpdater.updateFeatures user_id, subscription.planCode, cb
-			jobs.push (cb)-> ReferalAllocator.assignBonus subscription.admin_id, cb
+					SubscriptionUpdater._setUsersMinimumFeatures user_id, cb
 			async.series jobs, callback
 
+	_setUsersMinimumFeatures: (user_id, callback)->
+		jobs =
+			subscription: (cb)->
+				SubscriptionLocator.getUsersSubscription user_id, cb
+			groupSubscription: (cb)->
+				SubscriptionLocator.getGroupSubscriptionMemberOf user_id, cb
+		async.series jobs, (err, results)->
+			if err?
+				logger.err err:err, user_id:user, "error getting subscription or group for _setUsersMinimumFeatures"
+				return callback(err)
+			{subscription, groupSubscription} = results
+			if subscription? and subscription.planCode? and subscription.planCode != Settings.defaultPlanCode
+				logger.log user_id:user_id, "using users subscription plan code for features"
+				UserFeaturesUpdater.updateFeatures user_id, subscription.planCode, callback
+			else if groupSubscription? and groupSubscription.planCode?
+				logger.log user_id:user_id, "using group which user is memor of for features"
+				UserFeaturesUpdater.updateFeatures user_id, groupSubscription.planCode, callback
+			else
+				logger.log user_id:user_id, "using default features for user with no subscription or group"
+				UserFeaturesUpdater.updateFeatures user_id, Settings.defaultPlanCode, (err)->
+					if err?
+						logger.err err:err, user_id:user_id, "Error setting minimum user feature"
+						return callback(err)
+					ReferalAllocator.assignBonus user_id, callback
 

services/web/app/coffee/Features/ThirdPartyDataStore/TpdsUpdateSender.coffee

@@ -5,6 +5,7 @@ Project = require('../../models/Project').Project
 keys = require('../../infrastructure/Keys')
 metrics = require("../../infrastructure/Metrics")
 request = require("request")
+CollaboratorsHandler = require('../Collaborators/CollaboratorsHandler')
 
 buildPath = (user_id, project_name, filePath)->
 	projectPath = path.join(project_name, "/", filePath)
@@ -122,9 +123,11 @@ module.exports = TpdsUpdateSender =
 		TpdsUpdateSender._enqueue "poll-dropbox:#{user_id}", "standardHttpRequest", options, callback
 
 getProjectsUsersIds = (project_id, callback = (err, owner_id, allUserIds)->)->
-	Project.findById project_id, "_id owner_ref readOnly_refs collaberator_refs", (err, project)->
-		allUserIds = [].concat(project.collaberator_refs).concat(project.readOnly_refs).concat(project.owner_ref)
-		callback err, project.owner_ref, allUserIds
+	Project.findById project_id, "_id owner_ref", (err, project) ->
+		return callback(err) if err?
+		CollaboratorsHandler.getMemberIds project_id, (err, member_ids) ->
+			return callback(err) if err?
+			callback err, project?.owner_ref, member_ids
 
 mergeProjectNameAndPath = (project_name, path)->
 	if(path.indexOf('/') == 0)

services/web/app/coffee/Features/Uploads/ArchiveManager.coffee

@@ -25,33 +25,31 @@ module.exports = ArchiveManager =
 			error += chunk
 
 		unzip.on "error", (err) ->
-			logger.error {err, source, destination}, "unzip failed"
+			logger.error {err, source}, "unzip failed"
 			if err.code == "ENOENT"
 				logger.error "unzip command not found. Please check the unzip command is installed"
 			callback(err)
 
-		unzip.on "exit", () ->
+		unzip.on "close", (exitCode) ->
 			if error?
 				error = new Error(error)
-				logger.error err:error, source: source, destination: destination, "error checking zip size"
+				logger.error err:error, source: source, "error checking zip size"
 
 			lines = output.split("\n")
 			lastLine = lines[lines.length - 2]?.trim()
 			totalSizeInBytes = lastLine?.split(" ")?[0]
 
-			totalSizeInBytes = parseInt(totalSizeInBytes)
+			totalSizeInBytesAsInt = parseInt(totalSizeInBytes)
 
-			if !totalSizeInBytes? or isNaN(totalSizeInBytes)
-				logger.err source:source, "error getting bytes of zip"
-				return callback(new Error("something went wrong"))
+			if !totalSizeInBytesAsInt? or isNaN(totalSizeInBytesAsInt)
+				logger.err source:source, totalSizeInBytes:totalSizeInBytes, totalSizeInBytesAsInt:totalSizeInBytesAsInt, lastLine:lastLine, exitCode:exitCode, "error getting bytes of zip"
+				return callback(new Error("error getting bytes of zip"))
 
 			isTooLarge = totalSizeInBytes > (ONE_MEG * 300)
 
 			callback(error, isTooLarge)
 
 
-
-
 					
 	extractZipArchive: (source, destination, _callback = (err) ->) ->
 		callback = (args...) ->
@@ -87,7 +85,7 @@ module.exports = ArchiveManager =
 					logger.error "unzip command not found. Please check the unzip command is installed"
 				callback(err)
 
-			unzip.on "exit", () ->
+			unzip.on "close", () ->
 				timer.done()
 				if error?
 					error = new Error(error)

services/web/app/coffee/Features/Uploads/UploadsRouter.coffee

@@ -1,4 +1,4 @@
-SecurityManager = require('../../managers/SecurityManager')
+AuthorizationMiddlewear = require('../Authorization/AuthorizationMiddlewear')
 AuthenticationController = require('../Authentication/AuthenticationController')
 ProjectUploadController = require "./ProjectUploadController"
 RateLimiterMiddlewear = require('../Security/RateLimiterMiddlewear')
@@ -16,6 +16,7 @@ module.exports =
 				maxRequests: 200
 				timeInterval: 60 * 30
 			}),
-			SecurityManager.requestCanModifyProject,
+			AuthenticationController.requireLogin(),
+			AuthorizationMiddlewear.ensureUserCanWriteProjectContent,
 			ProjectUploadController.uploadFile
 

services/web/app/coffee/Features/User/UserInfoController.coffee

@@ -6,10 +6,6 @@ sanitize = require('sanitizer')
 
 module.exports = UserController =
 	getLoggedInUsersPersonalInfo: (req, res, next = (error) ->) ->
-		# this is funcky as hell, we don't use the current session to get the user
-		# we use the auth token, actually destroying session from the chat api request
-		if req.query?.auth_token?
-			req.session?.destroy() 
 		logger.log user: req.user, "reciving request for getting logged in users personal info"
 		return next(new Error("User is not logged in")) if !req.user?
 		UserGetter.getUser req.user._id, {

services/web/app/coffee/Features/Wiki/WikiController.coffee

@@ -4,7 +4,7 @@ logger = require("logger-sharelatex")
 ErrorController = require "../Errors/ErrorController"
 _ = require("underscore")
 AuthenticationController = require("../Authentication/AuthenticationController")
-
+async = require("async")
 other_lngs = ["es"]
 
 module.exports = WikiController = 
@@ -28,19 +28,22 @@ module.exports = WikiController =
 				lngPage = "#{page}_#{req.lng}"
 			else
 				lngPage = page
-
-			WikiController._getPageContent "Contents", (error, contents) ->
+			jobs =
+				contents: (cb)-> 
+					WikiController._getPageContent "Contents", cb
+				pageData: (cb)->
+					WikiController._getPageContent lngPage, cb
+			async.parallel jobs, (error, results)->
 				return next(error) if error?
-				WikiController._getPageContent lngPage, (error, pageData) ->
-					return next(error) if error?
-					if pageData.content?.length > 280
-						if _.include(other_lngs, req.lng)
-							pageData.title = pageData.title.slice(0, pageData.title.length - (req.lng.length+1) )
+				{pageData, contents} = results
+				if pageData.content?.length > 280
+					if _.include(other_lngs, req.lng)
+						pageData.title = pageData.title.slice(0, pageData.title.length - (req.lng.length+1) )
+					WikiController._renderPage(pageData, contents, res)
+				else
+					WikiController._getPageContent page, (error, pageData) ->
+						return next(error) if error?
 						WikiController._renderPage(pageData, contents, res)
-					else
-						WikiController._getPageContent page, (error, pageData) ->
-							return next(error) if error?
-							WikiController._renderPage(pageData, contents, res)
 
 
 					
@@ -62,7 +65,6 @@ module.exports = WikiController =
 			result = 
 				content: data?.parse?.text?['*']
 				title: data?.parse?.title
-
 			callback null, result
 
 

services/web/app/coffee/infrastructure/ExpressLocals.coffee

@@ -64,6 +64,12 @@ module.exports = (app, webRouter, apiRouter)->
 			Settings.siteUrl.substring(Settings.siteUrl.indexOf("//")+2)
 		next()
 
+	webRouter.use (req, res, next)->
+		res.locals.getUserEmail = ->
+			email = req?.session?.user?.email or ""
+			return email
+		next()
+
 	webRouter.use (req, res, next)->
 		res.locals.formatProjectPublicAccessLevel = (privilegeLevel)->
 			formatedPrivileges = private:"Private", readOnly:"Public: Read Only", readAndWrite:"Public: Read and Write"

services/web/app/coffee/infrastructure/Server.coffee

@@ -30,6 +30,8 @@ OldAssetProxy = require("./OldAssetProxy")
 translations = require("translations-sharelatex").setup(Settings.i18n)
 Modules = require "./Modules"
 
+ErrorController = require "../Features/Errors/ErrorController"
+
 metrics.mongodb.monitor(Path.resolve(__dirname + "/../../../node_modules/mongojs/node_modules/mongodb"), logger)
 metrics.mongodb.monitor(Path.resolve(__dirname + "/../../../node_modules/mongoose/node_modules/mongodb"), logger)
 
@@ -123,7 +125,7 @@ apiRouter.get "/profile", (req, res) ->
 	, time
 
 app.get "/heapdump", (req, res)->
-	require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.clsi.heapsnapshot', (err, filename)->
+	require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.web.heapsnapshot', (err, filename)->
 		res.send filename
 
 logger.info ("creating HTTP server").yellow
@@ -136,6 +138,8 @@ app.use(webRouter)
 
 router = new Router(webRouter, apiRouter)
 
+app.use ErrorController.handleError
+
 module.exports =
 	app: app
 	server: server

services/web/app/coffee/managers/SecurityManager.coffee

@@ -1,194 +0,0 @@
-logger = require('logger-sharelatex')
-crypto = require 'crypto'
-Assert = require 'assert'
-Settings = require 'settings-sharelatex'
-User = require('../models/User').User
-Project = require('../models/Project').Project
-ErrorController = require("../Features/Errors/ErrorController")
-AuthenticationController = require("../Features/Authentication/AuthenticationController")
-_ = require('underscore')
-metrics = require('../infrastructure/Metrics')
-querystring = require('querystring')
-async = require "async"
-
-module.exports = SecurityManager =
-	restricted : (req, res, next)->
-		if req.session.user?
-			res.render 'user/restricted',
-				title:'restricted'
-		else
-			logger.log "user not logged in and trying to access #{req.url}, being redirected to login"
-			res.redirect '/register'
-
-	getCurrentUser: (req, callback) ->
-		if req.session.user?
-			User.findById req.session.user._id, callback
-		else
-			callback null, null
-
-	requestCanAccessMultipleProjects: (req, res, next) ->
-		project_ids = req.query.project_ids?.split(",")
-		jobs = []
-		for project_id in project_ids or []
-			do (project_id) ->
-				jobs.push (callback) ->
-					# This is a bit hacky - better to have an abstracted method
-					# that we can pass project_id to, but this whole file needs
-					# a serious refactor ATM.
-					req.params.Project_id = project_id
-					SecurityManager.requestCanAccessProject req, res, (error) ->
-						delete req.params.Project_id
-						callback(error)
-		async.series jobs, next
-
-	requestCanAccessProject : (req, res, next)->
-		doRequest = (req, res, next) ->
-			getRequestUserAndProject req, res, {allow_auth_token: options?.allow_auth_token}, (err, user, project)->
-				if !project? or project.archived
-					return ErrorController.notFound(req, res, next)
-				userCanAccessProject user, project, (canAccess, permissionLevel)->
-					if canAccess
-						next()
-					else if user?
-						logger.log "user_id: #{user._id} email: #{user.email} trying to access restricted page #{req.path}"
-						res.redirect('/restricted')
-					else
-						logger.log "user not logged in and trying to access #{req.url}, being redirected to login"
-						AuthenticationController._redirectToLoginOrRegisterPage(req, res)
-		if arguments.length > 1
-			options =
-				allow_auth_token: false
-			doRequest.apply(this, arguments)
-		else
-			options = req
-			return doRequest
-
-	requestCanModifyProject : (req, res, next)->
-		getRequestUserAndProject req, res, {}, (err, user, project)=>
-			userCanModifyProject user, project, (canModify)->
-				if canModify
-					next()
-				else
-					logger.log "user_id: #{user?._id} email: #{user?.email} can not modify project redirecting to restricted page"
-					res.redirect('/restricted')
-
-	userCanModifyProject : userCanModifyProject = (user, project, callback)->
-		if !user? or !project?
-			callback false
-		else if userIsOwner user, project
-			callback true
-		else if userIsCollaberator user, project
-			callback true
-		else if project.publicAccesLevel == "readAndWrite"
-			callback true
-		else if user.isAdmin
-			callback true
-		else
-			callback false
-
-
-	requestIsOwner : (req, res, next)->
-		getRequestUserAndProject req, res, {}, (err, user, project)->
-			if !user?
-				return res.redirect('/restricted')
-			else if userIsOwner user, project || user.isAdmin
-				next()
-			else
-				logger.log user_id: user?._id, email: user?.email, "user is not owner of project redirecting to restricted page"
-				res.redirect('/restricted')
-
-	requestIsAdmin : isAdmin = (req, res, next)->
-		logger.log "checking if user is admin"
-		user = req.session.user
-		if(user? && user.isAdmin)
-			logger.log user: user, "User is admin"
-			next()
-		else
-			res.redirect('/restricted')
-			logger.log user:user, "is not admin redirecting to restricted page"
-
-	userCanAccessProject : userCanAccessProject = (user, project, callback)=>
-		if !user?
-			user = {_id:'anonymous-user'}
-		if !project?
-			callback false
-		logger.log user:user, project:project, "Checking if can access"
-		if userIsOwner user, project
-			callback true, "owner"
-		else if userIsCollaberator user, project
-			callback true, "readAndWrite"
-		else if userIsReadOnly user, project
-			callback true, "readOnly"
-		else if user.isAdmin
-			logger.log  user:user, project:project, "user is admin and can access project"
-			callback true, "owner"
-		else if project.publicAccesLevel == "readAndWrite"
-			logger.log  user:user, project:project, "project is a public read and write project"
-			callback true, "readAndWrite"
-		else if project.publicAccesLevel == "readOnly"
-			logger.log  user:user, project:project,  "project is a public read only project"
-			callback true, "readOnly"
-		else
-			metrics.inc "security.denied"
-			logger.log  user:user, project:project, "Security denied - user can not enter project"
-			callback false
-
-	userIsOwner : userIsOwner = (user, project)->
-		if !user?
-			return false
-		else
-			userId = user._id+''
-			ownerRef = getProjectIdFromRef(project.owner_ref)
-			if userId == ownerRef
-				true
-			else
-				false
-
-	userIsCollaberator : userIsCollaberator = (user, project)->
-		if !user?
-			return false
-		else
-			userId = user._id+''
-			result = false
-			_.each project.collaberator_refs, (colabRef)->
-				colabRef = getProjectIdFromRef(colabRef)
-				if colabRef == userId
-					result = true
-			return result
-
-	userIsReadOnly : userIsReadOnly = (user, project)->
-		if !user?
-			return false
-		else
-			userId = user._id+''
-			result = false
-			_.each project.readOnly_refs, (readOnlyRef)->
-				readOnlyRef = getProjectIdFromRef(readOnlyRef)
-				
-				if readOnlyRef == userId
-					result = true
-			return result
-
-getRequestUserAndProject = (req, res, options, callback)->
-	project_id = req.params.Project_id
-	if !project_id?
-		logger.log project_id:project_id, options:options, url:req?.url, "no project_id trying to getRequestUserAndProject"
-		return res.send 422
-	Project.findById project_id, 'name owner_ref readOnly_refs collaberator_refs publicAccesLevel archived', (err, project)=>
-		if err?
-			logger.err err:err, "error getting project for security check"
-			return callback err
-		AuthenticationController.getLoggedInUser req, options, (err, user)=>
-			if err?
-				logger.err err:err, "error getting last logged in user for security check"
-			callback err, user, project
-
-getProjectIdFromRef = (ref)->
-	if !ref?
-		return null
-	else if ref._id?
-		return ref._id+''
-	else
-		return ref+''
-
-

services/web/app/coffee/models/Project.coffee

@@ -5,7 +5,7 @@ FolderSchema = require('./Folder.js').FolderSchema
 logger = require('logger-sharelatex')
 sanitize = require('sanitizer')
 concreteObjectId = require('mongoose').Types.ObjectId
-Errors  = require "../errors"
+Errors  = require "../Features/Errors/Errors"
 
 
 Schema = mongoose.Schema
@@ -43,33 +43,6 @@ ProjectSchema.statics.getProject = (project_or_id, fields, callback)->
 			return callback(new Errors.NotFoundError(e.message))
 		this.findById project_or_id, fields, callback
 
-ProjectSchema.statics.findPopulatedById = (project_id, callback)->
-	logger.log project_id:project_id, "findPopulatedById"
-	this.find(_id: project_id )
-			.populate('collaberator_refs')
-			.populate('readOnly_refs')
-			.populate('owner_ref')
-			.exec (err, projects)->
-				if err? 
-					logger.err err:err, project_id:project_id, "something went wrong looking for project findPopulatedById"
-					callback(err)
-				else if !projects? || projects.length == 0
-					logger.err project_id:project_id, "something went wrong looking for project findPopulatedById, no project could be found"
-					callback "not found"
-				else
-					logger.log project_id:project_id, "finished findPopulatedById"
-					callback(null, projects[0])
-
-ProjectSchema.statics.findAllUsersProjects = (user_id, requiredFields, callback)->
-	this.find {owner_ref:user_id}, requiredFields, (err, projects)=>
-		this.find {collaberator_refs:user_id}, requiredFields, (err, collabertions)=>
-			this.find {readOnly_refs:user_id}, requiredFields, (err, readOnlyProjects)=>
-				callback(err, projects, collabertions, readOnlyProjects)
-
-
-
-
-
 applyToAllFilesRecursivly = ProjectSchema.statics.applyToAllFilesRecursivly = (folder, fun)->
 	_.each folder.fileRefs, (file)->
 		fun(file)

services/web/app/coffee/models/ProjectInvite.coffee

@@ -0,0 +1,23 @@
+mongoose = require 'mongoose'
+Settings = require 'settings-sharelatex'
+
+Schema = mongoose.Schema
+ObjectId = Schema.ObjectId
+
+ProjectInviteSchema = new Schema
+	project_id:     ObjectId
+	from_user_id:   ObjectId
+	privilegeLevel: String
+	# For existing users
+	to_user_id:     ObjectId
+	# For non-existant users
+	hashed_token:   String
+	email:          String
+
+conn = mongoose.createConnection(Settings.mongo.url, server: poolSize: Settings.mongo.poolSize || 10)
+
+ProjectInvite = conn.model('ProjectInvite', ProjectInviteSchema)
+
+mongoose.model 'ProjectInvite', ProjectInviteSchema
+exports.ProjectInvite = ProjectInvite
+exports.ProjectInviteSchema = ProjectInviteSchema

services/web/app/coffee/models/User.coffee

@@ -54,32 +54,10 @@ UserSchema = new Schema
 					# For example, a user signing up directly for a paid plan
 					# has this set to true, despite never having had a free trial
 					hadFreeTrial: {type: Boolean, default: false}
-
-UserSchema.statics.getAllIds = (callback)->
-	this.find {}, ["first_name"], callback
-
-
-UserSchema.statics.findReadOnlyProjects = (user_id, callback)->
-	@find({'projects.readOnly_refs':user_id}).populate('projects.readOnly_refs').run (err, users)->
-		projects = []
-		_.each users, (user)->
-			_.each user.projects, (project)->
-				_.each project.readOnly_refs, (subUser)->
-					if(subUser._id == user_id)
-						projects.push(project)
-		callback(projects)
-
-UserSchema.statics.findCollaborationProjects = (user_id, callback)->
-	@find({'projects.collaberator_refs':user_id}).populate('projects.collaberator_refs').run (err, users)->
-		projects = []
-		_.each users, (user)->
-			_.each user.projects, (project)->
-				_.each project.collaberator_refs, (subUser)->
-					if(subUser._id == user_id)
-						projects.push(project)
-		callback(projects)
-
-
+	refProviders: {
+		mendeley: Boolean  # coerce the refProviders values to Booleans
+		zotero: Boolean
+	}
 
 conn = mongoose.createConnection(Settings.mongo.url, server: poolSize: 10)
 

services/web/app/coffee/router.coffee

@@ -3,8 +3,6 @@ ErrorController = require('./Features/Errors/ErrorController')
 ProjectController = require("./Features/Project/ProjectController")
 ProjectApiController = require("./Features/Project/ProjectApiController")
 SpellingController = require('./Features/Spelling/SpellingController')
-SecurityManager = require('./managers/SecurityManager')
-AuthorizationManager = require('./Features/Security/AuthorizationManager')
 EditorController = require("./Features/Editor/EditorController")
 EditorRouter = require("./Features/Editor/EditorRouter")
 Settings = require('settings-sharelatex')
@@ -39,6 +37,7 @@ RealTimeProxyRouter = require('./Features/RealTimeProxy/RealTimeProxyRouter')
 InactiveProjectController = require("./Features/InactiveData/InactiveProjectController")
 ContactRouter = require("./Features/Contacts/ContactRouter")
 ReferencesController = require('./Features/References/ReferencesController')
+AuthorizationMiddlewear = require('./Features/Authorization/AuthorizationMiddlewear')
 
 logger = require("logger-sharelatex")
 _ = require("underscore")
@@ -54,7 +53,7 @@ module.exports = class Router
 
 		webRouter.post '/login', AuthenticationController.login
 		webRouter.get  '/logout', UserController.logout
-		webRouter.get  '/restricted', SecurityManager.restricted
+		webRouter.get  '/restricted', AuthorizationMiddlewear.restricted
 
 		# Left as a placeholder for implementing a public register page
 		webRouter.get  '/register', UserPagesController.registerPage
@@ -88,8 +87,7 @@ module.exports = class Router
 		webRouter.delete '/user/newsletter/unsubscribe', AuthenticationController.requireLogin(), UserController.unsubscribe
 		webRouter.delete '/user', AuthenticationController.requireLogin(), UserController.deleteUser
 
-		webRouter.get  '/user/auth_token', AuthenticationController.requireLogin(), AuthenticationController.getAuthToken
-		webRouter.get  '/user/personal_info', AuthenticationController.requireLogin(allow_auth_token: true), UserInfoController.getLoggedInUsersPersonalInfo
+		webRouter.get  '/user/personal_info', AuthenticationController.requireLogin(), UserInfoController.getLoggedInUsersPersonalInfo
 		apiRouter.get  '/user/:user_id/personal_info', AuthenticationController.httpAuth, UserInfoController.getPersonalInfo
 
 		webRouter.get  '/project', AuthenticationController.requireLogin(), ProjectController.projectListPage
@@ -100,12 +98,13 @@ module.exports = class Router
 			params: ["Project_id"]
 			maxRequests: 10
 			timeInterval: 60
-		}), SecurityManager.requestCanAccessProject, ProjectController.loadEditor
-		webRouter.get  '/Project/:Project_id/file/:File_id', SecurityManager.requestCanAccessProject, FileStoreController.getFile
-		webRouter.post '/project/:Project_id/settings', SecurityManager.requestCanModifyProject, ProjectController.updateProjectSettings
+		}), AuthorizationMiddlewear.ensureUserCanReadProject, ProjectController.loadEditor
+		webRouter.get  '/Project/:Project_id/file/:File_id', AuthorizationMiddlewear.ensureUserCanReadProject, FileStoreController.getFile
+		webRouter.post '/project/:Project_id/settings', AuthorizationMiddlewear.ensureUserCanWriteProjectSettings, ProjectController.updateProjectSettings
+		webRouter.post '/project/:Project_id/settings/admin', AuthorizationMiddlewear.ensureUserCanAdminProject, ProjectController.updateProjectAdminSettings
 
-		webRouter.post '/project/:Project_id/compile', SecurityManager.requestCanAccessProject, CompileController.compile
-		webRouter.get  '/Project/:Project_id/output/output.pdf', SecurityManager.requestCanAccessProject, CompileController.downloadPdf
+		webRouter.post '/project/:Project_id/compile', AuthorizationMiddlewear.ensureUserCanReadProject, CompileController.compile
+		webRouter.get  '/Project/:Project_id/output/output.pdf', AuthorizationMiddlewear.ensureUserCanReadProject, CompileController.downloadPdf
 		webRouter.get  /^\/project\/([^\/]*)\/output\/(.*)$/,
 			((req, res, next) ->
 				params =
@@ -113,24 +112,35 @@ module.exports = class Router
 					"file":       req.params[1]
 				req.params = params
 				next()
-			), SecurityManager.requestCanAccessProject, CompileController.getFileFromClsi
-		webRouter.delete "/project/:Project_id/output", SecurityManager.requestCanAccessProject, CompileController.deleteAuxFiles
-		webRouter.get "/project/:Project_id/sync/code", SecurityManager.requestCanAccessProject, CompileController.proxySync
-		webRouter.get "/project/:Project_id/sync/pdf", SecurityManager.requestCanAccessProject, CompileController.proxySync
-		webRouter.get "/project/:Project_id/wordcount", SecurityManager.requestCanAccessProject, CompileController.wordCount
+			), AuthorizationMiddlewear.ensureUserCanReadProject, CompileController.getFileFromClsi
+		# direct url access to output files for a specific build (query string not required)
+		webRouter.get  /^\/project\/([^\/]*)\/build\/([0-9a-f-]+)\/output\/(.*)$/,
+			((req, res, next) ->
+				params =
+					"Project_id": req.params[0]
+					"build":      req.params[1]
+					"file":       req.params[2]
+				req.params = params
+				next()
+			), AuthorizationMiddlewear.ensureUserCanReadProject, CompileController.getFileFromClsi
 
-		webRouter.delete '/Project/:Project_id', SecurityManager.requestIsOwner, ProjectController.deleteProject
-		webRouter.post '/Project/:Project_id/restore', SecurityManager.requestIsOwner, ProjectController.restoreProject
-		webRouter.post '/Project/:Project_id/clone', SecurityManager.requestCanAccessProject, ProjectController.cloneProject
+		webRouter.delete "/project/:Project_id/output", AuthorizationMiddlewear.ensureUserCanReadProject, CompileController.deleteAuxFiles
+		webRouter.get "/project/:Project_id/sync/code", AuthorizationMiddlewear.ensureUserCanReadProject, CompileController.proxySyncCode
+		webRouter.get "/project/:Project_id/sync/pdf", AuthorizationMiddlewear.ensureUserCanReadProject, CompileController.proxySyncPdf
+		webRouter.get "/project/:Project_id/wordcount", AuthorizationMiddlewear.ensureUserCanReadProject, CompileController.wordCount
 
-		webRouter.post '/project/:Project_id/rename', SecurityManager.requestIsOwner, ProjectController.renameProject
+		webRouter.delete '/Project/:Project_id', AuthorizationMiddlewear.ensureUserCanAdminProject, ProjectController.deleteProject
+		webRouter.post '/Project/:Project_id/restore', AuthorizationMiddlewear.ensureUserCanAdminProject, ProjectController.restoreProject
+		webRouter.post '/Project/:Project_id/clone', AuthorizationMiddlewear.ensureUserCanReadProject, ProjectController.cloneProject
 
-		webRouter.get  "/project/:Project_id/updates", SecurityManager.requestCanAccessProject, TrackChangesController.proxyToTrackChangesApi
-		webRouter.get  "/project/:Project_id/doc/:doc_id/diff", SecurityManager.requestCanAccessProject, TrackChangesController.proxyToTrackChangesApi
-		webRouter.post "/project/:Project_id/doc/:doc_id/version/:version_id/restore", SecurityManager.requestCanAccessProject, TrackChangesController.proxyToTrackChangesApi
+		webRouter.post '/project/:Project_id/rename', AuthorizationMiddlewear.ensureUserCanAdminProject, ProjectController.renameProject
 
-		webRouter.get  '/Project/:Project_id/download/zip', SecurityManager.requestCanAccessProject, ProjectDownloadsController.downloadProject
-		webRouter.get  '/project/download/zip', SecurityManager.requestCanAccessMultipleProjects, ProjectDownloadsController.downloadMultipleProjects
+		webRouter.get  "/project/:Project_id/updates", AuthorizationMiddlewear.ensureUserCanReadProject, TrackChangesController.proxyToTrackChangesApi
+		webRouter.get  "/project/:Project_id/doc/:doc_id/diff", AuthorizationMiddlewear.ensureUserCanReadProject, TrackChangesController.proxyToTrackChangesApi
+		webRouter.post "/project/:Project_id/doc/:doc_id/version/:version_id/restore", AuthorizationMiddlewear.ensureUserCanReadProject, TrackChangesController.proxyToTrackChangesApi
+
+		webRouter.get  '/Project/:Project_id/download/zip', AuthorizationMiddlewear.ensureUserCanReadProject, ProjectDownloadsController.downloadProject
+		webRouter.get  '/project/download/zip', AuthorizationMiddlewear.ensureUserCanReadMultipleProjects, ProjectDownloadsController.downloadMultipleProjects
 
 		webRouter.get    '/tag', AuthenticationController.requireLogin(), TagsController.getAllTags
 		webRouter.post   '/tag', AuthenticationController.requireLogin(), TagsController.createTag
@@ -174,26 +184,31 @@ module.exports = class Router
 		webRouter.post "/spelling/check", AuthenticationController.requireLogin(), SpellingController.proxyRequestToSpellingApi
 		webRouter.post "/spelling/learn", AuthenticationController.requireLogin(), SpellingController.proxyRequestToSpellingApi
 
-		webRouter.get  "/project/:Project_id/messages", SecurityManager.requestCanAccessProject, ChatController.getMessages
-		webRouter.post "/project/:Project_id/messages", SecurityManager.requestCanAccessProject, ChatController.sendMessage
+		webRouter.get  "/project/:Project_id/messages", AuthorizationMiddlewear.ensureUserCanReadProject, ChatController.getMessages
+		webRouter.post "/project/:Project_id/messages", AuthorizationMiddlewear.ensureUserCanReadProject, ChatController.sendMessage
 
-		webRouter.get  /learn(\/.*)?/, WikiController.getPage
+		webRouter.get  /learn(\/.*)?/, RateLimiterMiddlewear.rateLimit({
+			endpointName: "wiki"
+			params: []
+			maxRequests: 60
+			timeInterval: 60
+		}), WikiController.getPage
 
-		webRouter.post "/project/:Project_id/references/index", SecurityManager.requestCanAccessProject, ReferencesController.index
-		webRouter.post "/project/:Project_id/references/indexAll", SecurityManager.requestCanAccessProject, ReferencesController.indexAll
+		webRouter.post "/project/:Project_id/references/index", AuthorizationMiddlewear.ensureUserCanReadProject, ReferencesController.index
+		webRouter.post "/project/:Project_id/references/indexAll", AuthorizationMiddlewear.ensureUserCanReadProject, ReferencesController.indexAll
 
 		#Admin Stuff
-		webRouter.get  '/admin', SecurityManager.requestIsAdmin, AdminController.index
-		webRouter.get  '/admin/user', SecurityManager.requestIsAdmin, (req, res)-> res.redirect("/admin/register") #this gets removed by admin-panel addon
-		webRouter.get  '/admin/register', SecurityManager.requestIsAdmin, AdminController.registerNewUser
-		webRouter.post '/admin/register', SecurityManager.requestIsAdmin, UserController.register
-		webRouter.post '/admin/closeEditor', SecurityManager.requestIsAdmin, AdminController.closeEditor
-		webRouter.post '/admin/dissconectAllUsers', SecurityManager.requestIsAdmin, AdminController.dissconectAllUsers
-		webRouter.post '/admin/syncUserToSubscription', SecurityManager.requestIsAdmin, AdminController.syncUserToSubscription
-		webRouter.post '/admin/flushProjectToTpds', SecurityManager.requestIsAdmin, AdminController.flushProjectToTpds
-		webRouter.post '/admin/pollDropboxForUser', SecurityManager.requestIsAdmin, AdminController.pollDropboxForUser
-		webRouter.post '/admin/messages', SecurityManager.requestIsAdmin, AdminController.createMessage
-		webRouter.post '/admin/messages/clear', SecurityManager.requestIsAdmin, AdminController.clearMessages
+		webRouter.get  '/admin', AuthorizationMiddlewear.ensureUserIsSiteAdmin, AdminController.index
+		webRouter.get  '/admin/user', AuthorizationMiddlewear.ensureUserIsSiteAdmin, (req, res)-> res.redirect("/admin/register") #this gets removed by admin-panel addon
+		webRouter.get  '/admin/register', AuthorizationMiddlewear.ensureUserIsSiteAdmin, AdminController.registerNewUser
+		webRouter.post '/admin/register', AuthorizationMiddlewear.ensureUserIsSiteAdmin, UserController.register
+		webRouter.post '/admin/closeEditor', AuthorizationMiddlewear.ensureUserIsSiteAdmin, AdminController.closeEditor
+		webRouter.post '/admin/dissconectAllUsers', AuthorizationMiddlewear.ensureUserIsSiteAdmin, AdminController.dissconectAllUsers
+		webRouter.post '/admin/syncUserToSubscription', AuthorizationMiddlewear.ensureUserIsSiteAdmin, AdminController.syncUserToSubscription
+		webRouter.post '/admin/flushProjectToTpds', AuthorizationMiddlewear.ensureUserIsSiteAdmin, AdminController.flushProjectToTpds
+		webRouter.post '/admin/pollDropboxForUser', AuthorizationMiddlewear.ensureUserIsSiteAdmin, AdminController.pollDropboxForUser
+		webRouter.post '/admin/messages', AuthorizationMiddlewear.ensureUserIsSiteAdmin, AdminController.createMessage
+		webRouter.post '/admin/messages/clear', AuthorizationMiddlewear.ensureUserIsSiteAdmin, AdminController.clearMessages
 
 		apiRouter.get '/perfTest', (req,res)->
 			res.send("hello")
@@ -205,7 +220,7 @@ module.exports = class Router
 		webRouter.get '/health_check', HealthCheckController.check
 		webRouter.get '/health_check/redis', HealthCheckController.checkRedis
 
-		apiRouter.get "/status/compiler/:Project_id", SecurityManager.requestCanAccessProject, (req, res) ->
+		apiRouter.get "/status/compiler/:Project_id", AuthorizationMiddlewear.ensureUserCanReadProject, (req, res) ->
 			sendRes = _.once (statusCode, message)->
 				res.writeHead statusCode
 				res.end message
@@ -222,9 +237,9 @@ module.exports = class Router
 				headers: req.headers
 			})
 
-		apiRouter.get '/oops-express', (req, res, next) -> next(new Error("Test error"))
-		apiRouter.get '/oops-internal', (req, res, next) -> throw new Error("Test error")
-		apiRouter.get '/oops-mongo', (req, res, next) ->
+		webRouter.get '/oops-express', (req, res, next) -> next(new Error("Test error"))
+		webRouter.get '/oops-internal', (req, res, next) -> throw new Error("Test error")
+		webRouter.get '/oops-mongo', (req, res, next) ->
 			require("./models/Project").Project.findOne {}, () ->
 				throw new Error("Test error")
 
@@ -233,7 +248,8 @@ module.exports = class Router
 			res.send()
 
 		webRouter.post '/error/client', (req, res, next) ->
-			logger.error err: req.body.error, meta: req.body.meta, "client side error"
+			logger.warn err: req.body.error, meta: req.body.meta, "client side error"
+			metrics.inc("client-side-error")
 			res.sendStatus(204)
 
 		webRouter.get '*', ErrorController.notFound

services/web/app/views/contact-us-modal.jade

@@ -0,0 +1,30 @@
+script(type='text/ng-template', id='supportModalTemplate')
+	.modal-header
+		button.close(
+			type="button"
+			data-dismiss="modal"
+			ng-click="close()"
+		) ×
+		h3 #{translate("contact_us")}
+	.modal-body.contact-us-modal
+		span(ng-show="sent == false")
+			label
+				| #{translate("subject")}
+			.form-group
+				input.field.text.medium.span8.form-control(ng-model="form.subject", maxlength='255', tabindex='1', onkeyup='')
+			label.desc(ng-show="'#{getUserEmail()}'.length < 1")
+				| #{translate("email")}
+			.form-group(ng-show="'#{getUserEmail()}'.length < 1")
+				input.field.text.medium.span8.form-control(ng-model="form.email",  ng-init="form.email = '#{getUserEmail()}'", type='email', spellcheck='false', value='', maxlength='255', tabindex='2')
+			label#title12.desc
+				| #{translate("project_url")} (#{translate("optional")})
+			.form-group
+				input.field.text.medium.span8.form-control(ng-model="form.project_url", tabindex='3', onkeyup='')
+			label.desc
+				| #{translate("suggestion")}
+			.form-group
+				textarea.field.text.medium.span8.form-control(ng-model="form.message",type='text', value='', tabindex='4', onkeyup='')
+			.form-group.text-center
+				input.btn-success.btn.btn-lg(type='submit', ng-disabled="sending", ng-click="contactUs()" value='#{translate("contact_us")}')
+		span(ng-show="sent")
+			p #{translate("request_sent_thank_you")}

services/web/app/views/general/500.jade

@@ -0,0 +1,21 @@
+doctype html
+html(itemscope, itemtype='http://schema.org/Product')
+	head
+		title Something went wrong
+		link(rel="icon", href="/favicon.ico")
+		link(rel='stylesheet', href='/stylesheets/style.css')
+		link(href="//netdna.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css",rel="stylesheet")
+	body
+		.content
+			.container
+				.row
+					.col-md-8.col-md-offset-2.text-center
+						.page-header
+							h2 Oh dear, something went wrong.
+						p: img(src="/img/lion-sad-128.png", alt="Sad Lion")
+						p
+							| Something went wrong with your request, sorry. Our staff are probably looking into this, but if it continues, please contact us at #{settings.adminEmail}
+						p
+							a(href="/")
+								i.fa.fa-arrow-circle-o-left
+								|  Take me home

services/web/app/views/layout.jade

@@ -23,7 +23,9 @@ html(itemscope, itemtype='http://schema.org/Product')
 
 		if settings.i18n.subdomainLang
 			each subdomainDetails in settings.i18n.subdomainLang
-				link(rel="alternate", href=subdomainDetails.url+currentUrl, hreflang=subdomainDetails.lngCode)
+				if !subdomainDetails.hide
+					link(rel="alternate", href=subdomainDetails.url+currentUrl, hreflang=subdomainDetails.lngCode)
+
 
 		meta(itemprop="name", content="ShareLaTeX, the Online LaTeX Editor")
 		
@@ -60,8 +62,8 @@ html(itemscope, itemtype='http://schema.org/Product')
 				sixpackDomain: '#{settings.sixpack.domain}'
 			};
 			window.systemMessages = !{JSON.stringify(systemMessages).replace(/\//g, '\\/')};
-			window.ab = {}
-			window.user_id = '#{getLoggedInUserId()}'
+			window.ab = {};
+			window.user_id = '#{getLoggedInUserId()}';
 
 		- if (typeof(settings.algolia) != "undefined")
 			script.
@@ -80,6 +82,7 @@ html(itemscope, itemtype='http://schema.org/Product')
 				}
 				
 	body
+
 		- if(typeof(suppressSystemMessages) == "undefined")
 			.system-messages(
 				ng-cloak
@@ -118,21 +121,14 @@ html(itemscope, itemtype='http://schema.org/Product')
 					"paths" : {
 						"moment": "libs/moment-2.7.0"
 					}
-				};	
+				};
 			script(
 				data-main=jsPath+'main.js',
 				baseurl=jsPath,
 				src=jsPath+'libs/require.js?fingerprint='+fingerprint(jsPath + 'libs/require.js')
 			)
 
-		- if (typeof(tenderUrl) != "undefined")
-			script(src="https://#{tenderUrl}/tender_widget.js" )
-			script(type="text/javascript").
-				Tender = {
-					hideToggle: true,
-					widgetToggles: $(".js-tender-widget"),
-					category: "questions"
-				};
+		include contact-us-modal
 
 
 

services/web/app/views/project/editor/binary-file.jade

@@ -4,16 +4,41 @@ div.binary-file.full-size(
 	ng-if="openFile"
 )
 	img(
+		ng-show="!failedLoad"
 		ng-src="/project/{{ project_id }}/file/{{ openFile.id }}"
 		ng-if="['png', 'jpg', 'jpeg', 'gif'].indexOf(extension(openFile)) > -1"
+		ng-class="{'img-preview': !imgLoaded}"
+		onerror="sl_binaryFilePreviewError()"
+		onabort="sl_binaryFilePreviewError()"
+		onload="sl_binaryFilePreviewLoaded()"
 	)
+
 	img(
+		ng-show="!failedLoad"
 		ng-src="/project/{{ project_id }}/file/{{ openFile.id }}?format=png"
 		ng-if="['pdf', 'eps'].indexOf(extension(openFile)) > -1"
+		ng-class="{'img-preview': !imgLoaded}"
+		onerror="sl_binaryFilePreviewError()"
+		onabort="sl_binaryFilePreviewError()"
+		onload="sl_binaryFilePreviewLoaded()"
 	)
+	
+	div(ng-if="(['bib'].indexOf(extension(openFile)) > -1) && !bibtexPreview.error")
+
+		div.bib-loading(ng-show="bibtexPreview.loading && !bibtexPreview.error")
+			| #{translate('loading')}...
+
+		div.bib-preview(ng-show="bibtexPreview.data && !bibtexPreview.loading && !bibtexPreview.error")
+			div.scroll-container
+				p
+					| {{ bibtexPreview.data }}
+				p(ng-show="bibtexPreview.shouldShowDots")
+					| ...
+	
 	p.no-preview(
-		ng-if="['png', 'jpg', 'jpeg', 'gif', 'pdf', 'eps'].indexOf(extension(openFile)) == -1"
+		ng-if="failedLoad || bibtexPreview.error || ['bib', 'png', 'jpg', 'jpeg', 'gif', 'pdf', 'eps'].indexOf(extension(openFile)) == -1"
 	) #{translate("no_preview_available")}
+
 	a.btn.btn-info(
 		ng-href="/project/{{ project_id }}/file/{{ openFile.id }}"
 	) #{translate("download")} {{ openFile.name }}

services/web/app/views/project/editor/editor.jade

@@ -10,8 +10,12 @@ div.full-size(
 )
 	.ui-layout-center
 		.loading-panel(ng-show="!editor.sharejs_doc || editor.opening")
-			i.fa.fa-spin.fa-refresh
-			|   #{translate("loading")}...
+			span(ng-show="editor.open_doc_id")
+				i.fa.fa-spin.fa-refresh
+				|   #{translate("loading")}...
+			span(ng-show="!editor.open_doc_id")
+				i.fa.fa-arrow-left
+				|   #{translate("open_a_file_on_the_left")}
 
 		#editor(
 			ace-editor="editor",

services/web/app/views/project/editor/file-tree.jade

@@ -70,23 +70,23 @@ aside#file-tree(ng-controller="FileTreeController", ng-class="{ 'multi-selected'
 				ng-repeat="entity in rootFolder.children | orderBy:[orderByFoldersFirst, 'name']"
 			)
 
-		div(ng-show="deletedDocs.length > 0 && ui.view == 'track-changes'")
-			h3 #{translate("deleted_files")}
-			ul.list-unstyled.file-tree-list.deleted-docs
-				li(
-					ng-class="{ 'selected': entity.selected }",
-					ng-repeat="entity in deletedDocs | orderBy:'name'",
-					ng-controller="FileTreeEntityController"
-				)
-					.entity
-						.entity-name(
-							ng-click="select($event)"
-						)
-							//- Just a spacer to align with folders
-							i.fa.fa-fw.toggle
-							i.fa.fa-fw.fa-file
+			li(ng-show="deletedDocs.length > 0 && ui.view == 'track-changes'")
+				h3 #{translate("deleted_files")}
+			li(
+				ng-class="{ 'selected': entity.selected }",
+				ng-repeat="entity in deletedDocs | orderBy:'name'",
+				ng-controller="FileTreeEntityController",
+				ng-show="ui.view == 'track-changes'"
+			)
+				.entity
+					.entity-name(
+						ng-click="select($event)"
+					)
+						//- Just a spacer to align with folders
+						i.fa.fa-fw.toggle
+						i.fa.fa-fw.fa-file
 
-							span  {{ entity.name }}
+						span  {{ entity.name }}
 
 script(type='text/ng-template', id='entityListItemTemplate')
 	li(
@@ -107,8 +107,7 @@ script(type='text/ng-template', id='entityListItemTemplate')
 				//- Just a spacer to align with folders
 				i.fa.fa-fw.toggle(ng-if="entity.type != 'folder'")
 
-				i.fa.fa-fw.fa-file(ng-if="entity.type == 'doc'")
-				i.fa.fa-fw.fa-image(ng-if="entity.type == 'file'")
+				i.fa.fa-fw(ng-if="entity.type != 'folder'", ng-class="'fa-' + iconTypeFromName(entity.name)")
 				span(
 					ng-hide="entity.renaming"
 				)  {{ entity.name }}

services/web/app/views/project/editor/left-menu.jade

@@ -198,21 +198,21 @@ script(type='text/ng-template', id='wordCountModalTemplate')
 		div(ng-if="!status.loading")
 			.container-fluid
 				.row
-					.col-md-4
+					.col-xs-4
 						.pull-right #{translate("total_words")} :
-					.col-md-6 {{data.textWords}}
+					.col-xs-6 {{data.textWords}}
 				.row 
-					.col-md-4
+					.col-xs-4
 						.pull-right #{translate("headers")} :
-					.col-md-6 {{data.headers}}
+					.col-xs-6 {{data.headers}}
 				.row 
-					.col-md-4 
+					.col-xs-4 
 						.pull-right #{translate("math_inline")} :
-					.col-md-6 {{data.mathInline}}
+					.col-xs-6 {{data.mathInline}}
 				.row 
-					.col-md-4 
+					.col-xs-4 
 						.pull-right #{translate("math_display")} :
-					.col-md-6 {{data.mathDisplay}}
+					.col-xs-6 {{data.mathDisplay}}
 	.modal-footer
 		button.btn.btn-default(
 			ng-disabled="state.inflight"

services/web/app/views/project/editor/pdf.jade

@@ -32,7 +32,7 @@ div.full-size.pdf(ng-controller="PdfController")
 		a.log-btn(
 			href
 			ng-click="toggleLogs()"
-			ng-class="{ 'active': (pdf.view == 'logs' || pdf.failure) && !pdf.error && !pdf.timeout && !pdf.uncompiled }"
+			ng-class="{ 'active': shouldShowLogs == true }"
 			tooltip="#{translate('logs_and_output_files')}"
 			tooltip-placement="bottom"
 		)
@@ -74,93 +74,10 @@ div.full-size.pdf(ng-controller="PdfController")
 			)
 				i.split-screen
 				i.split-screen
+	// end of toolbar
 
-	.pdf-viewer(ng-show="pdf.url && pdf.view == 'pdf' && !pdf.failure && !pdf.timeout && !pdf.error")
-		div(
-			pdfng
-			ng-if="settings.pdfViewer == 'pdfjs'"
-			pdf-src="pdf.url"
-			key="{{ project_id }}"
-			resize-on="layout:main:resize,layout:pdf:resize"
-			highlights="pdf.highlights"
-			position="pdf.position"
-			dbl-click-callback="syncToCode"
-		)
-
-		iframe(
-			ng-src="{{ pdf.url }}"
-			ng-if="settings.pdfViewer == 'native'"
-		)
-
-	.pdf-uncompiled(ng-show="pdf.uncompiled && !pdf.compiling")
-		|  
-		i.fa.fa-level-up.fa-flip-horizontal.fa-2x
-		|   #{translate('click_here_to_preview_pdf')}
-
-	.pdf-errors(ng-show="pdf.timedout || pdf.error")
-		.alert.alert-danger(ng-show="pdf.error")
-			strong #{translate("server_error")} 
-			span #{translate("somthing_went_wrong_compiling")}
-		
-
-		.alert.alert-danger(ng-show="pdf.timedout")
-			p
-				strong #{translate("timedout")}. 
-				span #{translate("proj_timed_out_reason")}
-			p
-				a.text-info(href="https://www.sharelatex.com/learn/Debugging_Compilation_timeout_errors", target="_blank")
-					| #{translate("learn_how_to_make_documents_compile_quickly")}
-
-		.alert.alert-success(ng-show="pdf.timedout && !hasPremiumCompile")
-			p
-				strong #{translate("upgrade_for_faster_compiles")}
-			p #{translate("free_accounts_have_timeout_upgrade_to_increase")}
-			p Plus:
-			p
-				ul.list-unstyled
-					li
-						i.fa.fa-check  
-						| #{translate("unlimited_projects")}
-					
-					li
-						i.fa.fa-check  
-						| #{translate("collabs_per_proj", {collabcount:'Multiple'})}
-					
-					li
-						i.fa.fa-check  
-						| #{translate("full_doc_history")}
-					
-					li
-						i.fa.fa-check  
-						| #{translate("sync_to_dropbox")}
-
-					li
-						i.fa.fa-check  
-						| #{translate("sync_to_github")}
-
-					li
-						i.fa.fa-check  
-						|#{translate("compile_larger_projects")}
-
-			p(ng-controller="FreeTrialModalController")
-				a.btn.btn-success.row-spaced-small(
-					href
-					ng-class="buttonClass"
-					sixpack-convert="track_changes_feature_info"
-					ng-click="startFreeTrial('compile-timeout')"
-				) #{translate("start_free_trial")}
-
-	.pdf-errors(ng-show="pdf.projectTooLarge")
-		.alert.alert-danger
-			strong #{translate("project_too_large")} 
-			span #{translate("project_too_large_please_reduce")}
-		
-
-		
-
-
-
-	.pdf-logs(ng-show="(pdf.view == 'logs' || pdf.failure) && !pdf.error && !pdf.timeout && !pdf.uncompiled")
+	// logs view
+	.pdf-logs(ng-show="shouldShowLogs")
 		.alert.alert-success(ng-show="pdf.logEntries.all.length == 0")
 			| #{translate("no_errors_good_job")}
 
@@ -204,7 +121,7 @@ div.full-size.pdf(ng-controller="PdfController")
 					ul.dropdown-menu.dropdown-menu-right
 						li(ng-repeat="file in pdf.outputFiles")
 							a(
-								href="/project/{{project_id}}/output/{{file.path}}"
+								href="{{file.url}}"
 								target="_blank"
 							) {{ file.name }}
 			a.btn.btn-info.btn-sm(href, ng-click="toggleRawLog()")
@@ -212,6 +129,96 @@ div.full-size.pdf(ng-controller="PdfController")
 				span(ng-show="pdf.showRawLog") #{translate("hide_raw_logs")}
 
 		pre(ng-bind="pdf.rawLog", ng-show="pdf.showRawLog")
+	
+	
+	// non-log views (pdf and errors)
+	div(ng-show="!shouldShowLogs", ng-switch on="pdf.view")
+		.pdf-uncompiled(ng-switch-when="uncompiled" ng-show="!pdf.compiling")
+			|  
+			i.fa.fa-level-up.fa-flip-horizontal.fa-2x
+			|   #{translate('click_here_to_preview_pdf')}
+		
+		.pdf-viewer(ng-switch-when="pdf")
+			div(
+				pdfng
+				ng-if="settings.pdfViewer == 'pdfjs'"
+				pdf-src="pdf.url"
+				key="{{ project_id }}"
+				resize-on="layout:main:resize,layout:pdf:resize"
+				highlights="pdf.highlights"
+				position="pdf.position"
+				dbl-click-callback="syncToCode"
+			)
+			iframe(
+				ng-src="{{ pdf.url }}"
+				ng-if="settings.pdfViewer == 'native'"
+			)
+
+		.pdf-errors(ng-switch-when="errors")
+	
+			.alert.alert-danger(ng-show="pdf.error")
+				strong #{translate("server_error")} 
+				span #{translate("somthing_went_wrong_compiling")}
+
+			.alert.alert-danger(ng-show="pdf.renderingError")
+				strong #{translate("pdf_rendering_error")} 
+				span #{translate("something_went_wrong_rendering_pdf")}
+	
+			.alert.alert-danger(ng-show="pdf.clsiMaintenance")
+				strong #{translate("server_error")} 
+				span #{translate("clsi_maintenance")}
+
+			.alert.alert-danger(ng-show="pdf.tooRecentlyCompiled")
+				strong #{translate("server_error")} 
+				span #{translate("too_recently_compiled")}
+	
+			.alert.alert-danger(ng-show="pdf.timedout")
+				p
+					strong #{translate("timedout")}. 
+					span #{translate("proj_timed_out_reason")}
+				p
+					a.text-info(href="https://www.sharelatex.com/learn/Debugging_Compilation_timeout_errors", target="_blank")
+						| #{translate("learn_how_to_make_documents_compile_quickly")}
+
+			.alert.alert-success(ng-show="pdf.timedout && !hasPremiumCompile")
+				p(ng-if="project.owner._id == user.id")
+					strong #{translate("upgrade_for_faster_compiles")}
+				p(ng-if="project.owner._id != user.id")
+					strong #{translate("ask_proj_owner_to_upgrade_for_faster_compiles")}
+				p #{translate("free_accounts_have_timeout_upgrade_to_increase")}
+				p Plus:
+				p
+					ul.list-unstyled
+						li
+							i.fa.fa-check  
+							| #{translate("unlimited_projects")}
+						li
+							i.fa.fa-check  
+							| #{translate("collabs_per_proj", {collabcount:'Multiple'})}
+						li
+							i.fa.fa-check  
+							| #{translate("full_doc_history")}
+						li
+							i.fa.fa-check  
+							| #{translate("sync_to_dropbox")}
+						li
+							i.fa.fa-check  
+							| #{translate("sync_to_github")}
+						li
+							i.fa.fa-check  
+							|#{translate("compile_larger_projects")}
+				p(ng-controller="FreeTrialModalController", ng-if="project.owner._id == user.id")
+					a.btn.btn-success.row-spaced-small(
+						href
+						ng-class="buttonClass"
+						sixpack-convert="track_changes_feature_info"
+						ng-click="startFreeTrial('compile-timeout')"
+					) #{translate("start_free_trial")}
+
+			.alert.alert-danger(ng-show="pdf.projectTooLarge")
+				strong #{translate("project_too_large")} 
+				span #{translate("project_too_large_please_reduce")}
+
 
 script(type='text/ng-template', id='clearCacheModalTemplate')
 	.modal-header

services/web/app/views/project/list/modals.jade

@@ -183,7 +183,9 @@ script(type='text/ng-template', id='deleteProjectsModalTemplate')
 			data-dismiss="modal"
 			ng-click="cancel()"
 		) ×
-		h3 {{action}} #{translate("projects")}
+		h3(ng-if="action == 'delete'") #{translate("delete_projects")}
+		h3(ng-if="action == 'leave'") #{translate("leave_projects")}
+		h3(ng-if="action == 'delete-and-leave'") #{translate("delete_and_leave_projects")}
 	.modal-body
 		div(ng-show="projectsToDelete.length > 0")
 			p #{translate("about_to_delete_projects")}
@@ -201,7 +203,7 @@ script(type='text/ng-template', id='deleteProjectsModalTemplate')
 		) #{translate("cancel")}
 		button.btn.btn-danger(
 			ng-click="delete()"
-		) {{action}}
+		) #{translate("confirm")}
 
 script(type="text/ng-template", id="uploadProjectModalTemplate")
 	.modal-header

services/web/app/views/project/list/project-list.jade

@@ -24,7 +24,7 @@
 			.btn-toolbar(ng-show="filter != 'archived'")
 				.btn-group(ng-hide="selectedProjects.length < 1")
 					a.btn.btn-default(
-						href='#',
+						href,
 						tooltip="#{translate('download')}",
 						tooltip-placement="bottom",
 						tooltip-append-to-body="true",
@@ -32,7 +32,7 @@
 					)
 						i.fa.fa-cloud-download
 					a.btn.btn-default(
-						href='#',
+						href,
 						tooltip="#{translate('delete')}",
 						tooltip-placement="bottom",
 						tooltip-append-to-body="true",
@@ -42,7 +42,7 @@
 
 				.btn-group.dropdown(ng-hide="selectedProjects.length < 1", dropdown)
 					a.btn.btn-default.dropdown-toggle(
-						href="#",
+						href,
 						data-toggle="dropdown",
 						dropdown-toggle,
 						tooltip="#{translate('add_to_folders')}",
@@ -72,11 +72,11 @@
 								|  {{tag.name}}
 						li.divider
 						li
-							a(href="#", ng-click="openNewTagModal()", stop-propagation="click") #{translate("create_new_folder")}
+							a(href, ng-click="openNewTagModal()", stop-propagation="click") #{translate("create_new_folder")}
 
 				.btn-group(ng-hide="selectedProjects.length != 1", dropdown).dropdown
 					a.btn.btn-default.dropdown-toggle(
-						href='#',
+						href,
 						data-toggle="dropdown",
 						dropdown-toggle
 					) #{translate("more")} 
@@ -84,19 +84,19 @@
 					ul.dropdown-menu.dropdown-menu-right(role="menu")
 						li(ng-show="getFirstSelectedProject().accessLevel == 'owner'")
 							a(
-								href='#',
+								href,
 								ng-click="openRenameProjectModal()"
 							) #{translate("rename")}
 						li
 							a(
-								href='#',
+								href,
 								ng-click="openCloneProjectModal()"
 							) #{translate("make_copy")}
 
 			.btn-toolbar(ng-show="filter == 'archived'")
 				.btn-group(ng-hide="selectedProjects.length < 1")
 					a.btn.btn-default(
-						href='#',
+						href,
 						data-original-title="Restore",
 						data-toggle="tooltip",
 						data-placement="bottom",
@@ -105,7 +105,7 @@
 
 				.btn-group(ng-hide="selectedProjects.length < 1")
 					a.btn.btn-danger(
-						href='#',
+						href,
 						data-original-title="Delete Forever",
 						data-toggle="tooltip",
 						data-placement="bottom",

services/web/app/views/subscriptions/group/invite.jade

@@ -2,7 +2,8 @@ extends ../../layout
 
 block scripts
 	script(type='text/javascript').
-		window.subscription_id = '#{subscription_id}'
+		window.group_subscription_id = '#{group_subscription_id}'
+		window.has_personal_subscription = '#{has_personal_subscription}'
 
 block content
 	.content.content-alt
@@ -11,12 +12,26 @@ block content
 				.col-md-8.col-md-offset-2
 					-if (query.expired)
 						.alert.alert-warning #{translate("email_link_expired")}
+
+			.row
+				div  
 			.row
 				.col-md-8.col-md-offset-2(ng-cloak)
 					.card(ng-controller="GroupSubscriptionInviteController")
 						.page-header
 							h1.text-centered #{translate("you_are_invited_to_group", {groupName:licenceName})}
-						div(ng-show="!requestSent").row.text-centered
+
+						div(ng-show="view =='personalSubscription'").row.text-centered
+							div #{translate("cancel_personal_subscription_first")}
+							.row
+								.col-md-12  
+							.row
+								.col-md-12
+									a.btn.btn.btn-default(ng-click="keepPersonalSubscription()", ng-disabled="inflight") #{translate("not_now")}
+									span  
+									a.btn.btn.btn-primary(ng-click="cancelSubscription()", ng-disabled="inflight") #{translate("cancel_your_subscription")}
+
+						div(ng-show="view =='groupSubscriptionInvite'").row.text-centered
 							.row
 								.col-md-12 #{translate("group_provides_you_with_premium_account", {groupName:licenceName})}
 							.row
@@ -26,10 +41,10 @@ block content
 									.text-center
 										a.btn.btn-default(href="/project") #{translate("not_now")}
 										span  
-										a.btn.btn.btn-primary(ng-click="joinGroup()") #{translate("verify_email_address")}  
+										a.btn.btn.btn-primary(ng-click="joinGroup()", ng-disabled="inflight") #{translate("verify_email_address")}  
 
 
-						span(ng-show="requestSent").row.text-centered.text-center
+						span(ng-show="view =='requestSent'").row.text-centered.text-center
 							.row
 								.col-md-12 #{translate("check_email_to_complete_the_upgrade")}
 							.row

services/web/app/views/subscriptions/new.jade

@@ -108,7 +108,7 @@ block content
 														option(value="10") 10
 														option(value="11") 11
 														option(value="12") 12
-											.col-md-3
+											.col-md-4
 												.form-group(ng-class="validation.correctExpiry == false || validation.errorFields.year ? 'has-error' : ''")
 													select.form-control(data-recurly='year', ng-change="validateExpiry()", ng-model='data.year')
 														option(value="", disabled, selected) Year

services/web/app/views/wiki/page.jade

@@ -1,14 +1,68 @@
 extends ../layout
 
 block content
-	.content.content-alt(ng-non-bindable)
+	.content.content-alt(ng-cloak)
 		.container.wiki
+			.row.template-page-header
+				.col-md-8(ng-cloak)
+
 			.row
-				.col-xs-3.contents
+				.col-xs-3.contents(ng-non-bindable)
 					| !{contents.content}
+
 				.col-xs-9.page
-					.card
+					- if(typeof(settings.algolia) != "undefined" && typeof(settings.algolia.indexes) != "undefined" && typeof(settings.algolia.indexes.wiki) != "undefined")
+						span(ng-controller="SearchWikiController")
+							.row
+								form.project-search.form-horizontal.col-md-9(role="form")
+									.form-group.has-feedback.has-feedback-left.col-md-12
+										input.form-control.col-md-12(type='text', ng-model='searchQueryText', ng-keyup='search()', placeholder="Search help library....")
+										i.fa.fa-search.form-control-feedback-left
+										i.fa.fa-times.form-control-feedback(
+											ng-click="clearSearchText()",
+											style="cursor: pointer;",
+											ng-show="searchQueryText.length > 0"
+										)
+								.col-md-3.text-right
+									a.btn.btn-primary(ng-click="showMissingTemplateModal()") #{translate("suggest_new_doc")}
+
+							.row
+								.col-md-12(ng-cloak)
+									a(ng-href='{{hit.url}}',ng-repeat='hit in hits').search-result.card.card-thin
+										span(ng-bind-html='hit.name')
+										div.search-result-content(ng-show="hit.content != ''", ng-bind-html='hit.content')
+
+					.card.row-spaced(ng-non-bindable)
 						.page-header
 							h1 #{title}
 							
-						| !{page.content}
+						| !{page.content}
+
+
+
+
+	script(type="text/ng-template", id="missingWikiPageModal")
+		.modal-header
+			button.close(
+				type="button"
+				data-dismiss="modal"
+				ng-click="close()"
+			) ×
+			h3 #{translate("suggest_new_doc")}
+		.modal-body.contact-us-modal
+			span(ng-show="sent == false")
+				label.desc
+					| #{translate("email")} (#{translate("optional")})
+				.form-group
+					input.field.text.medium.span8.form-control(ng-model="form.email",  ng-init="form.email = '#{getUserEmail()}'", type='email', spellcheck='false', value='', maxlength='255', tabindex='2')
+				label.desc
+					| #{translate("suggestion")}
+				.form-group
+					textarea.field.text.medium.span8.form-control(ng-model="form.message",type='text', value='', maxlength='255', tabindex='4', onkeyup='')
+			span(ng-show="sent")
+				p #{translate("request_sent_thank_you")}
+		.modal-footer
+			button.btn.btn-default(ng-click="close()") 
+				span #{translate("dismiss")}
+			button.btn-success.btn(type='submit', ng-disabled="sending", ng-click="contactUs()") #{translate("contact_us")}
+

services/web/config/settings.defaults.coffee

@@ -88,8 +88,6 @@ module.exports =
 			url: "http://localhost:3009"
 		clsi:
 			url: "http://localhost:3013"
-		clsi_priority:
-			url: "http://localhost:3013"
 		templates:
 			url: "http://localhost:3007"
 		githubSync:
@@ -126,6 +124,9 @@ module.exports =
 	# cookieDomain: ".sharelatex.dev"
 	cookieName:"sharelatex.sid"
 
+	# this is only used if cookies are used for clsi backend
+	#clsiCookieKey: "clsiserver"
+	
 	# Same, but with http auth credentials.
 	httpAuthSiteUrl: 'http://#{httpAuthUser}:#{httpAuthPass}@localhost:3000'
 
@@ -259,7 +260,7 @@ module.exports =
 	
 	# Should we allow access to any page without logging in? This includes
 	# public projects, /learn, /templates, about pages, etc.
-	allowPublicAccess: false
+	allowPublicAccess: if process.env["SHARELATEX_ALLOW_PUBLIC_ACCESS"] == 'true' then true else false
 	
 	# Maximum size of text documents in the real-time editing system.
 	max_doc_length: 2 * 1024 * 1024 # 2mb
@@ -388,11 +389,15 @@ module.exports =
 	# 	filter: "(uid=:userKey)"
 	# 	failMessage: 'LDAP User Fail'
 	# 	fieldName: 'LDAP User'
-	# 	placeholder: 'LDAP User ID'
+	# 	placeholder: 'email@example.com'
 	# 	emailAtt: 'mail'
 	# 	anonymous: false
 	#	adminDN: 'cn=read-only-admin,dc=example,dc=com'	
 	#	adminPW: 'password'
+	#	starttls: true
+	#	tlsOptions:
+	#		rejectUnauthorized: false
+	#		ca: ['/etc/ldap/ca_certs.pem']
 	
 	#templateLinks: [{
 	#	name : "CV projects",

services/web/package.json

@@ -18,11 +18,13 @@
     "body-parser": "^1.13.1",
     "bufferedstream": "1.6.0",
     "connect-redis": "2.3.0",
+    "cookie": "^0.2.3",
     "cookie-parser": "1.3.5",
     "csurf": "^1.8.3",
     "dateformat": "1.0.4-1.2.3",
     "express": "4.13.0",
     "express-session": "1.11.3",
+    "grunt": "^0.4.5",
     "heapdump": "^0.3.7",
     "http-proxy": "^1.8.1",
     "jade": "~1.3.1",
@@ -39,16 +41,19 @@
     "multer": "^0.1.8",
     "node-uuid": "1.4.1",
     "nodemailer": "2.1.0",
+    "nodemailer-sendgrid-transport": "^0.2.0",
     "nodemailer-ses-transport": "^1.3.0",
     "optimist": "0.6.1",
     "redback": "0.4.0",
     "redis": "0.10.1",
     "redis-sharelatex": "0.0.9",
-    "request": "2.67.0",
+    "request": "^2.69.0",
+    "requests": "^0.1.7",
     "rimraf": "2.2.6",
     "sanitizer": "0.1.1",
     "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0",
     "sixpack-client": "^1.0.0",
+    "temp": "^0.8.3",
     "translations-sharelatex": "git+https://github.com/sharelatex/translations-sharelatex.git#master",
     "underscore": "1.6.0",
     "underscore.string": "^3.0.2",

services/web/public/coffee/analytics/AbTestingManager.coffee

@@ -22,24 +22,20 @@ define [
 
 		_buildCookieKey = (testName, bucket)-> 
 			key = "sl_abt_#{testName}_#{bucket}"
-			#console.log key
 			return key
 
 
 		_getTestCookie = (testName, bucket)->
 			cookieKey = _buildCookieKey(testName, bucket)
 			cookie =  ipCookie(cookieKey)
-			#console.log cookieKey, cookie
 			return cookie
 
 		_persistCookieStep = (testName, bucket, newStep)->
 			cookieKey = _buildCookieKey(testName, bucket)
 			ipCookie(cookieKey, {step:newStep}, {expires:100, path:"/"})
-			#console.log("persisting", cookieKey, {step:newStep})
 			ga('send', 'event', 'ab_tests', "#{testName}:#{bucket}", "step-#{newStep}")
 
 		_checkIfStepIsNext = (cookieStep, newStep)->
-			#console.log cookieStep, newStep, "checking if step is next"
 			if !cookieStep? and newStep != 0
 				return false
 			else if newStep == 0
@@ -68,8 +64,6 @@ define [
 			bucketIndex = parseInt(hash.toString().slice(0,2), 16) % (buckets?.length or 2)
 			return buckets[bucketIndex]
 
-
-
 	App.controller "AbTestController", ($scope, abTestManager)->
 		testKeys = _.keys(window.ab)
 

services/web/public/coffee/base.coffee

@@ -23,5 +23,9 @@ define [
 			baseUrl: window.sharelatex.sixpackDomain
 			client_id: window.user_id
 		})
+	
+	sl_debugging = window.location?.search?.match(/debug=true/)?
+	window.sl_console =
+		log: (args...) -> console.log(args...) if sl_debugging
 
 	return App

services/web/public/coffee/ide.coffee

@@ -112,4 +112,20 @@ define [
 
 		ide.localStorage = localStorage
 
+		ide.browserIsSafari = false
+		try
+			userAgent = navigator.userAgent
+			ide.browserIsSafari = (
+				userAgent &&
+				userAgent.match(/.*Safari\/.*/) &&
+				!userAgent.match(/.*Chrome\/.*/) &&
+				!userAgent.match(/.*Chromium\/.*/)
+			)
+		catch err
+			console.error err
+
+		# User can append ?ft=somefeature to url to activate a feature toggle
+		ide.featureToggle = location?.search?.match(/^\?ft=(\w+)$/)?[1]
+
+
 	angular.bootstrap(document.body, ["SharelatexApp"])

services/web/public/coffee/ide/binary-files/BinaryFilesManager.coffee

@@ -9,4 +9,12 @@ define [
 
 		openFile: (file) ->
 			@$scope.ui.view = "file"
-			@$scope.openFile = file
+			@$scope.openFile = null
+			@$scope.$apply()
+			window.setTimeout(
+				() =>
+					@$scope.openFile = file
+					@$scope.$apply()
+				, 0
+				, this
+			)

services/web/public/coffee/ide/binary-files/controllers/BinaryFileController.coffee

@@ -1,7 +1,70 @@
 define [
 	"base"
 ], (App) ->
-	App.controller "BinaryFileController", ["$scope", ($scope) ->
+	App.controller "BinaryFileController", ["$scope", "$rootScope", "$http", "$timeout", ($scope, $rootScope, $http, $timeout) ->
+
+		TWO_MEGABYTES = 2 * 1024 * 1024
+
+		$scope.bibtexPreview =
+			loading: false
+			shouldShowDots: false
+			error: false
+			data: null
+
+		# Callback fired when the `img` tag fails to load,
+		# `failedLoad` used to show the "No Preview" message
+		$scope.failedLoad = false
+		window.sl_binaryFilePreviewError = () =>
+			$scope.failedLoad = true
+			$scope.$apply()
+
+		# Callback fired when the `img` tag is done loading,
+		# `imgLoaded` is used to show the spinner gif while loading
+		$scope.imgLoaded = false
+		window.sl_binaryFilePreviewLoaded = () =>
+			$scope.imgLoaded = true
+			$scope.$apply()
+
 		$scope.extension = (file) ->
 			return file.name.split(".").pop()?.toLowerCase()
-	]
+
+		$scope.loadBibtexFilePreview = () ->
+			url = "/project/#{project_id}/file/#{$scope.openFile.id}?range=0-#{TWO_MEGABYTES}"
+			$scope.bibtexPreview.loading = true
+			$scope.bibtexPreview.shouldShowDots = false
+			$scope.$apply()
+			$http.get(url)
+				.success (data) ->
+					$scope.bibtexPreview.loading = false
+					$scope.bibtexPreview.error = false
+					# show dots when payload is closs to cutoff
+					if data.length >= (TWO_MEGABYTES - 200)
+						$scope.bibtexPreview.shouldShowDots = true
+					try
+						# remove last partial line
+						data = data.replace(/\n.*$/, '')
+					finally
+						$scope.bibtexPreview.data = data
+					$timeout($scope.setHeight, 0)
+				.error (err) ->
+					$scope.bibtexPreview.error = true
+					$scope.bibtexPreview.loading = false
+
+		$scope.setHeight = () ->
+			# Behold, a ghastly hack
+			guide = document.querySelector('.file-tree-inner')
+			table_wrap = document.querySelector('.bib-preview .scroll-container')
+			if table_wrap
+				desired_height = guide.offsetHeight - 44
+				if table_wrap.offsetHeight > desired_height
+					table_wrap.style.height = desired_height + 'px'
+					table_wrap.style['max-height'] = desired_height + 'px'
+
+		$scope.loadBibtexIfRequired = () ->
+			if $scope.extension($scope.openFile) == 'bib'
+				$scope.bibtexPreview.data = null
+				$scope.loadBibtexFilePreview()
+
+		$scope.loadBibtexIfRequired()
+
+	]

services/web/public/coffee/ide/connection/ConnectionManager.coffee

@@ -52,6 +52,7 @@ define [], () ->
 				"force new connection": true
 
 			@ide.socket.on "connect", () =>
+				sl_console.log "[socket.io connect] Connected"
 				@connected = true
 				@ide.pushEvent("connected")
 
@@ -73,6 +74,7 @@ define [], () ->
 
 
 			@ide.socket.on 'disconnect', () =>
+				sl_console.log "[socket.io disconnect] Disconnected"
 				@connected = false
 				@ide.pushEvent("disconnected")
 
@@ -97,9 +99,20 @@ define [], () ->
 				, 10 * 1000
 				
 		joinProject: () ->
+			sl_console.log "[joinProject] joining..."
 			@ide.socket.emit 'joinProject', {
 				project_id: @ide.project_id
 			}, (err, project, permissionsLevel, protocolVersion) =>
+				if err?
+					if err.message == "not authorized"
+						window.location = "/login?redir=#{encodeURI(window.location.pathname)}"
+					else
+						@ide.socket.disconnect()
+						@ide.showGenericMessageModal("Something went wrong connecting", """
+							Something went wrong connecting to your project. Please refresh is this continues to happen.
+						""")
+					return
+
 				if @$scope.protocolVersion? and @$scope.protocolVersion != protocolVersion
 					location.reload(true)
 

services/web/public/coffee/ide/editor/Document.coffee

@@ -6,7 +6,10 @@ define [
 		@getDocument: (ide, doc_id) ->
 			@openDocs ||= {}
 			if !@openDocs[doc_id]?
+				sl_console.log "[getDocument] Creating new document instance for #{doc_id}"
 				@openDocs[doc_id] = new Document(ide, doc_id)
+			else
+				sl_console.log "[getDocument] Returning existing document instance for #{doc_id}"
 			return @openDocs[doc_id]
 
 		@hasUnsavedChanges: () ->
@@ -107,11 +110,14 @@ define [
 			@wantToBeJoined = false
 			@_cancelJoin()
 			if (@doc? and @doc.hasBufferedOps())
+				sl_console.log "[leave] Doc has buffered ops, pushing callback for later"
 				@_leaveCallbacks ||= []
 				@_leaveCallbacks.push callback
 			else if !@connected
+				sl_console.log "[leave] Not connected, returning now"
 				callback()
 			else
+				sl_console.log "[leave] Leaving now"
 				@_leaveDoc(callback)
 
 		flush: () ->
@@ -146,12 +152,15 @@ define [
 			if !inflightOp? and !pendingOp?
 				# there's nothing going on
 				saved = true
-			else if inflightOp == @oldInflightOp
-				saved = false
-			else if pendingOp?
+				sl_console.log "[pollSavedStatus] no inflight or pending ops"
+			else if inflightOp? and inflightOp == @oldInflightOp
+				# The same inflight op has been sitting unacked since we
+				# last checked.
 				saved = false
+				sl_console.log "[pollSavedStatus] inflight op is same as before"
 			else
 				saved = true
+				sl_console.log "[pollSavedStatus] assuming saved (inflightOp?: #{inflightOp?}, pendingOp?: #{pendingOp?})"
 
 			@oldInflightOp = inflightOp
 			return saved
@@ -172,14 +181,14 @@ define [
 				update: update
 
 			if window.disconnectOnAck? and Math.random() < window.disconnectOnAck
-				console.log "Disconnecting on ack", update
+				sl_console.log "Disconnecting on ack", update
 				window._ide.socket.socket.disconnect()
 				# Pretend we never received the ack
 				return
 
 			if window.dropAcks? and Math.random() < window.dropAcks
 				if !update.op? # Only drop our own acks, not collaborator updates
-					console.log "Simulating a lost ack", update
+					sl_console.log "Simulating a lost ack", update
 					return
 
 			if update?.doc == @doc_id and @doc?
@@ -189,15 +198,18 @@ define [
 					@leave()
 
 		_onDisconnect: () ->
+			sl_console.log '[onDisconnect] disconnecting'
 			@connected = false
 			@joined = false
 			@doc?.updateConnectionState "disconnected"
 
 		_onReconnect: () ->
+			sl_console.log "[onReconnect] reconnected (joined project)"
 			@ide.pushEvent "reconnected:afterJoinProject"
 
 			@connected = true
 			if @wantToBeJoined or @doc?.hasBufferedOps()
+				sl_console.log "[onReconnect] Rejoining (wantToBeJoined: #{@wantToBeJoined} OR hasBufferedOps: #{@doc?.hasBufferedOps()})"
 				@_joinDoc (error) =>
 					return @_onError(error) if error?
 					@doc.updateConnectionState "ok"
@@ -225,16 +237,25 @@ define [
 					callback()
 
 		_leaveDoc: (callback = (error) ->) ->
+			sl_console.log '[_leaveDoc] Sending leaveDoc request'
 			@ide.socket.emit 'leaveDoc', @doc_id, (error) =>
 				return callback(error) if error?
 				@joined = false
 				for callback in @_leaveCallbacks or []
+					sl_console.log '[_leaveDoc] Calling buffered callback', callback
 					callback(error)
 				delete @_leaveCallbacks
 				callback(error)
 
 		_cleanUp: () ->
-			delete Document.openDocs[@doc_id]
+			if Document.openDocs[@doc_id] == @
+				sl_console.log "[_cleanUp] Removing self (#{@doc_id}) from in openDocs"
+				delete Document.openDocs[@doc_id]
+			else
+				# It's possible that this instance has error, and the doc has been reloaded.
+				# This creates a new instance in Document.openDoc with the same id. We shouldn't
+				# clear it because it's not this instance.
+				sl_console.log "[_cleanUp] New instance of (#{@doc_id}) created. Not removing"
 			@_unBindFromEditorEvents()
 			@_unBindFromSocketEvents()
 
@@ -276,5 +297,9 @@ define [
 			console.error "ShareJS error", error, meta
 			ga?('send', 'event', 'error', "shareJsError", "#{error.message} - #{@ide.socket.socket.transport.name}" )
 			@doc?.clearInflightAndPendingOps()
-			@_cleanUp()
 			@trigger "error", error, meta
+			# The clean up should run after the error is triggered because the error triggers a
+			# disconnect. If we run the clean up first, we remove our event handlers and miss
+			# the disconnect event, which means we try to leaveDoc when the connection comes back.
+			# This could intefere with the new connection of a new instance of this document.
+			@_cleanUp()

services/web/public/coffee/ide/editor/EditorManager.coffee

@@ -41,6 +41,7 @@ define [
 			@openDoc(doc)
 
 		openDoc: (doc, options = {}) ->
+			sl_console.log "[openDoc] Opening #{doc.id}"
 			@$scope.ui.view = "editor"
 
 			done = () =>
@@ -74,8 +75,10 @@ define [
 					done()
 
 		_openNewDocument: (doc, callback = (error, sharejs_doc) ->) ->
+			sl_console.log "[_openNewDocument] Opening..."
 			current_sharejs_doc = @$scope.editor.sharejs_doc
 			if current_sharejs_doc?
+				sl_console.log "[_openNewDocument] Leaving existing open doc..."
 				current_sharejs_doc.leaveAndCleanUp()
 				@_unbindFromDocumentEvents(current_sharejs_doc)
 

services/web/public/coffee/ide/editor/ShareJsDoc.coffee

@@ -29,12 +29,13 @@ define [
 				send: (update) =>
 					@_startInflightOpTimeout(update)
 					if window.disconnectOnUpdate? and Math.random() < window.disconnectOnUpdate
-						console.log "Disconnecting on update", update
+						sl_console.log "Disconnecting on update", update
 						window._ide.socket.socket.disconnect()
 					if window.dropUpdates? and Math.random() < window.dropUpdates
-						console.log "Simulating a lost update", update
+						sl_console.log "Simulating a lost update", update
 						return
-					@socket.emit "applyOtUpdate", @doc_id, update
+					@socket.emit "applyOtUpdate", @doc_id, update, (error) =>
+						return @_handleError(error) if error?
 				state: "ok"
 				id:    @socket.socket.sessionid
 			}
@@ -95,6 +96,7 @@ define [
 			@_doc.flush()
 
 		updateConnectionState: (state) ->
+			sl_console.log "[updateConnectionState] Setting state to #{state}"
 			@connection.state = state
 			@connection.id = @socket.socket.sessionid
 			@_doc.autoOpen = false
@@ -110,12 +112,14 @@ define [
 		detachFromAce: () -> @_doc.detach_ace?()
 	
 		INFLIGHT_OP_TIMEOUT: 5000 # Retry sending ops after 5 seconds without an ack
+		WAIT_FOR_CONNECTION_TIMEOUT: 500 # If we're waiting for the project to join, try again in 0.5 seconds
 		_startInflightOpTimeout: (update) ->
 			@_startFatalTimeoutTimer(update)
-			timer = setTimeout () =>
+			retryOp = () =>
 				# Only send the update again if inflightOp is still populated
 				# This can be cleared when hard reloading the document in which
 				# case we don't want to keep trying to send it.
+				sl_console.log "[inflightOpTimeout] Trying op again"
 				if @_doc.inflightOp?
 					# When there is a socket.io disconnect, @_doc.inflightSubmittedIds
 					# is updated with the socket.io client id of the current op in flight
@@ -124,8 +128,18 @@ define [
 					# So we need both depending on whether the op was submitted before
 					# one or more disconnects, or if it was submitted during the current session.
 					update.dupIfSource = [@connection.id, @_doc.inflightSubmittedIds...]
-					@connection.send(update)
-			, @INFLIGHT_OP_TIMEOUT
+					
+					# We must be joined to a project for applyOtUpdate to work on the real-time
+					# service, so don't send an op if we're not. Connection state is set to 'ok'
+					# when we've joined the project
+					if @connection.state != "ok"
+						sl_console.log "[inflightOpTimeout] Not connected, retrying in 0.5s"
+						timer = setTimeout retryOp, @WAIT_FOR_CONNECTION_TIMEOUT
+					else
+						sl_console.log "[inflightOpTimeout] Sending"
+						@connection.send(update)
+			
+			timer = setTimeout retryOp, @INFLIGHT_OP_TIMEOUT
 			@_doc.inflightCallbacks.push () =>
 				@_clearFatalTimeoutTimer()
 				clearTimeout timer

services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/AutoCompleteManager.coffee

@@ -1,13 +1,13 @@
 define [
 	"ide/editor/directives/aceEditor/auto-complete/SuggestionManager"
-	"ide/editor/directives/aceEditor/auto-complete/Snippets"
+	"ide/editor/directives/aceEditor/auto-complete/SnippetManager"
 	"ace/ace"
 	"ace/ext-language_tools"
-], (SuggestionManager, Snippets) ->
+], (SuggestionManager, SnippetManager) ->
 	Range = ace.require("ace/range").Range
 
 	getLastCommandFragment = (lineUpToCursor) ->
-		if m = lineUpToCursor.match(/(\\[^\\ ]+)$/)
+		if m = lineUpToCursor.match(/(\\[^\\]+)$/)
 			return m[1]
 		else
 			return null
@@ -38,19 +38,20 @@ define [
 				enableLiveAutocompletion: false
 			})
 
-			SnippetCompleter =
-				getCompletions: (editor, session, pos, prefix, callback) ->
-					callback null, Snippets
+			SnippetCompleter = new SnippetManager()
 
 			references = @$scope.$root._references
 			ReferencesCompleter =
 				getCompletions: (editor, session, pos, prefix, callback) ->
-					range = new Range(pos.row, 0, pos.row, pos.column)
-					lineUpToCursor = editor.getSession().getTextRange(range)
+					upToCursorRange = new Range(pos.row, 0, pos.row, pos.column)
+					lineUpToCursor = editor.getSession().getTextRange(upToCursorRange)
 					commandFragment = getLastCommandFragment(lineUpToCursor)
 					if commandFragment
-						citeMatch = commandFragment.match(/^~?\\([a-z]*cite[a-z]?){(.*,)?(\w*)/)
+						citeMatch = commandFragment.match(/^~?\\([a-z]*cite[a-z]*(?:\[.*])?){([^}]*, *)?(\w*)/)
 						if citeMatch
+							beyondCursorRange = new Range(pos.row, pos.column, pos.row, 99999)
+							lineBeyondCursor = editor.getSession().getTextRange(beyondCursorRange)
+							needsClosingBrace = !lineBeyondCursor.match(/^[^{]*}/)
 							commandName = citeMatch[1]
 							previousArgs = citeMatch[2]
 							currentArg = citeMatch[3]
@@ -59,8 +60,8 @@ define [
 							previousArgsCaption = if previousArgs.length > 8 then "…," else previousArgs
 							result = []
 							result.push {
-								caption: "\\#{commandName}{",
-								snippet: "\\#{commandName}{",
+								caption: "\\#{commandName}{}",
+								snippet: "\\#{commandName}{}",
 								meta: "reference",
 								score: 11000
 							}
@@ -68,8 +69,8 @@ define [
 								references.keys.forEach (key) ->
 									if !(key in [null, undefined])
 										result.push({
-											caption: "\\#{commandName}{#{previousArgsCaption}#{key}",
-											value: "\\#{commandName}{#{previousArgs}#{key}",
+											caption: "\\#{commandName}{#{previousArgsCaption}#{key}#{if needsClosingBrace then '}' else ''}",
+											value: "\\#{commandName}{#{previousArgs}#{key}#{if needsClosingBrace then '}' else ''}",
 											meta: "reference",
 											score: 10000
 										})

services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/SnippetManager.coffee

@@ -0,0 +1,159 @@
+define () ->
+	environments = [
+		"abstract",
+		"align", "align*",
+		"equation", "equation*",
+		"gather", "gather*",
+		"multline", "multline*",
+		"split",
+		"verbatim"
+	]
+
+	staticSnippets = for env in environments
+		{
+			caption: "\\begin{#{env}}..."
+			snippet: """
+				\\begin{#{env}}
+				\t$1
+				\\end{#{env}}
+			"""
+			meta: "env"
+		}
+
+	staticSnippets = staticSnippets.concat [{
+		caption: "\\begin{array}..."
+		snippet: """
+			\\begin{array}{${1:cc}}
+			\t$2 & $3 \\\\\\\\
+			\t$4 & $5
+			\\end{array}
+		"""
+		meta: "env"
+	}, {
+		caption: "\\begin{figure}..."
+		snippet: """
+			\\begin{figure}
+			\t\\centering
+			\t\\includegraphics{$1}
+			\t\\caption{${2:Caption}}
+			\t\\label{${3:fig:my_label}}
+			\\end{figure}
+		"""
+		meta: "env"
+	}, {
+		caption: "\\begin{tabular}..."
+		snippet: """
+			\\begin{tabular}{${1:c|c}}
+			\t$2 & $3 \\\\\\\\
+			\t$4 & $5
+			\\end{tabular}
+		"""
+		meta: "env"
+	}, {
+		caption: "\\begin{table}..."
+		snippet: """
+			\\begin{table}[$1]
+			\t\\centering
+			\t\\begin{tabular}{${2:c|c}}
+			\t\t$3 & $4 \\\\\\\\
+			\t\t$5 & $6
+			\t\\end{tabular}
+			\t\\caption{${7:Caption}}
+			\t\\label{${8:tab:my_label}}
+			\\end{table}
+		"""
+		meta: "env"
+	}, {
+		caption: "\\begin{list}..."
+		snippet: """
+			\\begin{list}
+			\t\\item $1
+			\\end{list}
+		"""
+		meta: "env"
+	}, {
+		caption: "\\begin{enumerate}..."
+		snippet: """
+			\\begin{enumerate}
+			\t\\item $1
+			\\end{enumerate}
+		"""
+		meta: "env"
+	}, {
+		caption: "\\begin{itemize}..."
+		snippet: """
+			\\begin{itemize}
+			\t\\item $1
+			\\end{itemize}
+		"""
+		meta: "env"
+	}, {
+		caption: "\\begin{frame}..."
+		snippet: """
+			\\begin{frame}{${1:Frame Title}}
+			\t$2
+			\\end{frame}
+		"""
+		meta: "env"
+	}]
+
+
+	parseCustomEnvironments = (text) ->
+		re = /^\\newenvironment{(\w+)}.*$/gm
+		result = []
+		iterations = 0
+		while match = re.exec(text)
+			result.push {name: match[1], whitespace: null}
+			iterations += 1
+			if iterations >= 1000
+				return result
+		return result
+
+
+	parseBeginCommands = (text) ->
+		re = /^\\begin{(\w+)}.*\n([\t ]*).*$/gm
+		result = []
+		iterations = 0
+		while match = re.exec(text)
+			result.push {name: match[1], whitespace: match[2]}
+			iterations += 1
+			if iterations >= 1000
+				return result
+		return result
+
+	class SnippetManager
+		getCompletions: (editor, session, pos, prefix, callback) ->
+			docText = session.getValue()
+			customEnvironments = parseCustomEnvironments(docText)
+			beginCommands = parseBeginCommands(docText)
+			parsedItemsMap = {}
+			for environment in customEnvironments
+				parsedItemsMap[environment.name] = environment
+			for command in beginCommands
+				parsedItemsMap[command.name] = command
+			parsedItems = _.values(parsedItemsMap)
+			snippets = staticSnippets.concat(
+				parsedItems.map (item) ->
+					{
+						caption: "\\begin{#{item.name}}..."
+						snippet: """
+							\\begin{#{item.name}}
+							#{item.whitespace || ''}$0
+							\\end{#{item.name}}
+						"""
+						meta: "env"
+					}
+			).concat(
+				# arguably these `end` commands shouldn't be here, as they're not snippets
+				# but this is where we have access to the `begin` environment names
+				# *shrug*
+				parsedItems.map (item) ->
+					{
+						caption: "\\end{#{item.name}}"
+						value: "\\end{#{item.name}}"
+						meta: "env"
+					}
+			)
+			callback null, snippets
+
+	return SnippetManager

services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/Snippets.coffee

@@ -1,100 +0,0 @@
-define () ->
-	environments = [
-		"abstract", 
-		"align", "align*",
-		"equation", "equation*",
-		"gather", "gather*",
-		"multline", "multline*",
-		"split",
-		"verbatim"
-	]
-
-	snippets = for env in environments
-		{
-			caption: "\\begin{#{env}}..."
-			snippet: """
-				\\begin{#{env}}
-				\t$1
-				\\end{#{env}}
-			"""
-			meta: "env"
-		}
-
-	snippets = snippets.concat [{
-		caption: "\\begin{array}..."
-		snippet: """
-			\\begin{array}{${1:cc}}
-			\t$2 & $3 \\\\\\\\
-			\t$4 & $5
-			\\end{array}
-		"""
-		meta: "env"
-	}, {
-		caption: "\\begin{figure}..."
-		snippet: """
-			\\begin{figure}
-			\t\\centering
-			\t\\includegraphics{$1}
-			\t\\caption{${2:Caption}}
-			\t\\label{${3:fig:my_label}}
-			\\end{figure}
-		"""
-		meta: "env"
-	}, {
-		caption: "\\begin{tabular}..."
-		snippet: """
-			\\begin{tabular}{${1:c|c}}
-			\t$2 & $3 \\\\\\\\
-			\t$4 & $5
-			\\end{tabular}
-		"""
-		meta: "env"
-	}, {
-		caption: "\\begin{table}..."
-		snippet: """
-			\\begin{table}[$1]
-			\t\\centering
-			\t\\begin{tabular}{${2:c|c}}
-			\t\t$3 & $4 \\\\\\\\
-			\t\t$5 & $6
-			\t\\end{tabular}
-			\t\\caption{${7:Caption}}
-			\t\\label{${8:tab:my_label}}
-			\\end{table}
-		"""
-		meta: "env"
-	}, {
-		caption: "\\begin{list}..."
-		snippet: """
-			\\begin{list}
-			\t\\item $1
-			\\end{list}
-		"""
-		meta: "env"
-	}, {
-		caption: "\\begin{enumerate}..."
-		snippet: """
-			\\begin{enumerate}
-			\t\\item $1
-			\\end{enumerate}
-		"""
-		meta: "env"
-	}, {
-		caption: "\\begin{itemize}..."
-		snippet: """
-			\\begin{itemize}
-			\t\\item $1
-			\\end{itemize}
-		"""
-		meta: "env"
-	}, {
-		caption: "\\begin{frame}..."
-		snippet: """
-			\\begin{frame}{${1:Frame Title}}
-			\t$2
-			\\end{frame}
-		"""
-		meta: "env"
-	}]
-
-	return snippets

services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/SuggestionManager.coffee

@@ -1,11 +1,23 @@
 define [], () ->
+
 	class Parser
 		constructor: (@doc) ->
 
 		parse: () ->
+			# Safari regex is super slow, freezes browser for minutes on end,
+			# hacky solution: limit iterations
+			limit = null
+			if window?._ide?.browserIsSafari
+				limit = 100
+
 			commands = []
 			seen = {}
+			iterations = 0
 			while command = @nextCommand()
+				iterations += 1
+				if limit && iterations > limit
+					return commands
+
 				docState = @doc
 
 				optionalArgs = 0
@@ -28,7 +40,7 @@ define [], () ->
 
 		# Ignore single letter commands since auto complete is moot then.
 		commandRegex: /\\([a-zA-Z][a-zA-Z]+)/
-		
+
 		nextCommand: () ->
 			i = @doc.search(@commandRegex)
 			if i == -1
@@ -123,4 +135,3 @@ define [], () ->
 					completionBeforeCursor: completionBeforeCursor
 					completionAfterCursor: completionAfterCursor
 				}
-

services/web/public/coffee/ide/editor/sharejs/vendor/client/doc.coffee

@@ -192,6 +192,13 @@ class Doc
         callback error for callback in @inflightCallbacks
       else
         # The op applied successfully.
+        
+        # We may get multiple acks of the same message if we retried it, 
+        # so its ok if we receive an ack for a version that we've already gone past.
+        # If so, just ignore it
+        if msg.v < @version
+            return
+
         throw new Error('Invalid version from server') unless msg.v == @version
 
         @serverOps[@version] = oldInflightOp

services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee

@@ -1,7 +1,7 @@
 define [
 	"base"
 ], (App) ->
-	App.controller "FileTreeController", ["$scope", "$modal", "ide", ($scope, $modal, ide) ->
+	App.controller "FileTreeController", ["$scope", "$modal", "ide", "$rootScope", ($scope, $modal, ide, $rootScope) ->
 		$scope.openNewDocModal = () ->
 			$modal.open(
 				templateUrl: "newDocModalTemplate"

services/web/public/coffee/ide/file-tree/controllers/FileTreeEntityController.coffee

@@ -51,6 +51,19 @@ define [
 
 		$scope.$on "delete:selected", () ->
 			$scope.openDeleteModal() if $scope.entity.selected
+
+		$scope.iconTypeFromName = (name) ->
+			ext = name.split(".").pop()?.toLowerCase()
+			if ext in ["png", "pdf", "jpg", "jpeg", "gif"]
+				return "image"
+			else if ext in ["csv", "xls", "xlsx"]
+				return "table"
+			else if ext in ["py", "r"]
+				return "file-text"
+			else if ext in ['bib']
+				return 'book'
+			else
+				return "file"
 	]
 
 	App.controller "DeleteEntityModalController", [

services/web/public/coffee/ide/pdf/controllers/PdfController.coffee

@@ -4,7 +4,13 @@ define [
 	"libs/bib-log-parser"
 ], (App, LogParser, BibLogParser) ->
 	App.controller "PdfController", ($scope, $http, ide, $modal, synctex, event_tracking, localStorage) ->
+
 		autoCompile = true
+
+		# pdf.view = uncompiled | pdf | errors
+		$scope.pdf.view = if $scope?.pdf?.url then 'pdf' else 'uncompiled'
+		$scope.shouldShowLogs = false
+
 		$scope.$on "project:joined", () ->
 			return if !autoCompile
 			autoCompile = false
@@ -12,7 +18,8 @@ define [
 			$scope.hasPremiumCompile = $scope.project.features.compileGroup == "priority"
 
 		$scope.$on "pdf:error:display", () ->
-			$scope.pdf.error = true
+			$scope.pdf.view = 'errors'
+			$scope.pdf.renderingError = true
 
 		$scope.draft = localStorage("draft:#{$scope.project_id}") or false
 		$scope.$watch "draft", (new_value, old_value) ->
@@ -29,41 +36,72 @@ define [
 				_csrf: window.csrfToken
 			}
 
-		parseCompileResponse = (response) ->
+		parseCompileResponse = (response) ->		
+
 			# Reset everything
 			$scope.pdf.error      = false
 			$scope.pdf.timedout   = false
 			$scope.pdf.failure    = false
-			$scope.pdf.uncompiled = false
 			$scope.pdf.projectTooLarge = false
 			$scope.pdf.url        = null
+			$scope.pdf.clsiMaintenance = false
+			$scope.pdf.tooRecentlyCompiled = false
+			$scope.pdf.renderingError = false
+
+			# make a cache to look up files by name
+			fileByPath = {}
+			for file in response.outputFiles
+				fileByPath[file.path] = file
 
 			if response.status == "timedout"
+				$scope.pdf.view = 'errors'
 				$scope.pdf.timedout = true
 			else if response.status == "autocompile-backoff"
-				$scope.pdf.uncompiled = true
+				$scope.pdf.view = 'uncompiled'
 			else if response.status == "project-too-large"
+				$scope.pdf.view = 'errors'
 				$scope.pdf.projectTooLarge = true
 			else if response.status == "failure"
+				$scope.pdf.view = 'errors'
 				$scope.pdf.failure = true
-				fetchLogs()
+				$scope.shouldShowLogs = true
+				fetchLogs(fileByPath['output.log'], fileByPath['output.blg'])
+			else if response.status == 'clsi-maintenance'
+				$scope.pdf.view = 'errors'
+				$scope.pdf.clsiMaintenance = true
+			else if response.status == "too-recently-compiled"
+				$scope.pdf.view = 'errors'
+				$scope.pdf.tooRecentlyCompiled = true
 			else if response.status == "success"
-				# define the base url
-				$scope.pdf.url = "/project/#{$scope.project_id}/output/output.pdf?cache_bust=#{Date.now()}"
+				$scope.pdf.view = 'pdf'
+				$scope.shouldShowLogs = false
+
+				# prepare query string
+				qs = {}
+				# define the base url. if the pdf file has a build number, pass it to the clsi in the url
+				if fileByPath['output.pdf']?.url?
+					$scope.pdf.url = fileByPath['output.pdf'].url
+				else if fileByPath['output.pdf']?.build?
+					build = fileByPath['output.pdf'].build
+					$scope.pdf.url = "/project/#{$scope.project_id}/build/#{build}/output/output.pdf"
+				else
+					$scope.pdf.url = "/project/#{$scope.project_id}/output/output.pdf"
+				# check if we need to bust cache (build id is unique so don't need it in that case)
+				if not fileByPath['output.pdf']?.build?
+					qs.cache_bust = "#{Date.now()}"
 				# add a query string parameter for the compile group
 				if response.compileGroup?
 					$scope.pdf.compileGroup = response.compileGroup
-					$scope.pdf.url = $scope.pdf.url + "&compileGroup=#{$scope.pdf.compileGroup}"
-				# make a cache to look up files by name
-				fileByPath = {}
-				for file in response.outputFiles
-					fileByPath[file.path] = file
-				# if the pdf file has a build number, pass it to the clsi
-				if fileByPath['output.pdf']?.build?
-					build = fileByPath['output.pdf'].build
-					$scope.pdf.url = $scope.pdf.url + "&build=#{build}"
+					qs.compileGroup = "#{$scope.pdf.compileGroup}"
+				if response.clsiServerId?
+					qs.clsiserverid = response.clsiServerId
+					ide.clsiServerId = response.clsiServerId
+				# convert the qs hash into a query string and append it
+				qs_args = ("#{k}=#{v}" for k, v of qs)
+				$scope.pdf.qs = if qs_args.length then "?" + qs_args.join("&") else ""
+				$scope.pdf.url += $scope.pdf.qs
 
-				fetchLogs(fileByPath['output.log'])
+				fetchLogs(fileByPath['output.log'], fileByPath['output.blg'])
 
 			IGNORE_FILES = ["output.fls", "output.fdb_latexmk"]
 			$scope.pdf.outputFiles = []
@@ -77,51 +115,84 @@ define [
 						file.name = "#{file.path.replace(/^output\./, "")} file"
 					else
 						file.name = file.path
+					file.url = "/project/#{project_id}/output/#{file.path}"
+					if response.clsiServerId?
+						file.url = file.url + "?clsiserverid=#{response.clsiServerId}"
 					$scope.pdf.outputFiles.push file
 
-		fetchLogs = (outputFile) ->
-			qs = if outputFile?.build? then "?build=#{outputFile.build}" else ""
-			$http.get "/project/#{$scope.project_id}/output/output.log" + qs
-				.success (log) ->
-					#console.log ">>", log
-					$scope.pdf.rawLog = log
-					logEntries = LogParser.parse(log, ignoreDuplicates: true)
-					#console.log ">>", logEntries
-					$scope.pdf.logEntries = logEntries
-					$scope.pdf.logEntries.all = logEntries.errors.concat(logEntries.warnings).concat(logEntries.typesetting)
-					# # # #
-					proceed = () ->
-						$scope.pdf.logEntryAnnotations = {}
-						for entry in logEntries.all
-							if entry.file?
-								entry.file = normalizeFilePath(entry.file)
-								entity = ide.fileTreeManager.findEntityByPath(entry.file)
-								if entity?
-									$scope.pdf.logEntryAnnotations[entity.id] ||= []
-									$scope.pdf.logEntryAnnotations[entity.id].push {
-										row: entry.line - 1
-										type: if entry.level == "error" then "error" else "warning"
-										text: entry.message
-									}
-					# Get the biber log and parse it too
-					$http.get "/project/#{$scope.project_id}/output/output.blg" + qs
-						.success (log) ->
-							window._s = $scope
-							biberLogEntries = BibLogParser.parse(log, {})
-							if $scope.pdf.logEntries
-								entries = $scope.pdf.logEntries
-								all = biberLogEntries.errors.concat(biberLogEntries.warnings)
-								entries.all = entries.all.concat(all)
-								entries.errors = entries.errors.concat(biberLogEntries.errors)
-								entries.warnings = entries.warnings.concat(biberLogEntries.warnings)
-							proceed()
-						.error (e) ->
-							console.error ">> error", e
-							proceed()
-					# # # #
-				.error () ->
-					$scope.pdf.logEntries = []
-					$scope.pdf.rawLog = ""
+
+		fetchLogs = (logFile, blgFile) ->
+
+			getFile = (name, file) ->
+				opts =
+					method:"GET"
+					params:
+						build:file.build
+						clsiserverid:ide.clsiServerId
+				if file.url?  # FIXME clean this up when we have file.urls out consistently
+					opts.url = file.url
+				else if file?.build?
+					opts.url = "/project/#{$scope.project_id}/build/#{file.build}/output/#{name}"
+				else
+					opts.url = "/project/#{$scope.project_id}/output/#{name}"
+				return $http(opts)
+
+			# accumulate the log entries
+			logEntries =
+				all: []
+				errors: []
+				warnings: []
+
+			accumulateResults = (newEntries) ->
+				for key in ['all', 'errors', 'warnings']
+					logEntries[key] = logEntries[key].concat newEntries[key]
+
+			# use the parsers for each file type
+			processLog = (log) ->
+				$scope.pdf.rawLog = log
+				{errors, warnings, typesetting} = LogParser.parse(log, ignoreDuplicates: true)
+				all = [].concat errors, warnings, typesetting
+				accumulateResults {all, errors, warnings}
+
+			processBiber = (log) ->
+				{errors, warnings} = BibLogParser.parse(log, {})
+				all = [].concat errors, warnings
+				accumulateResults {all, errors, warnings}
+
+			# output the results
+			handleError = () ->
+				$scope.pdf.logEntries = []
+				$scope.pdf.rawLog = ""
+
+			annotateFiles = () ->
+				$scope.pdf.logEntries = logEntries
+				$scope.pdf.logEntryAnnotations = {}
+				for entry in logEntries.all
+					if entry.file?
+						entry.file = normalizeFilePath(entry.file)
+						entity = ide.fileTreeManager.findEntityByPath(entry.file)
+						if entity?
+							$scope.pdf.logEntryAnnotations[entity.id] ||= []
+							$scope.pdf.logEntryAnnotations[entity.id].push {
+								row: entry.line - 1
+								type: if entry.level == "error" then "error" else "warning"
+								text: entry.message
+							}
+
+			# retrieve the logfile and process it
+			response = getFile('output.log', logFile)
+				.success processLog
+				.error handleError
+
+			if blgFile?	# retrieve the blg file if present
+				response.success () ->
+					getFile('output.blg', blgFile)
+						# ignore errors in biber file
+						.success processBiber
+						# display the combined result
+						.then annotateFiles
+			else # otherwise just display the result
+				response.success annotateFiles
 
 		getRootDocOverride_id = () ->
 			doc = ide.editorManager.getCurrentDocValue()
@@ -157,7 +228,9 @@ define [
 					parseCompileResponse(data)
 				.error () ->
 					$scope.pdf.compiling = false
+					$scope.pdf.renderingError = false
 					$scope.pdf.error = true
+					$scope.pdf.view = 'errors'
 
 		# This needs to be public.
 		ide.$scope.recompile = $scope.recompile
@@ -166,18 +239,18 @@ define [
 			$http {
 				url: "/project/#{$scope.project_id}/output"
 				method: "DELETE"
+				params:
+					clsiserverid:ide.clsiServerId
 				headers:
 					"X-Csrf-Token": window.csrfToken
 			}
 
 		$scope.toggleLogs = () ->
-			if !$scope.pdf.view? or $scope.pdf.view == "pdf"
-				$scope.pdf.view = "logs"
-			else
-				$scope.pdf.view = "pdf"
+			$scope.shouldShowLogs = !$scope.shouldShowLogs
 
 		$scope.showPdf = () ->
 			$scope.pdf.view = "pdf"
+			$scope.shouldShowLogs = false
 
 		$scope.toggleRawLog = () ->
 			$scope.pdf.showRawLog = !$scope.pdf.showRawLog
@@ -250,6 +323,7 @@ define [
 							file: path
 							line: row + 1
 							column: column
+							clsiserverid:ide.clsiServerId
 						}
 					})
 					.success (data) ->
@@ -277,6 +351,7 @@ define [
 							page: position.page + 1
 							h: position.offset.left.toFixed(2)
 							v: position.offset.top.toFixed(2)
+							clsiserverid:ide.clsiServerId
 						}
 					})
 					.success (data) ->

services/web/public/coffee/ide/pdfng/directives/pdfRenderer.coffee

@@ -16,7 +16,7 @@ define [
 				# PDFJS.disableStream
 				# PDFJS.disableRange
 				@scale = @options.scale || 1
-				@pdfjs = PDFJS.getDocument @url
+				@pdfjs = PDFJS.getDocument {url: @url, rangeChunkSize: 2*65536}
 				@pdfjs.onProgress = @options.progressCallback
 				@document = $q.when(@pdfjs)
 				@navigateFn = @options.navigateFn

services/web/public/coffee/ide/pdfng/directives/pdfViewer.coffee

@@ -290,11 +290,18 @@ define [
 						rescaleTimer = null
 					, 0
 
+				spinnerTimer = null
 				doRescale = (scale) ->
 					# console.log 'doRescale', scale
 					return unless scale?
 					origposition = angular.copy scope.position
 					# console.log 'origposition', origposition
+					
+					if not spinnerTimer?
+						spinnerTimer = setTimeout () ->
+							spinner.add(element)
+							spinnerTimer = null
+						, 100
 					layoutReady.promise.then (parentSize) ->
 						[h, w] = parentSize
 						# console.log 'in promise', h, w
@@ -312,7 +319,6 @@ define [
 							scope.$emit 'pdf:error', error
 
 				elementTimer = null
-				spinnerTimer = null
 				updateLayout = () ->
 					# if element is zero-sized keep checking until it is ready
 					# console.log 'checking element ready', element.height(), element.width()
@@ -329,11 +335,6 @@ define [
 						]
 						# console.log 'resolving layoutReady with', scope.parentSize
 						$timeout () ->
-							if not spinnerTimer?
-								spinnerTimer = setTimeout () ->
-									spinner.add(element)
-									spinnerTimer = null
-								, 100
 							layoutReady.resolve scope.parentSize
 							scope.$emit 'flash-controls'
 

services/web/public/coffee/ide/references/ReferencesManager.coffee

@@ -11,6 +11,9 @@ define [
 					if entity?.name?.match /.*\.bib$/
 						@indexReferences([doc.doc_id], true)
 
+			@$scope.$on 'references:should-reindex', (e, data) =>
+				@indexAllReferences(true)
+
 			# When we join the project:
 			#   index all references files
 			#   and don't broadcast to all clients

services/web/public/coffee/ide/settings/services/settings.coffee

@@ -10,5 +10,10 @@ define [
 			saveProjectSettings: (data) ->
 				data._csrf = window.csrfToken
 				ide.$http.post "/project/#{ide.project_id}/settings", data
+				
+			saveProjectAdminSettings: (data) ->
+				data._csrf = window.csrfToken
+				ide.$http.post "/project/#{ide.project_id}/settings/admin", data
+				
 		}
 	]

services/web/public/coffee/ide/share/controllers/ShareProjectModalController.coffee

@@ -143,7 +143,7 @@ define [
 
 		$scope.makePublic = () ->
 			$scope.project.publicAccesLevel = $scope.inputs.privileges
-			settings.saveProjectSettings({publicAccessLevel: $scope.inputs.privileges})
+			settings.saveProjectAdminSettings({publicAccessLevel: $scope.inputs.privileges})
 			$modalInstance.close()
 
 		$scope.cancel = () ->
@@ -153,7 +153,7 @@ define [
 	App.controller "MakePrivateModalController", ["$scope", "$modalInstance", "settings", ($scope, $modalInstance, settings) ->
 		$scope.makePrivate = () ->
 			$scope.project.publicAccesLevel = "private"
-			settings.saveProjectSettings({publicAccessLevel: "private"})
+			settings.saveProjectAdminSettings({publicAccessLevel: "private"})
 			$modalInstance.close()
 
 		$scope.cancel = () ->

services/web/public/coffee/ide/wordcount/controllers/WordCountModalController.coffee

@@ -5,11 +5,15 @@ define [
 		$scope.status = 
 			loading:true
 			
-		$http.get("/project/#{ide.project_id}/wordcount")
+		opts =
+			url:"/project/#{ide.project_id}/wordcount"
+			method:"GET"
+			params:
+				clsiserverid:ide.clsiServerId
+		$http opts
 			.success (data) ->
 				$scope.status.loading = false
 				$scope.data = data.texcount
-				console.log $scope.data
 			.error () ->
 				$scope.status.error = true
 

services/web/public/coffee/libs.coffee

@@ -12,6 +12,7 @@ define [
 	"libs/angular-cookies"
 	"libs/passfield"
 	"libs/sixpack"
+	"libs/groove"
 	"libs/angular-sixpack"
 	"libs/ng-tags-input-3.0.0"
 ], () ->

services/web/public/coffee/main.coffee

@@ -15,7 +15,8 @@ define [
 	"main/annual-upgrade"
 	"main/register-users"
 	"main/subscription/group-subscription-invite-controller"
-	"main/universties-site"
+	"main/contact-us"
+	"main/learn"
 	"analytics/AbTestingManager"
 	"directives/asyncForm"
 	"directives/stopPropagation"
@@ -30,3 +31,6 @@ define [
 	"__MAIN_CLIENTSIDE_INCLUDES__"
 ], () ->
 	angular.bootstrap(document.body, ["SharelatexApp"])
+
+	
+

services/web/public/coffee/main/contact-us.coffee

@@ -0,0 +1,66 @@
+define [
+	"base"
+	"libs/platform"
+], (App, platform) ->
+
+
+	App.controller 'ContactModal', ($scope, $modal) ->
+		$scope.contactUsModal = () ->
+			modalInstance = $modal.open(
+				templateUrl: "supportModalTemplate"
+				controller: "SupportModalController"
+			)
+
+	App.controller 'SupportModalController', ($scope, $modalInstance) ->
+		$scope.form = {}
+		$scope.sent = false
+		$scope.sending = false
+		$scope.contactUs = ->
+			if !$scope.form.email?
+				console.log "email not set"
+				return
+			$scope.sending = true
+			ticketNumber = Math.floor((1 + Math.random()) * 0x10000).toString(32)
+			message = $scope.form.message
+			if $scope.form.project_url?
+				message	= "#{message}\n\n project_url = #{$scope.form.project_url}" 
+			params =
+				email: $scope.form.email
+				message: message or ""
+				subject: $scope.form.subject + " - [#{ticketNumber}]"
+				labels: "support"
+				about: "<div>browser: #{platform?.name} #{platform?.version}</div>
+						<div>os: #{platform?.os?.family} #{platform?.os?.version}</div>"
+
+			Groove.createTicket params, (err, json)->
+				$scope.sent = true
+				$scope.$apply()
+
+		$scope.close = () ->
+			$modalInstance.close()
+
+
+	App.controller 'UniverstiesContactController', ($scope, $modal) ->
+
+		$scope.form = {}
+		$scope.sent = false
+		$scope.sending = false
+		$scope.contactUs = ->
+			if !$scope.form.email?
+				console.log "email not set"
+				return
+			$scope.sending = true
+			ticketNumber = Math.floor((1 + Math.random()) * 0x10000).toString(32)
+			params =
+				name: $scope.form.name || $scope.form.email
+				email: $scope.form.email
+				labels: "#{$scope.form.source} accounts"
+				message: "Please contact me with more details"
+				subject: $scope.form.subject + " - [#{ticketNumber}]"
+				about : "#{$scope.form.position || ''} #{$scope.form.university || ''}"
+
+			Groove.createTicket params, (err, json)->
+				$scope.sent = true
+				$scope.$apply()
+
+

services/web/public/coffee/main/learn.coffee

@@ -0,0 +1,92 @@
+define [
+	"base"
+], (App) ->
+	
+	App.factory "algoliawiki", ->
+		if window.sharelatex?.algolia? and window.sharelatex.algolia?.indexes?.wiki?
+			client = new AlgoliaSearch(window.sharelatex.algolia?.app_id, window.sharelatex.algolia?.api_key)
+			index = client.initIndex(window.sharelatex.algolia?.indexes?.wiki)
+			return index
+
+	App.controller "SearchWikiController", ($scope, algoliawiki, _, $modal) ->
+		algolia = algoliawiki
+		$scope.hits = []
+
+		$scope.clearSearchText = ->
+			$scope.searchQueryText = ""
+			updateHits []
+
+		$scope.safeApply = (fn)->
+			phase = $scope.$root.$$phase
+			if(phase == '$apply' || phase == '$digest')
+				$scope.$eval(fn)
+			else
+				$scope.$apply(fn)
+
+		buildHitViewModel = (hit)->
+			page_underscored = hit.pageName.replace(/\s/g,'_')
+			section_underscored = hit.sectionName.replace(/\s/g,'_')
+			content = hit._highlightResult.content.value
+			# Replace many new lines
+			content = content.replace(/\n\n+/g, "\n\n")
+			lines = content.split("\n")
+			# Only show the lines that have a highlighted match
+			matching_lines = []
+			for line in lines
+				if !line.match(/^\[edit\]/)
+					content += line + "\n"
+					if line.match(/<em>/)
+						matching_lines.push line
+			content = matching_lines.join("\n...\n")
+			result =
+				name : hit._highlightResult.pageName.value + " - " + hit._highlightResult.sectionName.value
+				url :"/learn/#{page_underscored}##{section_underscored}"
+				content: content
+			return result
+			
+		updateHits = (hits)->
+			$scope.safeApply ->
+				$scope.hits = hits
+
+		$scope.search = ->
+			query = $scope.searchQueryText
+			if !query? or query.length == 0
+				updateHits []
+				return
+				
+			algolia.search query, (err, response)->
+				if response.hits.length == 0
+					updateHits []
+				else
+					hits = _.map response.hits, buildHitViewModel
+					updateHits hits
+
+		$scope.showMissingTemplateModal = () ->
+			modalInstance = $modal.open(
+				templateUrl: "missingWikiPageModal"
+				controller: "MissingWikiPageController"
+			)
+
+
+	App.controller 'MissingWikiPageController', ($scope, $modalInstance) ->
+		$scope.form = {}
+		$scope.sent = false
+		$scope.sending = false
+		$scope.contactUs = ->
+			if !$scope.form.message?
+				console.log "message not set"
+				return
+			$scope.sending = true
+			ticketNumber = Math.floor((1 + Math.random()) * 0x10000).toString(32)
+			params =
+				email: $scope.form.email or "support@sharelatex.com"
+				message: $scope.form.message or ""
+				subject: "new wiki page sujection - [#{ticketNumber}]"
+				labels: "support wiki"
+
+			Groove.createTicket params, (err, json)->
+				$scope.sent = true
+				$scope.$apply()
+
+		$scope.close = () ->
+			$modalInstance.close()

services/web/public/coffee/main/project-list/modal-controllers.coffee

@@ -65,11 +65,11 @@ define [
 		$scope.projectsToLeave = projects.filter (project) -> project.accessLevel != "owner"
 
 		if $scope.projectsToLeave.length > 0 and $scope.projectsToDelete.length > 0
-			$scope.action = "Delete & Leave"
+			$scope.action = "delete-and-leave"
 		else if $scope.projectsToLeave.length == 0 and $scope.projectsToDelete.length > 0
-			$scope.action = "Delete"
+			$scope.action = "delete"
 		else
-			$scope.action = "Leave"
+			$scope.action = "leave"
 
 		$scope.delete = () ->
 			$modalInstance.close()

services/web/public/coffee/main/subscription/group-subscription-invite-controller.coffee

@@ -3,12 +3,31 @@ define [
 ], (App) ->
 	App.controller "GroupSubscriptionInviteController", ($scope, $http) ->
 
-		$scope.requestSent = false
+		$scope.inflight = false
+
+		if has_personal_subscription
+			$scope.view = "personalSubscription"
+		else 
+			$scope.view = "groupSubscriptionInvite"
+
+		$scope.keepPersonalSubscription = ->
+			$scope.view = "groupSubscriptionInvite"
+
+		$scope.cancelSubscription = ->
+			$scope.inflight = true
+			request = $http.post "/user/subscription/cancel", {_csrf:window.csrfToken}
+			request.success (data, status)->
+				$scope.inflight = false
+				$scope.view = "groupSubscriptionInvite"
+			request.error (data, status)->
+				console.log "the request failed"					
 
 		$scope.joinGroup = ->
-			$scope.requestSent = true
-			request = $http.post "/user/subscription/#{subscription_id}/group/begin-join", {_csrf:window.csrfToken}
+			$scope.view = "requestSent"
+			$scope.inflight = true
+			request = $http.post "/user/subscription/#{group_subscription_id}/group/begin-join", {_csrf:window.csrfToken}
 			request.success (data, status)->
+				$scope.inflight = false
 				if status != 200 # assume request worked
 					$scope.requestSent = false
 			request.error (data, status)->

services/web/public/coffee/main/templates.coffee

@@ -0,0 +1,62 @@
+define [
+	"base"
+], (App) ->
+	
+	App.factory "algoliawiki", ->
+		if window.sharelatex?.algolia? and window.sharelatex.algolia?.indexes?.wiki?
+			client = new AlgoliaSearch(window.sharelatex.algolia?.app_id, window.sharelatex.algolia?.api_key)
+			index = client.initIndex(window.sharelatex.algolia?.indexes?.wiki)
+			return index
+
+	App.controller "SearchWikiController", ($scope, algoliawiki, _) ->
+		algolia = algoliawiki
+		$scope.hits = []
+
+		$scope.clearSearchText = ->
+			$scope.searchQueryText = ""
+			updateHits []
+
+		$scope.safeApply = (fn)->
+			phase = $scope.$root.$$phase
+			if(phase == '$apply' || phase == '$digest')
+				$scope.$eval(fn)
+			else
+				$scope.$apply(fn)
+
+		buildHitViewModel = (hit)->
+			page_underscored = hit.pageName.replace(/\s/g,'_')
+			section_underscored = hit.sectionName.replace(/\s/g,'_')
+			content = hit._highlightResult.content.value
+			# Replace many new lines
+			content = content.replace(/\n\n+/g, "\n\n")
+			lines = content.split("\n")
+			# Only show the lines that have a highlighted match
+			matching_lines = []
+			for line in lines
+				if !line.match(/^\[edit\]/)
+					content += line + "\n"
+					if line.match(/<em>/)
+						matching_lines.push line
+			content = matching_lines.join("\n...\n")
+			result =
+				name : hit._highlightResult.pageName.value + " - " + hit._highlightResult.sectionName.value
+				url :"/learn/#{page_underscored}##{section_underscored}"
+				content: content
+			return result
+
+		updateHits = (hits)->
+			$scope.safeApply ->
+				$scope.hits = hits
+
+		$scope.search = ->
+			query = $scope.searchQueryText
+			if !query? or query.length == 0
+				updateHits []
+				return
+
+			algolia.search query, (err, response)->
+				if response.hits.length == 0
+					updateHits []
+				else
+					hits = _.map response.hits, buildHitViewModel
+					updateHits hits

services/web/public/coffee/main/universties-site.coffee

@@ -1,26 +0,0 @@
-define [
-	"base"
-], (App) ->
-
-	App.controller 'UniverstiesContactController', ($scope, $modal) ->
-
-		$scope.form = {}
-		$scope.sent = false
-		$scope.sending = false
-		$scope.contactUs = ->
-			if !$scope.form.email?
-				console.log "email not set"
-				return
-			$scope.sending = true
-			ticketNumber = Math.floor((1 + Math.random()) * 0x10000).toString(32)
-			params = 
-				name: $scope.form.name || $scope.form.email
-				email: $scope.form.email
-				labels: $scope.form.source
-				message: "Please contact me with more details"
-				subject: $scope.form.subject + " - [#{ticketNumber}]"
-				about : "#{$scope.form.position || ''} #{$scope.form.university || ''}"
-
-			Groove.createTicket params, (err, json)->
-				$scope.sent = true
-				$scope.$apply()

services/web/public/coffee/main/user-details.coffee

@@ -16,6 +16,7 @@ define [
 				_csrf : window.csrfToken
 
 		$scope.showForm = ->
+			GrooveWidget.toggle()
 			$scope.formVisable = true
 
 		$scope.getPercentComplete = ->