From 85f25b810c40c5cc9648bc9189904692ba1186cf Mon Sep 17 00:00:00 2001
From: James Allen <james@sharelatex.com>
Date: Wed, 14 Feb 2018 15:12:46 +0000
Subject: [PATCH 1/9] First pass at URL based linked files

---
 .../Features/Editor/EditorController.coffee   | 14 +--
 .../LinkedFiles/LinkedFilesController.coffee  | 25 +++++
 .../LinkedFiles/LinkedFilesRouter.coffee      | 11 +++
 .../Features/LinkedFiles/UrlAgent.coffee      | 15 +++
 .../Project/ProjectCreationHandler.coffee     |  2 +-
 .../ProjectEntityMongoUpdateHandler.coffee    |  3 +-
 .../Project/ProjectEntityUpdateHandler.coffee | 25 ++---
 .../ThirdPartyDataStore/UpdateMerger.coffee   | 26 +----
 .../Uploads/FileSystemImportManager.coffee    |  4 +-
 .../coffee/infrastructure/FileWriter.coffee   | 23 +++++
 services/web/app/coffee/models/File.coffee    |  1 +
 services/web/app/coffee/router.coffee         |  2 +
 .../acceptance/coffee/LinkedFilesTests.coffee | 94 +++++++++++++++++++
 .../coffee/helpers/MockFileStoreApi.coffee    | 12 ++-
 .../coffee/helpers/MockURLSource.coffee       |  7 ++
 15 files changed, 217 insertions(+), 47 deletions(-)
 create mode 100644 services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee
 create mode 100644 services/web/app/coffee/Features/LinkedFiles/LinkedFilesRouter.coffee
 create mode 100644 services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee
 create mode 100644 services/web/app/coffee/infrastructure/FileWriter.coffee
 create mode 100644 services/web/test/acceptance/coffee/LinkedFilesTests.coffee
 create mode 100644 services/web/test/acceptance/coffee/helpers/MockURLSource.coffee

diff --git a/services/web/app/coffee/Features/Editor/EditorController.coffee b/services/web/app/coffee/Features/Editor/EditorController.coffee
index b222dc297a..ad00445adb 100644
--- a/services/web/app/coffee/Features/Editor/EditorController.coffee
+++ b/services/web/app/coffee/Features/Editor/EditorController.coffee
@@ -23,11 +23,11 @@ module.exports = EditorController =
 			EditorRealTimeController.emitToRoom(project_id, 'reciveNewDoc', folder_id, doc, source)
 			callback(err, doc)
 
-	addFile: (project_id, folder_id, fileName, fsPath, source, user_id, callback = (error, file)->)->
+	addFile: (project_id, folder_id, fileName, fsPath, linkedFileData, source, user_id, callback = (error, file)->)->
 		fileName = fileName.trim()
-		logger.log {project_id, folder_id, fileName, fsPath}, "sending new file to project"
+		logger.log {project_id, folder_id, fileName, fsPath, linkedFileData, source, user_id}, "sending new file to project"
 		Metrics.inc "editor.add-file"
-		ProjectEntityUpdateHandler.addFile project_id, folder_id, fileName, fsPath, user_id, (err, fileRef, folder_id)=>
+		ProjectEntityUpdateHandler.addFile project_id, folder_id, fileName, fsPath, linkedFileData, user_id, (err, fileRef, folder_id)=>
 			if err?
 				logger.err err:err, project_id:project_id, folder_id:folder_id, fileName:fileName, "error adding file without lock"
 				return callback(err)
@@ -40,8 +40,8 @@ module.exports = EditorController =
 				EditorRealTimeController.emitToRoom(project_id, 'reciveNewDoc', folder_id, doc, source)
 			callback err, doc
 
-	upsertFile: (project_id, folder_id, fileName, fsPath, source, user_id, callback = (err, file) ->) ->
-		ProjectEntityUpdateHandler.upsertFile project_id, folder_id, fileName, fsPath, user_id, (err, file, didAddFile) ->
+	upsertFile: (project_id, folder_id, fileName, fsPath, linkedFileData, source, user_id, callback = (err, file) ->) ->
+		ProjectEntityUpdateHandler.upsertFile project_id, folder_id, fileName, fsPath, linkedFileData, user_id, (err, file, didAddFile) ->
 			return callback(err) if err?
 			if didAddFile
 				EditorRealTimeController.emitToRoom project_id, 'reciveNewFile', folder_id, file, source
@@ -56,8 +56,8 @@ module.exports = EditorController =
 					EditorRealTimeController.emitToRoom project_id, 'reciveNewDoc', lastFolder._id, doc, source
 				callback()
 
-	upsertFileWithPath: (project_id, elementPath, fsPath, source, user_id, callback) ->
-		ProjectEntityUpdateHandler.upsertFileWithPath project_id, elementPath, fsPath, user_id, (err, file, didAddFile, newFolders, lastFolder) ->
+	upsertFileWithPath: (project_id, elementPath, fsPath, linkedFileData, source, user_id, callback) ->
+		ProjectEntityUpdateHandler.upsertFileWithPath project_id, elementPath, fsPath, linkedFileData, user_id, (err, file, didAddFile, newFolders, lastFolder) ->
 			return callback(err) if err?
 			EditorController._notifyProjectUsersOfNewFolders project_id, newFolders, (err) ->
 				return callback(err) if err?
diff --git a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee
new file mode 100644
index 0000000000..dd0b088674
--- /dev/null
+++ b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee
@@ -0,0 +1,25 @@
+AuthenticationController = require '../Authentication/AuthenticationController'
+EditorController = require '../Editor/EditorController'
+
+module.exports = LinkedFilesController = {
+	Agents: {
+		url: require('./UrlAgent')
+	}
+
+	createLinkedFile: (req, res, next) ->
+		{project_id} = req.params
+		{name, provider, data, parent_folder_id} = req.body
+		user_id = AuthenticationController.getLoggedInUserId(req)
+
+		if !LinkedFilesController.Agents.hasOwnProperty(provider)
+			return res.send(400)
+		Agent = LinkedFilesController.Agents[provider]
+
+		linkedFileData = Agent.sanitizeData(data)
+		linkedFileData.provider = provider
+		Agent.writeIncomingFileToDisk project_id, linkedFileData, user_id, (error, fsPath) ->
+			return next(error) if error?
+			EditorController.upsertFile project_id, parent_folder_id, name, fsPath, linkedFileData, "upload", user_id, (error) ->
+				return next(error) if error?
+				res.send(204) # created
+}
\ No newline at end of file
diff --git a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesRouter.coffee b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesRouter.coffee
new file mode 100644
index 0000000000..e7ffd5e41f
--- /dev/null
+++ b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesRouter.coffee
@@ -0,0 +1,11 @@
+AuthorizationMiddlewear = require('../Authorization/AuthorizationMiddlewear')
+AuthenticationController = require('../Authentication/AuthenticationController')
+LinkedFilesController = require "./LinkedFilesController"
+
+module.exports =
+	apply: (webRouter) ->
+		webRouter.post '/project/:project_id/linked_file',
+			AuthenticationController.requireLogin(),
+			AuthorizationMiddlewear.ensureUserCanWriteProjectContent,
+			LinkedFilesController.createLinkedFile
+
diff --git a/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee
new file mode 100644
index 0000000000..147f2b8a25
--- /dev/null
+++ b/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee
@@ -0,0 +1,15 @@
+request = require 'request'
+FileWriter = require('../../infrastructure/FileWriter')
+
+module.exports = UrlAgent = {
+	sanitizeData: (data) ->
+		return {
+			url: data.url
+		}
+
+	writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) ->
+		# TODO: proxy through external API
+		url = data.url
+		readStream = request.get(url)
+		FileWriter.writeStreamToDisk project_id, readStream, callback
+}
\ No newline at end of file
diff --git a/services/web/app/coffee/Features/Project/ProjectCreationHandler.coffee b/services/web/app/coffee/Features/Project/ProjectCreationHandler.coffee
index fa6307ae1b..8af70e5fc5 100644
--- a/services/web/app/coffee/Features/Project/ProjectCreationHandler.coffee
+++ b/services/web/app/coffee/Features/Project/ProjectCreationHandler.coffee
@@ -79,7 +79,7 @@ module.exports = ProjectCreationHandler =
 							callback(error)
 				(callback) ->
 					universePath = Path.resolve(__dirname + "/../../../templates/project_files/universe.jpg")
-					ProjectEntityUpdateHandler.addFile project._id, project.rootFolder[0]._id, "universe.jpg", universePath, owner_id, callback
+					ProjectEntityUpdateHandler.addFile project._id, project.rootFolder[0]._id, "universe.jpg", universePath, null, owner_id, callback
 			], (error) ->
 				callback(error, project)
 
diff --git a/services/web/app/coffee/Features/Project/ProjectEntityMongoUpdateHandler.coffee b/services/web/app/coffee/Features/Project/ProjectEntityMongoUpdateHandler.coffee
index 47a8d0c553..63138d1ca0 100644
--- a/services/web/app/coffee/Features/Project/ProjectEntityMongoUpdateHandler.coffee
+++ b/services/web/app/coffee/Features/Project/ProjectEntityMongoUpdateHandler.coffee
@@ -48,7 +48,7 @@ module.exports = ProjectEntityMongoUpdateHandler = self =
 			self._confirmFolder project, folder_id, (folder_id)->
 				self._putElement project, folder_id, fileRef, "file", callback
 
-	replaceFile: wrapWithLock (project_id, file_id, callback) ->
+	replaceFile: wrapWithLock (project_id, file_id, linkedFileData, callback) ->
 		ProjectGetter.getProjectWithoutLock project_id, {rootFolder: true, name:true}, (err, project) ->
 			return callback(err) if err?
 			ProjectLocator.findElement {project:project, element_id: file_id, type: 'file'}, (err, fileRef, path)=>
@@ -63,6 +63,7 @@ module.exports = ProjectEntityMongoUpdateHandler = self =
 				inc['version'] = 1
 				set = {}
 				set["#{path.mongo}.created"] = new Date()
+				set["#{path.mongo}.linkedFileData"] = linkedFileData
 				update =
 					"$inc": inc
 					"$set": set
diff --git a/services/web/app/coffee/Features/Project/ProjectEntityUpdateHandler.coffee b/services/web/app/coffee/Features/Project/ProjectEntityUpdateHandler.coffee
index 1f5cf4da33..aadc0ac0f8 100644
--- a/services/web/app/coffee/Features/Project/ProjectEntityUpdateHandler.coffee
+++ b/services/web/app/coffee/Features/Project/ProjectEntityUpdateHandler.coffee
@@ -129,8 +129,8 @@ module.exports = ProjectEntityUpdateHandler = self =
 				return callback(error) if error?
 				callback null, doc, folder_id
 
-	addFile: wrapWithLock (project_id, folder_id, fileName, fsPath, userId, callback = (error, fileRef, folder_id) ->)->
-		self.addFileWithoutUpdatingHistory.withoutLock project_id, folder_id, fileName, fsPath, userId, (error, fileRef, folder_id, path, fileStoreUrl) ->
+	addFile: wrapWithLock (project_id, folder_id, fileName, fsPath, linkedFileData, userId, callback = (error, fileRef, folder_id) ->)->
+		self.addFileWithoutUpdatingHistory.withoutLock project_id, folder_id, fileName, fsPath, linkedFileData, userId, (error, fileRef, folder_id, path, fileStoreUrl) ->
 			return callback(error) if error?
 			newFiles = [
 				file: fileRef
@@ -141,10 +141,10 @@ module.exports = ProjectEntityUpdateHandler = self =
 				return callback(error) if error?
 				callback null, fileRef, folder_id
 
-	replaceFile: wrapWithLock (project_id, file_id, fsPath, userId, callback)->
+	replaceFile: wrapWithLock (project_id, file_id, fsPath, linkedFileData, userId, callback)->
 		FileStoreHandler.uploadFileFromDisk project_id, file_id, fsPath, (err, fileStoreUrl)->
 			return callback(err) if err?
-			ProjectEntityMongoUpdateHandler.replaceFile project_id, file_id, (err, fileRef, project, path) ->
+			ProjectEntityMongoUpdateHandler.replaceFile project_id, file_id, linkedFileData, (err, fileRef, project, path) ->
 				return callback(err) if err?
 				newFiles = [
 					file: fileRef
@@ -180,7 +180,7 @@ module.exports = ProjectEntityUpdateHandler = self =
 					return callback(err) if err?
 					callback(null, doc, folder_id, result?.path?.fileSystem)
 
-	addFileWithoutUpdatingHistory: wrapWithLock (project_id, folder_id, fileName, fsPath, userId, callback = (error, fileRef, folder_id, path, fileStoreUrl) ->)->
+	addFileWithoutUpdatingHistory: wrapWithLock (project_id, folder_id, fileName, fsPath, linkedFileData, userId, callback = (error, fileRef, folder_id, path, fileStoreUrl) ->)->
 		# This method should never be called directly, except when importing a project
 		# from Overleaf. It skips sending updates to the project history, which will break
 		# the history unless you are making sure it is updated in some other way.
@@ -188,7 +188,10 @@ module.exports = ProjectEntityUpdateHandler = self =
 		if not SafePath.isCleanFilename fileName
 			return callback new Errors.InvalidNameError("invalid element name")
 
-		fileRef = new File name : fileName
+		fileRef = new File(
+			name: fileName
+			linkedFileData: linkedFileData
+		)
 		FileStoreHandler.uploadFileFromDisk project_id, fileRef._id, fsPath, (err, fileStoreUrl)->
 			if err?
 				logger.err err:err, project_id: project_id, folder_id: folder_id, file_name: fileName, fileRef:fileRef, "error uploading image to s3"
@@ -221,7 +224,7 @@ module.exports = ProjectEntityUpdateHandler = self =
 					return callback(err) if err?
 					callback null, doc, !existingDoc?
 
-	upsertFile: wrapWithLock (project_id, folder_id, fileName, fsPath, userId, callback = (err, file, isNewFile)->)->
+	upsertFile: wrapWithLock (project_id, folder_id, fileName, fsPath, linkedFileData, userId, callback = (err, file, isNewFile)->)->
 		ProjectLocator.findElement project_id: project_id, element_id: folder_id, type: "folder", (error, folder) ->
 			return callback(error) if error?
 			return callback(new Error("Couldn't find folder")) if !folder?
@@ -231,11 +234,11 @@ module.exports = ProjectEntityUpdateHandler = self =
 					existingFile = fileRef
 					break
 			if existingFile?
-				self.replaceFile.withoutLock project_id, existingFile._id, fsPath, userId, (err) ->
+				self.replaceFile.withoutLock project_id, existingFile._id, fsPath, linkedFileData, userId, (err) ->
 					return callback(err) if err?
 					callback null, existingFile, !existingFile?
 			else
-				self.addFile.withoutLock project_id, folder_id, fileName, fsPath, userId, (err, file) ->
+				self.addFile.withoutLock project_id, folder_id, fileName, fsPath, linkedFileData, userId, (err, file) ->
 					return callback(err) if err?
 					callback null, file, !existingFile?
 
@@ -248,12 +251,12 @@ module.exports = ProjectEntityUpdateHandler = self =
 				return callback(err) if err?
 				callback null, doc, isNewDoc, newFolders, folder
 
-	upsertFileWithPath: wrapWithLock (project_id, elementPath, fsPath, userId, callback) ->
+	upsertFileWithPath: wrapWithLock (project_id, elementPath, fsPath, linkedFileData, userId, callback) ->
 		fileName = path.basename(elementPath)
 		folderPath = path.dirname(elementPath)
 		self.mkdirp.withoutLock project_id, folderPath, (err, newFolders, folder) ->
 			return callback(err) if err?
-			self.upsertFile.withoutLock project_id, folder._id, fileName, fsPath, userId, (err, file, isNewFile) ->
+			self.upsertFile.withoutLock project_id, folder._id, fileName, fsPath, linkedFileData, userId, (err, file, isNewFile) ->
 				return callback(err) if err?
 				callback null, file, isNewFile, newFolders, folder
 
diff --git a/services/web/app/coffee/Features/ThirdPartyDataStore/UpdateMerger.coffee b/services/web/app/coffee/Features/ThirdPartyDataStore/UpdateMerger.coffee
index d5c22b45ef..f101d4431e 100644
--- a/services/web/app/coffee/Features/ThirdPartyDataStore/UpdateMerger.coffee
+++ b/services/web/app/coffee/Features/ThirdPartyDataStore/UpdateMerger.coffee
@@ -1,15 +1,14 @@
 _ = require('underscore')
 fs = require('fs')
 logger = require('logger-sharelatex')
-uuid = require('uuid')
 EditorController = require('../Editor/EditorController')
 FileTypeManager = require('../Uploads/FileTypeManager')
-Settings = require('settings-sharelatex')
+FileWriter = require('../../infrastructure/FileWriter')
 
 module.exports = UpdateMerger =
 	mergeUpdate: (user_id, project_id, path, updateRequest, source, callback = (error) ->)->
 		logger.log project_id:project_id, path:path, "merging update from tpds"
-		UpdateMerger.p.writeStreamToDisk project_id, updateRequest, (err, fsPath)->
+		FileWriter.writeStreamToDisk project_id, updateRequest, (err, fsPath)->
 			return callback(err) if err?
 			UpdateMerger._mergeUpdate user_id, project_id, path, fsPath, source, (mergeErr) ->
 				fs.unlink fsPath, (deleteErr) ->
@@ -44,29 +43,10 @@ module.exports = UpdateMerger =
 
 		processFile: (project_id, fsPath, path, source, user_id, callback)->
 			logger.log project_id:project_id, "processing file update from tpds"
-			EditorController.upsertFileWithPath project_id, path, fsPath, source, user_id, (err) ->
+			EditorController.upsertFileWithPath project_id, path, fsPath, null, source, user_id, (err) ->
 				logger.log project_id:project_id, "completed processing file update from tpds"
 				callback(err)
 
-		writeStreamToDisk: (project_id, stream, callback = (err, fsPath)->)->
-			dumpPath = "#{Settings.path.dumpFolder}/#{project_id}_#{uuid.v4()}"
-
-			writeStream = fs.createWriteStream(dumpPath)
-			stream.pipe(writeStream)
-
-			stream.on 'error', (err)->
-				logger.err {err, project_id, dumpPath},
-					"something went wrong with incoming tpds update stream"
-			writeStream.on 'error', (err)->
-				logger.err {err, project_id, dumpPath},
-					"something went wrong with writing tpds update to disk"
-
-			stream.on 'end', ->
-				logger.log {project_id, dumpPath}, "incoming tpds update stream ended"
-			writeStream.on "finish", ->
-				logger.log {project_id, dumpPath}, "tpds update write stream finished"
-				callback null, dumpPath
-
 		readFileIntoTextArray: (path, callback)->
 			fs.readFile path, "utf8", (error, content = "") ->
 				if error?
diff --git a/services/web/app/coffee/Features/Uploads/FileSystemImportManager.coffee b/services/web/app/coffee/Features/Uploads/FileSystemImportManager.coffee
index 778610f6ef..6ab23b2d5a 100644
--- a/services/web/app/coffee/Features/Uploads/FileSystemImportManager.coffee
+++ b/services/web/app/coffee/Features/Uploads/FileSystemImportManager.coffee
@@ -27,9 +27,9 @@ module.exports = FileSystemImportManager =
 				return callback("path is symlink")
 
 			if replace
-				EditorController.upsertFile project_id, folder_id, name, path, "upload", user_id, callback
+				EditorController.upsertFile project_id, folder_id, name, path, null, "upload", user_id, callback
 			else
-				EditorController.addFile project_id, folder_id, name, path, "upload", user_id, callback
+				EditorController.addFile project_id, folder_id, name, path, null, "upload", user_id, callback
 
 	addFolder: (user_id, project_id, folder_id, name, path, replace, callback = (error)-> ) ->
 		FileSystemImportManager._isSafeOnFileSystem path, (err, isSafe)->
diff --git a/services/web/app/coffee/infrastructure/FileWriter.coffee b/services/web/app/coffee/infrastructure/FileWriter.coffee
new file mode 100644
index 0000000000..b19dc83336
--- /dev/null
+++ b/services/web/app/coffee/infrastructure/FileWriter.coffee
@@ -0,0 +1,23 @@
+fs = require 'fs'
+logger = require 'logger-sharelatex'
+uuid = require 'uuid'
+_ = require 'underscore'
+Settings = require 'settings-sharelatex'
+
+module.exports = 
+	writeStreamToDisk: (identifier, stream, callback = (error, fsPath) ->) ->
+		callback = _.once(callback)
+		fsPath = "#{Settings.path.dumpFolder}/#{identifier}_#{uuid.v4()}"
+
+		writeStream = fs.createWriteStream(fsPath)
+		stream.pipe(writeStream)
+
+		stream.on 'error', (err)->
+			logger.err {err, identifier, fsPath},	"[writeStreamToDisk] something went wrong with incoming stream"
+			callback(err)
+		writeStream.on 'error', (err)->
+			logger.err {err, identifier, fsPath},	"[writeStreamToDisk] something went wrong with writing to disk"
+			callback(err)
+		writeStream.on "finish", ->
+			logger.log {identifier, fsPath}, "[writeStreamToDisk] write stream finished"
+			callback null, fsPath
\ No newline at end of file
diff --git a/services/web/app/coffee/models/File.coffee b/services/web/app/coffee/models/File.coffee
index 3ca32b8f8f..f785fd2bfe 100644
--- a/services/web/app/coffee/models/File.coffee
+++ b/services/web/app/coffee/models/File.coffee
@@ -8,6 +8,7 @@ FileSchema = new Schema
 	name            :     type:String, default:''
 	created     	:     type:Date, default: () -> new Date()
 	rev 			:	  {type:Number, default:0}
+	linkedFileData: { type: Schema.Types.Mixed }
 
 mongoose.model 'File', FileSchema
 exports.File = mongoose.model 'File'
diff --git a/services/web/app/coffee/router.coffee b/services/web/app/coffee/router.coffee
index 072f5a1f6a..1241e16684 100644
--- a/services/web/app/coffee/router.coffee
+++ b/services/web/app/coffee/router.coffee
@@ -46,6 +46,7 @@ AnnouncementsController = require("./Features/Announcements/AnnouncementsControl
 MetaController = require('./Features/Metadata/MetaController')
 TokenAccessController = require('./Features/TokenAccess/TokenAccessController')
 Features = require('./infrastructure/Features')
+LinkedFilesRouter = require './Features/LinkedFiles/LinkedFilesRouter'
 
 logger = require("logger-sharelatex")
 _ = require("underscore")
@@ -77,6 +78,7 @@ module.exports = class Router
 		RealTimeProxyRouter.apply(webRouter, privateApiRouter)
 		ContactRouter.apply(webRouter, privateApiRouter)
 		AnalyticsRouter.apply(webRouter, privateApiRouter, publicApiRouter)
+		LinkedFilesRouter.apply(webRouter, privateApiRouter, publicApiRouter)
 
 		Modules.applyRouter(webRouter, privateApiRouter, publicApiRouter)
 
diff --git a/services/web/test/acceptance/coffee/LinkedFilesTests.coffee b/services/web/test/acceptance/coffee/LinkedFilesTests.coffee
new file mode 100644
index 0000000000..f492f0ba52
--- /dev/null
+++ b/services/web/test/acceptance/coffee/LinkedFilesTests.coffee
@@ -0,0 +1,94 @@
+async = require "async"
+expect = require("chai").expect
+
+MockFileStoreApi = require './helpers/MockFileStoreApi'
+MockURLSource	= require './helpers/MockURLSource'
+request = require "./helpers/request"
+User = require "./helpers/User"
+
+MockURLSource.app.get "/foo", (req, res, next) =>
+	res.send('foo foo foo')
+MockURLSource.app.get "/bar", (req, res, next) =>
+	res.send('bar bar bar')
+
+describe "LinkedFiles", ->
+	before (done) ->
+		MockURLSource.run (error) =>
+			return done(error) if error?
+			@owner = new User() 
+			@owner.login done
+
+	describe "creating a URL based linked file", ->
+		before (done) ->
+			@owner.createProject "url-linked-files-project", {template: "blank"}, (error, project_id) =>
+				throw error if error?
+				@project_id = project_id
+				@owner.getProject project_id, (error, project) =>
+					throw error if error?
+					@project = project
+					@root_folder_id = project.rootFolder[0]._id.toString()
+					done()
+
+		it "should download the URL and create a file with the contents and linkedFileData", (done) ->
+			@owner.request.post {
+				url: "/project/#{@project_id}/linked_file",
+				json:
+					provider: 'url'
+					data: {
+						url: "http://localhost:6543/foo"
+					}
+					parent_folder_id: @root_folder_id
+					name: 'url-test-file-1'
+			}, (error, response, body) =>
+				throw error if error?
+				expect(response.statusCode).to.equal 204
+				@owner.getProject @project_id, (error, project) =>
+					throw error if error?
+					file = project.rootFolder[0].fileRefs[0]
+					expect(file.linkedFileData).to.deep.equal({
+						provider: 'url'
+						url: "http://localhost:6543/foo"
+					})
+					@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
+						throw error if error?
+						expect(response.statusCode).to.equal 200
+						expect(body).to.equal "foo foo foo"
+						done()
+
+		it "should replace and update a URL based linked file", (done) ->
+			@owner.request.post {
+				url: "/project/#{@project_id}/linked_file",
+				json:
+					provider: 'url'
+					data: {
+						url: "http://localhost:6543/foo"
+					}
+					parent_folder_id: @root_folder_id
+					name: 'url-test-file-2'
+			}, (error, response, body) =>
+				throw error if error?
+				expect(response.statusCode).to.equal 204
+				@owner.request.post {
+					url: "/project/#{@project_id}/linked_file",
+					json:
+						provider: 'url'
+						data: {
+							url: "http://localhost:6543/bar"
+						}
+						parent_folder_id: @root_folder_id
+						name: 'url-test-file-2'
+				}, (error, response, body) =>
+					throw error if error?
+					expect(response.statusCode).to.equal 204
+					@owner.getProject @project_id, (error, project) =>
+						throw error if error?
+						file = project.rootFolder[0].fileRefs[1]
+						expect(file.linkedFileData).to.deep.equal({
+							provider: 'url'
+							url: "http://localhost:6543/bar"
+						})
+						@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
+							throw error if error?
+							expect(response.statusCode).to.equal 200
+							expect(body).to.equal "bar bar bar"
+							done()
diff --git a/services/web/test/acceptance/coffee/helpers/MockFileStoreApi.coffee b/services/web/test/acceptance/coffee/helpers/MockFileStoreApi.coffee
index f5fa79b641..c096e58b40 100644
--- a/services/web/test/acceptance/coffee/helpers/MockFileStoreApi.coffee
+++ b/services/web/test/acceptance/coffee/helpers/MockFileStoreApi.coffee
@@ -6,14 +6,22 @@ module.exports = MockFileStoreApi =
 
 	run: () ->
 		app.post "/project/:project_id/file/:file_id", (req, res, next) =>
-			req.on 'data', ->
+			chunks = []
+			req.on 'data', (chunk) ->
+				chunks.push(chunk)
 
 			req.on 'end', =>
+				content = Buffer.concat(chunks).toString()
 				{project_id, file_id} = req.params
 				@files[project_id] ?= {}
-				@files[project_id][file_id] = { content : "test-file-content" }
+				@files[project_id][file_id] = { content }
 				res.sendStatus 200
 
+		app.get "/project/:project_id/file/:file_id", (req, res, next) =>
+			{project_id, file_id} = req.params
+			{ content } = @files[project_id][file_id]
+			res.send content
+
 		app.listen 3009, (error) ->
 			throw error if error?
 		.on "error", (error) ->
diff --git a/services/web/test/acceptance/coffee/helpers/MockURLSource.coffee b/services/web/test/acceptance/coffee/helpers/MockURLSource.coffee
new file mode 100644
index 0000000000..1071a0e20f
--- /dev/null
+++ b/services/web/test/acceptance/coffee/helpers/MockURLSource.coffee
@@ -0,0 +1,7 @@
+express = require("express")
+app = express()
+
+module.exports = MockUrlSource =
+	app: app
+	run: (callback) ->
+		app.listen 6543, callback

From b1dda931f43c1fb59cb18a0ee465862dd12c4b10 Mon Sep 17 00:00:00 2001
From: James Allen <james@sharelatex.com>
Date: Fri, 16 Feb 2018 16:35:51 +0000
Subject: [PATCH 2/9] Add in modal for creating linked URL file

---
 .../Features/LinkedFiles/UrlAgent.coffee      |  4 +-
 .../app/views/project/editor/file-tree.pug    | 46 ++++++++++++++++
 services/web/nodemon.frontend.json            |  1 +
 services/web/nodemon.json                     |  1 +
 .../ide/file-tree/FileTreeManager.coffee      | 14 +++++
 .../controllers/FileTreeController.coffee     | 55 ++++++++++++++++++-
 6 files changed, 119 insertions(+), 2 deletions(-)

diff --git a/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee
index 147f2b8a25..7d7b30ff80 100644
--- a/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee
+++ b/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee
@@ -8,7 +8,9 @@ module.exports = UrlAgent = {
 		}
 
 	writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) ->
-		# TODO: proxy through external API
+		# TODO: Check it's a valid URL
+		# TODO: Proxy through external API
+		# TODO: Error unless valid status code
 		url = data.url
 		readStream = request.get(url)
 		FileWriter.writeStreamToDisk project_id, readStream, callback
diff --git a/services/web/app/views/project/editor/file-tree.pug b/services/web/app/views/project/editor/file-tree.pug
index c4490204ac..741076c0ad 100644
--- a/services/web/app/views/project/editor/file-tree.pug
+++ b/services/web/app/views/project/editor/file-tree.pug
@@ -339,6 +339,52 @@ script(type='text/ng-template', id='newDocModalTemplate')
 			span(ng-show="state.inflight") #{translate("creating")}...
 
 
+script(type='text/ng-template', id='linkedFileModalTemplate')
+	.modal-header
+		h3 New file from URL
+	.modal-body
+		form(novalidate, name="newLinkedFileForm")
+			div.alert.alert-danger(ng-if="error")
+				div(ng-switch="error")
+					span(ng-switch-when="already exists") #{translate("file_already_exists")}
+					span(ng-switch-default) {{error}}
+			label(for="url") URL to fetch the file from
+			input.form-control(
+				type="text",
+				placeholder="www.example.com/my_file",
+				required,
+				ng-model="inputs.url",
+				focus-on="open",
+				on-enter="create()",
+				name="url"
+			)
+			.row-spaced
+				label(for="name") File name in this project
+				input.form-control(
+					type="text",
+					placeholder="my_file",
+					required,
+					ng-model="inputs.name",
+					ng-change="nameChangedByUser = true"
+					valid-file,
+					on-enter="create()",
+					name="name"
+				)
+				.text-danger.row-spaced-small(ng-show="newDocForm.name.$error.validFile")
+					| #{translate('files_cannot_include_invalid_characters')}
+	.modal-footer
+		button.btn.btn-default(
+			ng-disabled="state.inflight"
+			ng-click="cancel()"
+		) #{translate("cancel")}
+		button.btn.btn-primary(
+			ng-disabled="newLinkedFileForm.$invalid || state.inflight"
+			ng-click="create()"
+		)
+			span(ng-hide="state.inflight") #{translate("create")}
+			span(ng-show="state.inflight") #{translate("creating")}...
+
+
 script(type='text/ng-template', id='newFolderModalTemplate')
 	.modal-header
 		h3 #{translate("new_folder")}
diff --git a/services/web/nodemon.frontend.json b/services/web/nodemon.frontend.json
index fcba014929..a5897558c0 100644
--- a/services/web/nodemon.frontend.json
+++ b/services/web/nodemon.frontend.json
@@ -4,6 +4,7 @@
     "node_modules/"
   ],
   "verbose": true,
+  "legacyWatch": true,
   "exec": "make compile",
   "watch": [
     "public/coffee/",
diff --git a/services/web/nodemon.json b/services/web/nodemon.json
index 5f79257458..7f7195cbab 100644
--- a/services/web/nodemon.json
+++ b/services/web/nodemon.json
@@ -4,6 +4,7 @@
     "node_modules/"
   ],
   "verbose": true,
+  "legacyWatch": true,
   "execMap": {
     "js": "npm run start"
   },
diff --git a/services/web/public/coffee/ide/file-tree/FileTreeManager.coffee b/services/web/public/coffee/ide/file-tree/FileTreeManager.coffee
index 9891bd8e74..4914028120 100644
--- a/services/web/public/coffee/ide/file-tree/FileTreeManager.coffee
+++ b/services/web/public/coffee/ide/file-tree/FileTreeManager.coffee
@@ -355,6 +355,20 @@ define [
 				_csrf: window.csrfToken
 			}
 
+		createLinkedFile: (name, parent_folder  = @getCurrentFolder(), provider, data) ->
+			# check if a doc/file/folder already exists with this name
+			if @existsInThisFolder parent_folder, name
+				return @nameExistsError()
+			# We'll wait for the socket.io notification to actually
+			# add the file for us.
+			return @ide.$http.post "/project/#{@ide.project_id}/linked_file", {
+				name: name,
+				parent_folder_id: parent_folder?.id
+				provider,
+				data,
+				_csrf: window.csrfToken
+			}
+
 		renameEntity: (entity, name, callback = (error) ->) ->
 			return if entity.name == name
 			return if name.length >= 150
diff --git a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee
index ba9eb8a81e..08f26862cb 100644
--- a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee
+++ b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee
@@ -30,6 +30,16 @@ define [
 				}
 			)
 
+		$scope.openLinkedFileModal = window.openLinkedFileModal = () ->
+			$modal.open(
+				templateUrl: "linkedFileModalTemplate"
+				controller:  "LinkedFileModalController"
+				scope: $scope
+				resolve: {
+					parent_folder: () -> ide.fileTreeManager.getCurrentFolder()
+				}
+			)
+
 		$scope.orderByFoldersFirst = (entity) ->
 			return '0' if entity?.type == "folder"
 			return '1'
@@ -186,4 +196,47 @@ define [
 
 			$scope.cancel = () ->
 				$modalInstance.dismiss('cancel')
-	]
\ No newline at end of file
+	]
+
+	App.controller "LinkedFileModalController", [
+		"$scope", "ide", "$modalInstance", "$timeout", "parent_folder",
+		($scope,   ide,   $modalInstance,   $timeout,   parent_folder) ->
+			$scope.inputs =
+				name: ""
+				url: ""
+			$scope.nameChangedByUser = false
+			$scope.state =
+				inflight: false
+
+			$modalInstance.opened.then () ->
+				$timeout () ->
+					$scope.$broadcast "open"
+				, 200
+
+			$scope.$watch "inputs.url", (url) ->
+				if url? and url != "" and !$scope.nameChangedByUser
+					url = url.replace("://", "") # Ignore http:// etc
+					parts = url.split("/").reverse()
+					if parts.length > 1 # Wait for at one /
+						$scope.inputs.name = parts[0]
+
+			$scope.create = () ->
+				{name, url} = $scope.inputs
+				if !name? or name.length == 0
+					return
+				if !url? or url.length == 0
+					return
+				$scope.state.inflight = true
+				ide.fileTreeManager
+					.createLinkedFile(name, parent_folder, 'url', {url})
+					.then () ->
+						$scope.state.inflight = false
+						$modalInstance.close()
+					.catch (response)->
+						{ data } = response
+						$scope.error = data
+						$scope.state.inflight = false
+
+			$scope.cancel = () ->
+				$modalInstance.dismiss('cancel')
+	]

From 01d84bd98365e2664a3f02caa732a05c5a7679fb Mon Sep 17 00:00:00 2001
From: James Allen <james@sharelatex.com>
Date: Fri, 16 Feb 2018 17:13:26 +0000
Subject: [PATCH 3/9] Handle error cases when downloading URL

---
 .../LinkedFiles/LinkedFilesController.coffee  |  3 +-
 .../Features/LinkedFiles/UrlAgent.coffee      | 53 +++++++++++-
 services/web/package.json                     |  1 +
 .../acceptance/coffee/LinkedFilesTests.coffee | 85 +++++++++++++++++++
 4 files changed, 137 insertions(+), 5 deletions(-)

diff --git a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee
index dd0b088674..f2566beb5c 100644
--- a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee
+++ b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee
@@ -18,7 +18,8 @@ module.exports = LinkedFilesController = {
 		linkedFileData = Agent.sanitizeData(data)
 		linkedFileData.provider = provider
 		Agent.writeIncomingFileToDisk project_id, linkedFileData, user_id, (error, fsPath) ->
-			return next(error) if error?
+			if error?
+				return Agent.handleError(error, req, res, next)
 			EditorController.upsertFile project_id, parent_folder_id, name, fsPath, linkedFileData, "upload", user_id, (error) ->
 				return next(error) if error?
 				res.send(204) # created
diff --git a/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee
index 7d7b30ff80..98e176bc06 100644
--- a/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee
+++ b/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee
@@ -1,17 +1,62 @@
 request = require 'request'
 FileWriter = require('../../infrastructure/FileWriter')
+_ = require "underscore"
+urlValidator = require 'valid-url'
+
+UrlFetchFailedError = (message) ->
+	error = new Error(message)
+	error.name = 'UrlFetchFailedError'
+	error.__proto__ = UrlFetchFailedError.prototype
+	return error
+UrlFetchFailedError.prototype.__proto__ = Error.prototype
+
+InvalidUrlError = (message) ->
+	error = new Error(message)
+	error.name = 'InvalidUrlError'
+	error.__proto__ = InvalidUrlError.prototype
+	return error
+InvalidUrlError.prototype.__proto__ = Error.prototype
+
 
 module.exports = UrlAgent = {
+	UrlFetchFailedError: UrlFetchFailedError
+	InvalidUrlError: InvalidUrlError
+
 	sanitizeData: (data) ->
 		return {
 			url: data.url
 		}
 
+	_prependHttpIfNeeded: (url) ->
+		if !url.match('://')
+			url = 'http://' + url
+		return url
+
 	writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) ->
-		# TODO: Check it's a valid URL
 		# TODO: Proxy through external API
-		# TODO: Error unless valid status code
-		url = data.url
+		callback = _.once(callback)
+		url = @._prependHttpIfNeeded(data.url)
+		if !urlValidator.isWebUri(url)
+			return callback(new InvalidUrlError())
 		readStream = request.get(url)
-		FileWriter.writeStreamToDisk project_id, readStream, callback
+		readStream.on "error", callback
+		readStream.on "response", (response) ->
+			if 200 <= response.statusCode < 300
+				FileWriter.writeStreamToDisk project_id, readStream, callback
+			else
+				error = new UrlFetchFailedError()
+				error.statusCode = response.statusCode
+				callback(error)
+
+	handleError: (error, req, res, next) ->
+		if error instanceof UrlFetchFailedError
+			res.status(422).send(
+				"Your URL could not be reached (#{error.statusCode} status code). Please check it and try again."
+			)
+		else if error instanceof InvalidUrlError
+			res.status(422).send(
+				"Your URL is not valid. Please check it and try again."
+			)
+		else
+			next(error)
 }
\ No newline at end of file
diff --git a/services/web/package.json b/services/web/package.json
index ffb75186b7..485450807d 100644
--- a/services/web/package.json
+++ b/services/web/package.json
@@ -85,6 +85,7 @@
     "underscore": "1.6.0",
     "uuid": "^3.0.1",
     "v8-profiler": "^5.2.3",
+    "valid-url": "^1.0.9",
     "xml2js": "0.2.0",
     "yauzl": "^2.8.0"
   },
diff --git a/services/web/test/acceptance/coffee/LinkedFilesTests.coffee b/services/web/test/acceptance/coffee/LinkedFilesTests.coffee
index f492f0ba52..2c16a0f0e0 100644
--- a/services/web/test/acceptance/coffee/LinkedFilesTests.coffee
+++ b/services/web/test/acceptance/coffee/LinkedFilesTests.coffee
@@ -1,5 +1,6 @@
 async = require "async"
 expect = require("chai").expect
+_ = require 'underscore'
 
 MockFileStoreApi = require './helpers/MockFileStoreApi'
 MockURLSource	= require './helpers/MockURLSource'
@@ -92,3 +93,87 @@ describe "LinkedFiles", ->
 							expect(response.statusCode).to.equal 200
 							expect(body).to.equal "bar bar bar"
 							done()
+
+		it "should return an error if the URL does not succeed", (done) ->
+			@owner.request.post {
+				url: "/project/#{@project_id}/linked_file",
+				json:
+					provider: 'url'
+					data: {
+						url: "http://localhost:6543/does-not-exist"
+					}
+					parent_folder_id: @root_folder_id
+					name: 'url-test-file-3'
+			}, (error, response, body) =>
+				throw error if error?
+				expect(response.statusCode).to.equal 422 # unprocessable
+				expect(body).to.equal(
+					"Your URL could not be reached (404 status code). Please check it and try again."
+				)
+				done()
+
+		it "should return an error if the URL is invalid", (done) ->
+			@owner.request.post {
+				url: "/project/#{@project_id}/linked_file",
+				json:
+					provider: 'url'
+					data: {
+						url: "!^$%"
+					}
+					parent_folder_id: @root_folder_id
+					name: 'url-test-file-4'
+			}, (error, response, body) =>
+				throw error if error?
+				expect(response.statusCode).to.equal 422 # unprocessable
+				expect(body).to.equal(
+					"Your URL is not valid. Please check it and try again."
+				)
+				done()
+
+		it "should return an error if the URL uses a non-http protocol", (done) ->
+			@owner.request.post {
+				url: "/project/#{@project_id}/linked_file",
+				json:
+					provider: 'url'
+					data: {
+						url: "ftp://localhost"
+					}
+					parent_folder_id: @root_folder_id
+					name: 'url-test-file-5'
+			}, (error, response, body) =>
+				throw error if error?
+				expect(response.statusCode).to.equal 422 # unprocessable
+				expect(body).to.equal(
+					"Your URL is not valid. Please check it and try again."
+				)
+				done()
+
+		it "should accept a URL withuot a leading http://, and add it", (done) ->
+			@owner.request.post {
+				url: "/project/#{@project_id}/linked_file",
+				json:
+					provider: 'url'
+					data: {
+						url: "http://localhost:6543/foo"
+					}
+					parent_folder_id: @root_folder_id
+					name: 'url-test-file-6'
+			}, (error, response, body) =>
+				throw error if error?
+				expect(response.statusCode).to.equal 204
+				@owner.getProject @project_id, (error, project) =>
+					throw error if error?
+					file = _.find project.rootFolder[0].fileRefs, (file) ->
+						file.name == 'url-test-file-6'
+					expect(file.linkedFileData).to.deep.equal({
+						provider: 'url'
+						url: "http://localhost:6543/foo"
+					})
+					@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
+						throw error if error?
+						expect(response.statusCode).to.equal 200
+						expect(body).to.equal "foo foo foo"
+						done()
+
+		# TODO: Add test for asking for host that return ENOTFOUND
+		# (This will probably end up handled by the proxy)
\ No newline at end of file

From 973322384088a96bc9a62a712ae66028a593693e Mon Sep 17 00:00:00 2001
From: James Allen <james@sharelatex.com>
Date: Tue, 20 Feb 2018 10:37:55 +0000
Subject: [PATCH 4/9] Show linked file status in front end

---
 .../Project/ProjectEditorHandler.coffee       |  2 +
 .../app/views/project/editor/binary-file.pug  | 45 +++++++---
 .../app/views/project/editor/file-tree.pug    |  3 +
 .../controllers/BinaryFileController.coffee   | 85 ++++++++++++-------
 .../ide/file-tree/FileTreeManager.coffee      |  2 +
 .../stylesheets/app/editor/binary-file.less   |  8 +-
 .../stylesheets/app/editor/file-tree.less     | 18 ++++
 7 files changed, 117 insertions(+), 46 deletions(-)

diff --git a/services/web/app/coffee/Features/Project/ProjectEditorHandler.coffee b/services/web/app/coffee/Features/Project/ProjectEditorHandler.coffee
index e7ebf9a24d..258d99ca12 100644
--- a/services/web/app/coffee/Features/Project/ProjectEditorHandler.coffee
+++ b/services/web/app/coffee/Features/Project/ProjectEditorHandler.coffee
@@ -76,6 +76,8 @@ module.exports = ProjectEditorHandler =
 	buildFileModelView: (file) ->
 		_id  : file._id
 		name : file.name
+		linkedFileData: file.linkedFileData
+		created: file.created
 
 	buildDocModelView: (doc) ->
 		_id   : doc._id
diff --git a/services/web/app/views/project/editor/binary-file.pug b/services/web/app/views/project/editor/binary-file.pug
index d0effce668..f13372cdeb 100644
--- a/services/web/app/views/project/editor/binary-file.pug
+++ b/services/web/app/views/project/editor/binary-file.pug
@@ -6,7 +6,7 @@ div.binary-file.full-size(
 	img(
 		ng-show="!failedLoad"
 		ng-src="/project/{{ project_id }}/file/{{ openFile.id }}"
-		ng-if="['png', 'jpg', 'jpeg', 'gif'].indexOf(extension(openFile)) > -1"
+		ng-if="isImageFile()"
 		ng-class="{'img-preview': !imgLoaded}"
 		onerror="sl_binaryFilePreviewError()"
 		onabort="sl_binaryFilePreviewError()"
@@ -16,29 +16,48 @@ div.binary-file.full-size(
 	img(
 		ng-show="!failedLoad"
 		ng-src="/project/{{ project_id }}/file/{{ openFile.id }}?format=png"
-		ng-if="['pdf', 'eps'].indexOf(extension(openFile)) > -1"
+		ng-if="isPreviewableFile()"
 		ng-class="{'img-preview': !imgLoaded}"
 		onerror="sl_binaryFilePreviewError()"
 		onabort="sl_binaryFilePreviewError()"
 		onload="sl_binaryFilePreviewLoaded()"
 	)
 	
-	div(ng-if="(['bib'].indexOf(extension(openFile)) > -1) && !bibtexPreview.error")
-
-		div.bib-loading(ng-show="bibtexPreview.loading && !bibtexPreview.error")
+	div(ng-if="isTextFile() && !textPreview.error")
+		div.text-loading(ng-show="textPreview.loading && !textPreview.error")
 			| #{translate('loading')}...
-
-		div.bib-preview(ng-show="bibtexPreview.data && !bibtexPreview.loading && !bibtexPreview.error")
+		div.text-preview(ng-show="textPreview.data && !textPreview.loading && !textPreview.error")
 			div.scroll-container
 				p
-					| {{ bibtexPreview.data }}
-				p(ng-show="bibtexPreview.shouldShowDots")
+					| {{ textPreview.data }}
+				p(ng-show="textPreview.shouldShowDots")
 					| ...
 	
 	p.no-preview(
-		ng-if="failedLoad || bibtexPreview.error || ['bib', 'png', 'jpg', 'jpeg', 'gif', 'pdf', 'eps'].indexOf(extension(openFile)) == -1"
+		ng-if="failedLoad || textPreview.error || isUnpreviewableFile()"
 	) #{translate("no_preview_available")}
 
-	a.btn.btn-info(
-		ng-href="/project/{{ project_id }}/file/{{ openFile.id }}"
-	) #{translate("download")} {{ openFile.name }}
+	div.binary-file-footer
+		div(ng-show="openFile.linkedFileData.provider == 'url'")
+			p
+				i.fa.fa-fw.fa-external-link-square.fa-rotate-180.linked-file-icon
+				| Imported from
+				|
+				a(ng-href='{{openFile.linkedFileData.url}}') {{ displayUrl(openFile.linkedFileData.url) }}
+				|
+				| at {{ openFile.created | formatDate:'h:mm a' }} {{ openFile.created | relativeDate }}
+
+		span(ng-show="openFile.linkedFileData.provider == 'url'")
+			a.btn.btn-success(
+				href
+			)
+				i.fa.fa-fw.fa-refresh
+				|
+				| Refresh
+			| &nbsp;
+		a.btn.btn-info(
+			ng-href="/project/{{ project_id }}/file/{{ openFile.id }}"
+		)
+			i.fa.fa-fw.fa-download
+			|
+			| #{translate("download")}
diff --git a/services/web/app/views/project/editor/file-tree.pug b/services/web/app/views/project/editor/file-tree.pug
index 741076c0ad..55fc660abc 100644
--- a/services/web/app/views/project/editor/file-tree.pug
+++ b/services/web/app/views/project/editor/file-tree.pug
@@ -108,6 +108,9 @@ script(type='text/ng-template', id='entityListItemTemplate')
 				i.fa.fa-fw.toggle(ng-if="entity.type != 'folder'")
 
 				i.fa.fa-fw(ng-if="entity.type != 'folder'", ng-class="'fa-' + iconTypeFromName(entity.name)")
+					i.fa.fa-external-link-square.fa-rotate-180.linked-file-highlight(
+						ng-if="entity.linkedFileData.provider"
+					)
 				span(
 					ng-hide="entity.renaming"
 				)  {{ entity.renamingToName || entity.name }}
diff --git a/services/web/public/coffee/ide/binary-files/controllers/BinaryFileController.coffee b/services/web/public/coffee/ide/binary-files/controllers/BinaryFileController.coffee
index f331c4dc80..11a73d20ce 100644
--- a/services/web/public/coffee/ide/binary-files/controllers/BinaryFileController.coffee
+++ b/services/web/public/coffee/ide/binary-files/controllers/BinaryFileController.coffee
@@ -1,16 +1,47 @@
 define [
 	"base"
-], (App) ->
-	App.controller "BinaryFileController", ["$scope", "$rootScope", "$http", "$timeout", ($scope, $rootScope, $http, $timeout) ->
+	"moment"
+], (App, moment) ->
+	App.controller "BinaryFileController", ["$scope", "$rootScope", "$http", "$timeout", "$element", ($scope, $rootScope, $http, $timeout, $element) ->
 
 		TWO_MEGABYTES = 2 * 1024 * 1024
 
-		$scope.bibtexPreview =
+		textExtensions = ['bib', 'tex', 'txt', 'cls', 'sty']
+		imageExtentions = ['png', 'jpg', 'jpeg', 'gif']
+		previewableExtensions = ['eps', 'pdf']
+
+		extension = (file) ->
+			return file.name.split(".").pop()?.toLowerCase()
+
+		$scope.isTextFile = () =>
+			textExtensions.indexOf(extension($scope.openFile)) > -1
+		$scope.isImageFile = () =>
+			imageExtentions.indexOf(extension($scope.openFile)) > -1
+		$scope.isPreviewableFile = () =>
+			previewableExtensions.indexOf(extension($scope.openFile)) > -1
+		$scope.isUnpreviewableFile = () ->
+			!$scope.isTextFile() and
+			!$scope.isImageFile() and
+			!$scope.isPreviewableFile()
+
+		$scope.textPreview =
 			loading: false
 			shouldShowDots: false
 			error: false
 			data: null
 
+		MAX_URL_LENGTH = 60
+		FRONT_OF_URL_LENGTH = 35
+		FILLER = '...'
+		TAIL_OF_URL_LENGTH = MAX_URL_LENGTH - FRONT_OF_URL_LENGTH - FILLER.length
+		$scope.displayUrl = (url) ->
+			if url.length > MAX_URL_LENGTH
+				front = url.slice(0, FRONT_OF_URL_LENGTH)
+				tail = url.slice(url.length - TAIL_OF_URL_LENGTH)
+				return front + FILLER + tail
+			else
+				return url
+
 		# Callback fired when the `img` tag fails to load,
 		# `failedLoad` used to show the "No Preview" message
 		$scope.failedLoad = false
@@ -25,47 +56,39 @@ define [
 			$scope.imgLoaded = true
 			$scope.$apply()
 
-		$scope.extension = (file) ->
-			return file.name.split(".").pop()?.toLowerCase()
-
-		$scope.loadBibtexFilePreview = () ->
+		loadTextFileFilePreview = () ->
 			url = "/project/#{project_id}/file/#{$scope.openFile.id}?range=0-#{TWO_MEGABYTES}"
-			$scope.bibtexPreview.loading = true
-			$scope.bibtexPreview.shouldShowDots = false
+			$scope.textPreview.loading = true
+			$scope.textPreview.shouldShowDots = false
 			$scope.$apply()
 			$http.get(url)
 				.then (response) ->
 					{ data } = response
-					$scope.bibtexPreview.loading = false
-					$scope.bibtexPreview.error = false
+					$scope.textPreview.error = false
 					# show dots when payload is closs to cutoff
 					if data.length >= (TWO_MEGABYTES - 200)
-						$scope.bibtexPreview.shouldShowDots = true
+						$scope.textPreview.shouldShowDots = true
 					try
 						# remove last partial line
 						data = data.replace(/\n.*$/, '')
 					finally
-						$scope.bibtexPreview.data = data
-					$timeout($scope.setHeight, 0)
+						$scope.textPreview.data = data
+					$timeout(setHeight, 0)
 				.catch () ->
-					$scope.bibtexPreview.error = true
-					$scope.bibtexPreview.loading = false
+					$scope.textPreview.error = true
+					$scope.textPreview.loading = false
 
-		$scope.setHeight = () ->
-			# Behold, a ghastly hack
-			guide = document.querySelector('.file-tree-inner')
-			table_wrap = document.querySelector('.bib-preview .scroll-container')
-			if table_wrap
-				desired_height = guide.offsetHeight - 44
-				if table_wrap.offsetHeight > desired_height
-					table_wrap.style.height = desired_height + 'px'
-					table_wrap.style['max-height'] = desired_height + 'px'
+		setHeight = () ->
+			$preview = $element.find('.text-preview .scroll-container')
+			$footer = $element.find('.binary-file-footer')
+			maxHeight = $element.height() - $footer.height() - 14 # borders + margin
+			$preview.css('max-height': maxHeight)
+			# Don't show the preview until we've set the height, otherwise we jump around
+			$scope.textPreview.loading = false
 
-		$scope.loadBibtexIfRequired = () ->
-			if $scope.extension($scope.openFile) == 'bib'
-				$scope.bibtexPreview.data = null
-				$scope.loadBibtexFilePreview()
-
-		$scope.loadBibtexIfRequired()
+		do loadTextFileIfRequired = () ->
+			if $scope.isTextFile()
+				$scope.textPreview.data = null
+				loadTextFileFilePreview()
 
 	]
diff --git a/services/web/public/coffee/ide/file-tree/FileTreeManager.coffee b/services/web/public/coffee/ide/file-tree/FileTreeManager.coffee
index 4914028120..e4d84f2592 100644
--- a/services/web/public/coffee/ide/file-tree/FileTreeManager.coffee
+++ b/services/web/public/coffee/ide/file-tree/FileTreeManager.coffee
@@ -252,6 +252,8 @@ define [
 					id:   file._id
 					type: "file"
 					selected: (file._id == @selected_entity_id)
+					linkedFileData: file.linkedFileData
+					created: file.created
 				}
 
 			for childFolder in rawFolder.folders or []
diff --git a/services/web/public/stylesheets/app/editor/binary-file.less b/services/web/public/stylesheets/app/editor/binary-file.less
index 04bc62931a..4fc54c82cc 100644
--- a/services/web/public/stylesheets/app/editor/binary-file.less
+++ b/services/web/public/stylesheets/app/editor/binary-file.less
@@ -22,14 +22,15 @@
 		font-size: 24px;
 		color: @gray;
 	}
-	.bib-loading {
+	.text-loading {
 		font-size: 24px;
 		color: @gray;
 		margin-bottom: 12px;
 	}
-	.bib-preview {
+	.text-preview {
 		margin-bottom: 12px;
 		.scroll-container {
+			background-color: white;
 			font-size: 0.8em;
 			line-height: 1.1em;
 			overflow: auto;
@@ -43,5 +44,8 @@
 			font-family: monospace;
 		}
 	}
+	.linked-file-icon {
+		color: @blue
+	}
 
 }
diff --git a/services/web/public/stylesheets/app/editor/file-tree.less b/services/web/public/stylesheets/app/editor/file-tree.less
index a7aae22938..4e26e4751d 100644
--- a/services/web/public/stylesheets/app/editor/file-tree.less
+++ b/services/web/public/stylesheets/app/editor/file-tree.less
@@ -101,6 +101,18 @@
 			i.fa {
 				color: @file-tree-item-icon-color;
 				font-size: 14px;
+				&.linked-file-highlight {
+					&when (@is-overleaf = true) {
+						color: white;
+					}
+					&when (@is-overleaf = false) {
+						color: @blue;
+					}
+					position: relative;
+					top: 4px;
+					left: -8px;
+					font-size: 12px;
+				}
 			}
 
 			i.fa-folder-open, i.fa-folder {
@@ -129,6 +141,9 @@
 						.entity-menu-toggle i.fa {
 							color: #FFF;
 						}
+						> i.fa i.linked-file-highlight {
+							color: @blue;
+						}
 						color: #FFF;
 						font-weight: bold;
 						background-color: @file-tree-multiselect-bg;
@@ -191,6 +206,9 @@
 					.entity-menu-toggle i.fa {
 						color: #FFF;
 					}
+					> i.fa i.linked-file-highlight {
+						color: @blue;
+					}
 					background-color: @file-tree-item-selected-bg;
 					font-weight: bold;
 					padding-right: 32px;

From 6848e97a82ee9e4bdc1eeb87ca62b19e2516cc57 Mon Sep 17 00:00:00 2001
From: James Allen <james@sharelatex.com>
Date: Tue, 20 Feb 2018 11:49:02 +0000
Subject: [PATCH 5/9] Hook up refresh button to backend

---
 .../Features/Editor/EditorController.coffee   |  6 ++--
 .../app/views/project/editor/binary-file.pug  | 12 ++++---
 .../controllers/BinaryFileController.coffee   | 30 +++++++++++-----
 .../ide/file-tree/FileTreeManager.coffee      | 35 ++++++++++++++++---
 4 files changed, 62 insertions(+), 21 deletions(-)

diff --git a/services/web/app/coffee/Features/Editor/EditorController.coffee b/services/web/app/coffee/Features/Editor/EditorController.coffee
index ad00445adb..7294e88bea 100644
--- a/services/web/app/coffee/Features/Editor/EditorController.coffee
+++ b/services/web/app/coffee/Features/Editor/EditorController.coffee
@@ -31,7 +31,7 @@ module.exports = EditorController =
 			if err?
 				logger.err err:err, project_id:project_id, folder_id:folder_id, fileName:fileName, "error adding file without lock"
 				return callback(err)
-			EditorRealTimeController.emitToRoom(project_id, 'reciveNewFile', folder_id, fileRef, source)
+			EditorRealTimeController.emitToRoom(project_id, 'reciveNewFile', folder_id, fileRef, source, linkedFileData)
 			callback(err, fileRef)
 
 	upsertDoc: (project_id, folder_id, docName, docLines, source, user_id, callback = (err)->)->
@@ -44,7 +44,7 @@ module.exports = EditorController =
 		ProjectEntityUpdateHandler.upsertFile project_id, folder_id, fileName, fsPath, linkedFileData, user_id, (err, file, didAddFile) ->
 			return callback(err) if err?
 			if didAddFile
-				EditorRealTimeController.emitToRoom project_id, 'reciveNewFile', folder_id, file, source
+				EditorRealTimeController.emitToRoom project_id, 'reciveNewFile', folder_id, file, source, linkedFileData
 			callback null, file
 
 	upsertDocWithPath: (project_id, elementPath, docLines, source, user_id, callback) ->
@@ -62,7 +62,7 @@ module.exports = EditorController =
 			EditorController._notifyProjectUsersOfNewFolders project_id, newFolders, (err) ->
 				return callback(err) if err?
 				if didAddFile
-					EditorRealTimeController.emitToRoom project_id, 'reciveNewFile', lastFolder._id, file, source
+					EditorRealTimeController.emitToRoom project_id, 'reciveNewFile', lastFolder._id, file, source, linkedFileData
 				callback()
 
 	addFolder : (project_id, folder_id, folderName, source, callback = (error, folder)->)->
diff --git a/services/web/app/views/project/editor/binary-file.pug b/services/web/app/views/project/editor/binary-file.pug
index f13372cdeb..e4c7183b16 100644
--- a/services/web/app/views/project/editor/binary-file.pug
+++ b/services/web/app/views/project/editor/binary-file.pug
@@ -35,7 +35,7 @@ div.binary-file.full-size(
 	
 	p.no-preview(
 		ng-if="failedLoad || textPreview.error || isUnpreviewableFile()"
-	) #{translate("no_preview_available")}
+	) #{translate("no_preview_available")} {{ failedLoad }} {{ textPreview.error }} {{ isUnpreviewableFile() }}
 
 	div.binary-file-footer
 		div(ng-show="openFile.linkedFileData.provider == 'url'")
@@ -48,12 +48,14 @@ div.binary-file.full-size(
 				| at {{ openFile.created | formatDate:'h:mm a' }} {{ openFile.created | relativeDate }}
 
 		span(ng-show="openFile.linkedFileData.provider == 'url'")
-			a.btn.btn-success(
-				href
+			button.btn.btn-success(
+				href, ng-click="refreshFile(openFile)",
+				ng-disabled="refreshing"
 			)
-				i.fa.fa-fw.fa-refresh
+				i.fa.fa-fw.fa-refresh(ng-class={'fa-spin': refreshing})
 				|
-				| Refresh
+				span(ng-show="!refreshing") Refresh
+				span(ng-show="refreshing") Refreshing...
 			| &nbsp;
 		a.btn.btn-info(
 			ng-href="/project/{{ project_id }}/file/{{ openFile.id }}"
diff --git a/services/web/public/coffee/ide/binary-files/controllers/BinaryFileController.coffee b/services/web/public/coffee/ide/binary-files/controllers/BinaryFileController.coffee
index 11a73d20ce..df8dd81569 100644
--- a/services/web/public/coffee/ide/binary-files/controllers/BinaryFileController.coffee
+++ b/services/web/public/coffee/ide/binary-files/controllers/BinaryFileController.coffee
@@ -2,7 +2,7 @@ define [
 	"base"
 	"moment"
 ], (App, moment) ->
-	App.controller "BinaryFileController", ["$scope", "$rootScope", "$http", "$timeout", "$element", ($scope, $rootScope, $http, $timeout, $element) ->
+	App.controller "BinaryFileController", ["$scope", "$rootScope", "$http", "$timeout", "$element", "ide", ($scope, $rootScope, $http, $timeout, $element, ide) ->
 
 		TWO_MEGABYTES = 2 * 1024 * 1024
 
@@ -30,6 +30,8 @@ define [
 			error: false
 			data: null
 
+		$scope.refreshing = false
+
 		MAX_URL_LENGTH = 60
 		FRONT_OF_URL_LENGTH = 35
 		FILLER = '...'
@@ -42,6 +44,14 @@ define [
 			else
 				return url
 
+		$scope.refreshFile = (file) ->
+			$scope.refreshing = true
+			ide.fileTreeManager.refreshLinkedFile(file)
+				.then () ->
+					loadTextFileFilePreview()
+				.finally () ->
+					$scope.refreshing = false
+
 		# Callback fired when the `img` tag fails to load,
 		# `failedLoad` used to show the "No Preview" message
 		$scope.failedLoad = false
@@ -56,12 +66,18 @@ define [
 			$scope.imgLoaded = true
 			$scope.$apply()
 
-		loadTextFileFilePreview = () ->
+		do loadTextFileFilePreview = () ->
+			return unless $scope.isTextFile()
 			url = "/project/#{project_id}/file/#{$scope.openFile.id}?range=0-#{TWO_MEGABYTES}"
+			$scope.textPreview.data = null
 			$scope.textPreview.loading = true
 			$scope.textPreview.shouldShowDots = false
 			$scope.$apply()
-			$http.get(url)
+			$http({
+				url: url,
+				method: 'GET',
+				transformResponse: null # Don't parse JSON
+			})
 				.then (response) ->
 					{ data } = response
 					$scope.textPreview.error = false
@@ -74,7 +90,8 @@ define [
 					finally
 						$scope.textPreview.data = data
 					$timeout(setHeight, 0)
-				.catch () ->
+				.catch (error) ->
+					console.error(error)
 					$scope.textPreview.error = true
 					$scope.textPreview.loading = false
 
@@ -86,9 +103,4 @@ define [
 			# Don't show the preview until we've set the height, otherwise we jump around
 			$scope.textPreview.loading = false
 
-		do loadTextFileIfRequired = () ->
-			if $scope.isTextFile()
-				$scope.textPreview.data = null
-				loadTextFileFilePreview()
-
 	]
diff --git a/services/web/public/coffee/ide/file-tree/FileTreeManager.coffee b/services/web/public/coffee/ide/file-tree/FileTreeManager.coffee
index e4d84f2592..30de3d423e 100644
--- a/services/web/public/coffee/ide/file-tree/FileTreeManager.coffee
+++ b/services/web/public/coffee/ide/file-tree/FileTreeManager.coffee
@@ -37,13 +37,14 @@ define [
 					}
 					@recalculateDocList()
 
-			@ide.socket.on "reciveNewFile", (parent_folder_id, file) =>
+			@ide.socket.on "reciveNewFile", (parent_folder_id, file, source, linkedFileData) =>
 				parent_folder = @findEntityById(parent_folder_id) or @$scope.rootFolder
 				@$scope.$apply () =>
 					parent_folder.children.push {
 						name: file.name
 						id:   file._id
-						type: "file"
+						type: "file",
+						linkedFileData: linkedFileData
 					}
 					@recalculateDocList()
 
@@ -175,6 +176,9 @@ define [
 		_findEntityByPathInFolder: (folder, path) ->
 			if !path? or !folder?
 				return null
+			if path == ""
+				return folder
+
 			parts = path.split("/")
 			name = parts.shift()
 			rest = parts.join("/")
@@ -222,10 +226,19 @@ define [
 		getRootDocDirname: () ->
 			rootDoc = @findEntityById @$scope.project.rootDoc_id
 			return if !rootDoc?
-			path = @getEntityPath(rootDoc)
+			return @_getEntityDirname(rootDoc)
+
+		_getEntityDirname: (entity) ->
+			path = @getEntityPath(entity)
 			return if !path?
 			return path.split("/").slice(0, -1).join("/")
 
+		_findParentFolder: (entity) ->
+			dirname = @_getEntityDirname(entity)
+			console.log('dirname', dirname)
+			return if !dirname?
+			return @findEntityByPath(dirname)
+
 		loadRootFolder: () ->
 			@$scope.rootFolder = @_parseFolder(@$scope?.project?.rootFolder[0])
 
@@ -357,7 +370,7 @@ define [
 				_csrf: window.csrfToken
 			}
 
-		createLinkedFile: (name, parent_folder  = @getCurrentFolder(), provider, data) ->
+		createLinkedFile: (name, parent_folder = @getCurrentFolder(), provider, data) ->
 			# check if a doc/file/folder already exists with this name
 			if @existsInThisFolder parent_folder, name
 				return @nameExistsError()
@@ -371,6 +384,20 @@ define [
 				_csrf: window.csrfToken
 			}
 
+		refreshLinkedFile: (file) ->
+			parent_folder = @_findParentFolder(file)
+			data = file.linkedFileData
+			provider = data?.provider
+			return if !provider?
+			console.log 'refreshLinkedFile', {parent_folder, provider, data}
+			return @ide.$http.post "/project/#{@ide.project_id}/linked_file", {
+				name: file.name,
+				parent_folder_id: parent_folder?.id
+				provider,
+				data,
+				_csrf: window.csrfToken
+			}
+
 		renameEntity: (entity, name, callback = (error) ->) ->
 			return if entity.name == name
 			return if name.length >= 150

From cae09028ff52fbb3e9c4cbfa16e3dab6cd46991f Mon Sep 17 00:00:00 2001
From: James Allen <james@sharelatex.com>
Date: Tue, 20 Feb 2018 12:52:57 +0000
Subject: [PATCH 6/9] Fix tests

---
 .../Editor/EditorControllerTests.coffee       | 25 ++++++++++---------
 .../ProjectCreationHandlerTests.coffee        |  3 ++-
 .../Project/ProjectEditorHandlerTests.coffee  |  4 +--
 ...rojectEntityMongoUpdateHandlerTests.coffee |  5 ++--
 .../ProjectEntityUpdateHandlerTests.coffee    | 24 ++++++++++--------
 .../SubscriptionControllerTests.coffee        |  2 ++
 .../UpdateMergerTests.coffee                  |  6 +++--
 .../FileSystemImportManagerTests.coffee       |  6 ++---
 8 files changed, 42 insertions(+), 33 deletions(-)

diff --git a/services/web/test/unit/coffee/Editor/EditorControllerTests.coffee b/services/web/test/unit/coffee/Editor/EditorControllerTests.coffee
index 4e928732ae..9e30ef3fd1 100644
--- a/services/web/test/unit/coffee/Editor/EditorControllerTests.coffee
+++ b/services/web/test/unit/coffee/Editor/EditorControllerTests.coffee
@@ -18,6 +18,7 @@ describe "EditorController", ->
 		@file = _id: @file_id ="dasdkjk"
 		@fileName = "file.png"
 		@fsPath = "/folder/file.png"
+		@linkedFileData = {provider: 'url'}
 
 		@folder_id = "123ksajdn"
 		@folder = _id: @folder_id
@@ -66,16 +67,16 @@ describe "EditorController", ->
 	describe 'addFile', ->
 		beforeEach ->
 			@ProjectEntityUpdateHandler.addFile = sinon.stub().yields(null, @file, @folder_id)
-			@EditorController.addFile @project_id, @folder_id, @fileName, @fsPath, @source, @user_id, @callback
+			@EditorController.addFile @project_id, @folder_id, @fileName, @fsPath, @linkedFileData, @source, @user_id, @callback
 
 		it 'should add the folder using the project entity handler', ->
 			@ProjectEntityUpdateHandler.addFile
-				.calledWith(@project_id, @folder_id, @fileName, @fsPath, @user_id)
+				.calledWith(@project_id, @folder_id, @fileName, @fsPath, @linkedFileData, @user_id)
 				.should.equal true
 
 		it 'should send the update of a new folder out to the users in the project', ->
 			@EditorRealTimeController.emitToRoom
-				.calledWith(@project_id, "reciveNewFile", @folder_id, @file, @source)
+				.calledWith(@project_id, "reciveNewFile", @folder_id, @file, @source, @linkedFileData)
 				.should.equal true
 
 		it 'calls the callback', ->
@@ -107,11 +108,11 @@ describe "EditorController", ->
 	describe 'upsertFile', ->
 		beforeEach ->
 			@ProjectEntityUpdateHandler.upsertFile = sinon.stub().yields(null, @file, false)
-			@EditorController.upsertFile @project_id, @folder_id, @fileName, @fsPath, @source, @user_id, @callback
+			@EditorController.upsertFile @project_id, @folder_id, @fileName, @fsPath, @linkedFileData, @source, @user_id, @callback
 
 		it 'upserts the file using the project entity handler', ->
 			@ProjectEntityUpdateHandler.upsertFile
-				.calledWith(@project_id, @folder_id, @fileName, @fsPath, @user_id)
+				.calledWith(@project_id, @folder_id, @fileName, @fsPath, @linkedFileData, @user_id)
 				.should.equal true
 
 		it 'returns the file', ->
@@ -120,11 +121,11 @@ describe "EditorController", ->
 		describe 'file does not exist', ->
 			beforeEach ->
 				@ProjectEntityUpdateHandler.upsertFile = sinon.stub().yields(null, @file, true)
-				@EditorController.upsertFile @project_id, @folder_id, @fileName, @fsPath, @source, @user_id, @callback
+				@EditorController.upsertFile @project_id, @folder_id, @fileName, @fsPath, @linkedFileData, @source, @user_id, @callback
 
 			it 'should send the update out to users in the project', ->
 				@EditorRealTimeController.emitToRoom
-					.calledWith(@project_id, "reciveNewFile", @folder_id, @file, @source)
+					.calledWith(@project_id, "reciveNewFile", @folder_id, @file, @source, @linkedFileData)
 					.should.equal true
 
 	describe "upsertDocWithPath", ->
@@ -171,21 +172,21 @@ describe "EditorController", ->
 			@filePath = '/folder/file'
 
 			@ProjectEntityUpdateHandler.upsertFileWithPath = sinon.stub().yields(null, @file, false, [], @folder)
-			@EditorController.upsertFileWithPath @project_id, @filePath, @fsPath, @source, @user_id, @callback
+			@EditorController.upsertFileWithPath @project_id, @filePath, @fsPath, @linkedFileData, @source, @user_id, @callback
 
 		it 'upserts the file using the project entity handler', ->
 			@ProjectEntityUpdateHandler.upsertFileWithPath
-				.calledWith(@project_id, @filePath, @fsPath)
+				.calledWith(@project_id, @filePath, @fsPath, @linkedFileData)
 				.should.equal true
 
 		describe 'file does not exist', ->
 			beforeEach ->
 				@ProjectEntityUpdateHandler.upsertFileWithPath = sinon.stub().yields(null, @file, true, [], @folder)
-				@EditorController.upsertFileWithPath @project_id, @filePath, @fsPath, @source, @user_id, @callback
+				@EditorController.upsertFileWithPath @project_id, @filePath, @fsPath, @linkedFileData, @source, @user_id, @callback
 
 			it 'should send the update for the file out to users in the project', ->
 				@EditorRealTimeController.emitToRoom
-					.calledWith(@project_id, "reciveNewFile", @folder_id, @file, @source)
+					.calledWith(@project_id, "reciveNewFile", @folder_id, @file, @source, @linkedFileData)
 					.should.equal true
 
 		describe 'folders required for file do not exist', ->
@@ -195,7 +196,7 @@ describe "EditorController", ->
 					@folderB = { _id: 3, parentFolder_id: 2}
 				]
 				@ProjectEntityUpdateHandler.upsertFileWithPath = sinon.stub().yields(null, @file, true, folders, @folderB)
-				@EditorController.upsertFileWithPath @project_id, @filePath, @fsPath, @source, @user_id, @callback
+				@EditorController.upsertFileWithPath @project_id, @filePath, @fsPath, @linkedFileData, @source, @user_id, @callback
 
 			it 'should send the update for each folder to users in the project', ->
 				@EditorRealTimeController.emitToRoom
diff --git a/services/web/test/unit/coffee/Project/ProjectCreationHandlerTests.coffee b/services/web/test/unit/coffee/Project/ProjectCreationHandlerTests.coffee
index 1be6f1106a..7c940ea323 100644
--- a/services/web/test/unit/coffee/Project/ProjectCreationHandlerTests.coffee
+++ b/services/web/test/unit/coffee/Project/ProjectCreationHandlerTests.coffee
@@ -34,7 +34,7 @@ describe 'ProjectCreationHandler', ->
 				{@name} = options
 		@ProjectEntityUpdateHandler =
 			addDoc: sinon.stub().callsArgWith(5, null, {_id: docId})
-			addFile: sinon.stub().callsArg(5)
+			addFile: sinon.stub().callsArg(6)
 			setRootDoc: sinon.stub().callsArg(2)
 		@ProjectDetailsHandler =
 			validateProjectName: sinon.stub().yields()
@@ -208,6 +208,7 @@ describe 'ProjectCreationHandler', ->
 				.calledWith(
 					project_id, rootFolderId, "universe.jpg",
 					Path.resolve(__dirname + "/../../../../app/templates/project_files/universe.jpg"),
+					null,
 					ownerId
 				)
 				.should.equal true
diff --git a/services/web/test/unit/coffee/Project/ProjectEditorHandlerTests.coffee b/services/web/test/unit/coffee/Project/ProjectEditorHandlerTests.coffee
index 4431bd2129..903dd66be1 100644
--- a/services/web/test/unit/coffee/Project/ProjectEditorHandlerTests.coffee
+++ b/services/web/test/unit/coffee/Project/ProjectEditorHandlerTests.coffee
@@ -33,7 +33,7 @@ describe "ProjectEditorHandler", ->
 					fileRefs : [{
 						_id     : "file-id"
 						name    : "image.png"
-						created : new Date()
+						created : @created = new Date()
 						size    : 1234
 					}]
 					folders  : []
@@ -141,7 +141,7 @@ describe "ProjectEditorHandler", ->
 			it "should include files in the project", ->
 				@result.rootFolder[0].folders[0].fileRefs[0]._id.should.equal "file-id"
 				@result.rootFolder[0].folders[0].fileRefs[0].name.should.equal "image.png"
-				should.not.exist @result.rootFolder[0].folders[0].fileRefs[0].created
+				@result.rootFolder[0].folders[0].fileRefs[0].created.should.equal @created
 				should.not.exist @result.rootFolder[0].folders[0].fileRefs[0].size
 
 			it "should include docs in the project but not the lines", ->
diff --git a/services/web/test/unit/coffee/Project/ProjectEntityMongoUpdateHandlerTests.coffee b/services/web/test/unit/coffee/Project/ProjectEntityMongoUpdateHandlerTests.coffee
index be5b177786..e05d5bc38a 100644
--- a/services/web/test/unit/coffee/Project/ProjectEntityMongoUpdateHandlerTests.coffee
+++ b/services/web/test/unit/coffee/Project/ProjectEntityMongoUpdateHandlerTests.coffee
@@ -97,10 +97,11 @@ describe 'ProjectEntityMongoUpdateHandler', ->
 		beforeEach ->
 			@file = _id: file_id
 			@path = mongo: 'file.png'
+			@linkedFileData = {provider: 'url'}
 			@ProjectLocator.findElement = sinon.stub().yields(null, @file, @path)
 			@ProjectModel.update = sinon.stub().yields()
 
-			@subject.replaceFile project_id, file_id, @callback
+			@subject.replaceFile project_id, file_id, @linkedFileData, @callback
 
 		it 'gets the project', ->
 			@ProjectGetter.getProjectWithoutLock
@@ -118,7 +119,7 @@ describe 'ProjectEntityMongoUpdateHandler', ->
 					{ _id: project_id },
 					{
 						'$inc': { 'file.png.rev': 1, 'version': 1 }
-						'$set': { 'file.png.created': new Date() }
+						'$set': { 'file.png.created': new Date(), 'file.png.linkedFileData': @linkedFileData }
 					}
 					{}
 				)
diff --git a/services/web/test/unit/coffee/Project/ProjectEntityUpdateHandlerTests.coffee b/services/web/test/unit/coffee/Project/ProjectEntityUpdateHandlerTests.coffee
index ed4bbe8e33..ba40fa32db 100644
--- a/services/web/test/unit/coffee/Project/ProjectEntityUpdateHandlerTests.coffee
+++ b/services/web/test/unit/coffee/Project/ProjectEntityUpdateHandlerTests.coffee
@@ -40,6 +40,8 @@ describe 'ProjectEntityUpdateHandler', ->
 		@fileName = "something.jpg"
 		@fileSystemPath = "somehintg"
 
+		@linkedFileData = {provider: 'url'}
+
 		@source = 'editor'
 		@callback = sinon.stub()
 		@ProjectEntityUpdateHandler = SandboxedModule.require modulePath, requires:
@@ -296,11 +298,11 @@ describe 'ProjectEntityUpdateHandler', ->
 			@newFile = _id: file_id
 			@ProjectEntityUpdateHandler.addFileWithoutUpdatingHistory =
 				withoutLock: sinon.stub().yields(null, @newFile, folder_id, @path, @fileUrl)
-			@ProjectEntityUpdateHandler.addFile project_id, folder_id, @docName, @fileSystemPath, userId, @callback
+			@ProjectEntityUpdateHandler.addFile project_id, folder_id, @docName, @fileSystemPath, @linkedFileData, userId, @callback
 
 		it "creates the doc without history", () ->
 			@ProjectEntityUpdateHandler.addFileWithoutUpdatingHistory.withoutLock
-				.calledWith(project_id, folder_id, @docName, @fileSystemPath, userId)
+				.calledWith(project_id, folder_id, @docName, @fileSystemPath, @linkedFileData, userId)
 				.should.equal true
 
 		it "sends the change in project structure to the doc updater", () ->
@@ -320,7 +322,7 @@ describe 'ProjectEntityUpdateHandler', ->
 			@project = _id: project_id, name: 'some project'
 			@ProjectEntityMongoUpdateHandler.replaceFile = sinon.stub().yields(null, @newFile, @project, fileSystem: @path)
 
-			@ProjectEntityUpdateHandler.replaceFile project_id, file_id, @fileSystemPath, userId, @callback
+			@ProjectEntityUpdateHandler.replaceFile project_id, file_id, @fileSystemPath, @linkedFileData, userId, @callback
 
 		it 'uploads a new version of the file', ->
 			@FileStoreHandler.uploadFileFromDisk
@@ -329,7 +331,7 @@ describe 'ProjectEntityUpdateHandler', ->
 
 		it 'replaces the file in mongo', ->
 			@ProjectEntityMongoUpdateHandler.replaceFile
-				.calledWith(project_id, file_id)
+				.calledWith(project_id, file_id, @linkedFileData)
 				.should.equal true
 
 		it 'notifies the tpds', ->
@@ -497,7 +499,7 @@ describe 'ProjectEntityUpdateHandler', ->
 		describe 'upserting into an invalid folder', ->
 			beforeEach ->
 				@ProjectLocator.findElement = sinon.stub().yields()
-				@ProjectEntityUpdateHandler.upsertFile project_id, folder_id, @fileName, @fileSystemPath, userId, @callback
+				@ProjectEntityUpdateHandler.upsertFile project_id, folder_id, @fileName, @fileSystemPath, @linkedFileData, userId, @callback
 
 			it 'returns an error', ->
 				errorMatcher = sinon.match.instanceOf(Error)
@@ -511,11 +513,11 @@ describe 'ProjectEntityUpdateHandler', ->
 				@ProjectLocator.findElement = sinon.stub().yields(null, @folder)
 				@ProjectEntityUpdateHandler.replaceFile = withoutLock: sinon.stub().yields(null, @newFile)
 
-				@ProjectEntityUpdateHandler.upsertFile project_id, folder_id, @fileName, @fileSystemPath, userId, @callback
+				@ProjectEntityUpdateHandler.upsertFile project_id, folder_id, @fileName, @fileSystemPath, @linkedFileData, userId, @callback
 
 			it 'replaces the file', ->
 				@ProjectEntityUpdateHandler.replaceFile.withoutLock
-					.calledWith(project_id, file_id, @fileSystemPath, userId)
+					.calledWith(project_id, file_id, @fileSystemPath, @linkedFileData, userId)
 					.should.equal true
 
 			it 'returns the file', ->
@@ -528,7 +530,7 @@ describe 'ProjectEntityUpdateHandler', ->
 				@ProjectLocator.findElement = sinon.stub().yields(null, @folder)
 				@ProjectEntityUpdateHandler.addFile = withoutLock: sinon.stub().yields(null, @newFile)
 
-				@ProjectEntityUpdateHandler.upsertFile project_id, folder_id, @fileName, @fileSystemPath, userId, @callback
+				@ProjectEntityUpdateHandler.upsertFile project_id, folder_id, @fileName, @fileSystemPath, @linkedFileData, userId, @callback
 
 			it 'tries to find the folder', ->
 				@ProjectLocator.findElement
@@ -537,7 +539,7 @@ describe 'ProjectEntityUpdateHandler', ->
 
 			it 'adds the file', ->
 				@ProjectEntityUpdateHandler.addFile.withoutLock
-					.calledWith(project_id, folder_id, @fileName, @fileSystemPath, userId)
+					.calledWith(project_id, folder_id, @fileName, @fileSystemPath, @linkedFileData, userId)
 					.should.equal true
 
 			it 'returns the file', ->
@@ -584,7 +586,7 @@ describe 'ProjectEntityUpdateHandler', ->
 			@ProjectEntityUpdateHandler.upsertFile =
 				withoutLock: sinon.stub().yields(null, @file, @isNewFile)
 
-			@ProjectEntityUpdateHandler.upsertFileWithPath project_id, @path, @fileSystemPath, userId, @callback
+			@ProjectEntityUpdateHandler.upsertFileWithPath project_id, @path, @fileSystemPath, @linkedFileData, userId, @callback
 
 		it 'creates any necessary folders', ->
 			@ProjectEntityUpdateHandler.mkdirp.withoutLock
@@ -593,7 +595,7 @@ describe 'ProjectEntityUpdateHandler', ->
 
 		it 'upserts the file', ->
 			@ProjectEntityUpdateHandler.upsertFile.withoutLock
-				.calledWith(project_id, @folder._id, 'file.png', @fileSystemPath, userId)
+				.calledWith(project_id, @folder._id, 'file.png', @fileSystemPath, @linkedFileData, userId)
 				.should.equal true
 
 		it 'calls the callback', ->
diff --git a/services/web/test/unit/coffee/Subscription/SubscriptionControllerTests.coffee b/services/web/test/unit/coffee/Subscription/SubscriptionControllerTests.coffee
index 0f779a3ede..22f571ffa8 100644
--- a/services/web/test/unit/coffee/Subscription/SubscriptionControllerTests.coffee
+++ b/services/web/test/unit/coffee/Subscription/SubscriptionControllerTests.coffee
@@ -74,6 +74,7 @@ describe "SubscriptionController", ->
 			"settings-sharelatex": @settings
 			"./SubscriptionDomainHandler":@SubscriptionDomainHandler
 			"../User/UserGetter": @UserGetter
+			"./RecurlyWrapper": @RecurlyWrapper = {}
 
 
 		@res = new MockResponse()
@@ -117,6 +118,7 @@ describe "SubscriptionController", ->
 	describe "paymentPage", ->
 		beforeEach ->
 			@req.headers = {}
+			@RecurlyWrapper.sign = sinon.stub().yields(null, @signature = "signature")
 			@SubscriptionHandler.validateNoSubscriptionInRecurly = sinon.stub().yields(null, true)
 			@GeoIpLookup.getCurrencyCode.callsArgWith(1, null, @stubbedCurrencyCode)
 
diff --git a/services/web/test/unit/coffee/ThirdPartyDataStore/UpdateMergerTests.coffee b/services/web/test/unit/coffee/ThirdPartyDataStore/UpdateMergerTests.coffee
index 3d121e0bce..f031022280 100644
--- a/services/web/test/unit/coffee/ThirdPartyDataStore/UpdateMergerTests.coffee
+++ b/services/web/test/unit/coffee/ThirdPartyDataStore/UpdateMergerTests.coffee
@@ -15,18 +15,20 @@ describe 'UpdateMerger :', ->
 				err: ->
 			'../Editor/EditorController': @EditorController = {}
 			'../Uploads/FileTypeManager':@FileTypeManager = {}
+			'../../infrastructure/FileWriter': @FileWriter = {}
 			'settings-sharelatex':{path:{dumpPath:"dump_here"}}
 		@project_id = "project_id_here"
 		@user_id = "mock-user-id"
 
 		@docPath = "/folder/doc.tex"
 		@filePath = "/folder/file.png"
+		@linkedFileData = {provider: 'url'}
 
 		@fsPath = "/tmp/file/path"
 
 		@source = "dropbox"
 		@updateRequest = new BufferedStream()
-		@updateMerger.p.writeStreamToDisk = sinon.stub().yields(null, @fsPath)
+		@FileWriter.writeStreamToDisk = sinon.stub().yields(null, @fsPath)
 		@callback = sinon.stub()
 
 	describe 'mergeUpdate', ->
@@ -94,5 +96,5 @@ describe 'UpdateMerger :', ->
 
 			it 'should upsert the file in the editor controller', ->
 				@EditorController.upsertFileWithPath
-					.calledWith(@project_id, @filePath, @fsPath, @source, @user_id)
+					.calledWith(@project_id, @filePath, @fsPath, null, @source, @user_id)
 					.should.equal true
diff --git a/services/web/test/unit/coffee/Uploads/FileSystemImportManagerTests.coffee b/services/web/test/unit/coffee/Uploads/FileSystemImportManagerTests.coffee
index 13db922dda..ee22a8f7d8 100644
--- a/services/web/test/unit/coffee/Uploads/FileSystemImportManagerTests.coffee
+++ b/services/web/test/unit/coffee/Uploads/FileSystemImportManagerTests.coffee
@@ -78,12 +78,12 @@ describe "FileSystemImportManager", ->
 
 	describe "addFile with replace set to false", ->
 		beforeEach ->
-			@EditorController.addFile = sinon.stub().callsArg(6)
+			@EditorController.addFile = sinon.stub().yields()
 			@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
 			@FileSystemImportManager.addFile @user_id, @project_id, @folder_id, @name, @path_on_disk, false, @callback
 
 		it "should add the file", ->
-			@EditorController.addFile.calledWith(@project_id, @folder_id, @name, @path_on_disk, "upload", @user_id)
+			@EditorController.addFile.calledWith(@project_id, @folder_id, @name, @path_on_disk, null, "upload", @user_id)
 				.should.equal true
 
 	describe "addFile with symlink", ->
@@ -105,7 +105,7 @@ describe "FileSystemImportManager", ->
 
 		it "should add the file", ->
 			@EditorController.upsertFile
-				.calledWith(@project_id, @folder_id, @name, @path_on_disk, "upload", @user_id)
+				.calledWith(@project_id, @folder_id, @name, @path_on_disk, null, "upload", @user_id)
 				.should.equal true
 
 	describe "addFolder", ->

From a99f55891f8a8c22ea07f01304313db1571f73bf Mon Sep 17 00:00:00 2001
From: James Allen <james@sharelatex.com>
Date: Wed, 21 Feb 2018 11:19:21 +0000
Subject: [PATCH 7/9] Use external proxy

---
 .../Features/LinkedFiles/UrlAgent.coffee      | 24 +++++++++-----
 services/web/config/settings.defaults.coffee  |  2 ++
 services/web/docker-compose.yml               |  1 +
 .../acceptance/coffee/LinkedFilesTests.coffee | 33 +++++++++++--------
 .../coffee/helpers/MockURLSource.coffee       |  7 ----
 5 files changed, 37 insertions(+), 30 deletions(-)
 delete mode 100644 services/web/test/acceptance/coffee/helpers/MockURLSource.coffee

diff --git a/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee
index 98e176bc06..cb80108826 100644
--- a/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee
+++ b/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee
@@ -2,6 +2,7 @@ request = require 'request'
 FileWriter = require('../../infrastructure/FileWriter')
 _ = require "underscore"
 urlValidator = require 'valid-url'
+Settings = require 'settings-sharelatex'
 
 UrlFetchFailedError = (message) ->
 	error = new Error(message)
@@ -17,27 +18,21 @@ InvalidUrlError = (message) ->
 	return error
 InvalidUrlError.prototype.__proto__ = Error.prototype
 
-
 module.exports = UrlAgent = {
 	UrlFetchFailedError: UrlFetchFailedError
 	InvalidUrlError: InvalidUrlError
 
 	sanitizeData: (data) ->
 		return {
-			url: data.url
+			url: @._prependHttpIfNeeded(data.url)
 		}
 
-	_prependHttpIfNeeded: (url) ->
-		if !url.match('://')
-			url = 'http://' + url
-		return url
-
 	writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) ->
-		# TODO: Proxy through external API
 		callback = _.once(callback)
-		url = @._prependHttpIfNeeded(data.url)
+		url = data.url
 		if !urlValidator.isWebUri(url)
 			return callback(new InvalidUrlError())
+		url = @._wrapWithProxy(url)
 		readStream = request.get(url)
 		readStream.on "error", callback
 		readStream.on "response", (response) ->
@@ -59,4 +54,15 @@ module.exports = UrlAgent = {
 			)
 		else
 			next(error)
+
+	_prependHttpIfNeeded: (url) ->
+		if !url.match('://')
+			url = 'http://' + url
+		return url
+
+	_wrapWithProxy: (url) ->
+		# TODO: Consider what to do for Community and Enterprise edition?
+		if !Settings.apis?.linkedUrlProxy?.url?
+			throw new Error('no linked url proxy configured')
+		return "#{Settings.apis.linkedUrlProxy.url}?url=#{encodeURIComponent(url)}"
 }
\ No newline at end of file
diff --git a/services/web/config/settings.defaults.coffee b/services/web/config/settings.defaults.coffee
index 0c9b71c3ae..303743731d 100644
--- a/services/web/config/settings.defaults.coffee
+++ b/services/web/config/settings.defaults.coffee
@@ -152,6 +152,8 @@ module.exports = settings =
 			url: "http://#{process.env['NOTIFICATIONS_HOST'] or 'localhost'}:3042"
 		analytics:
 			url: "http://#{process.env['ANALYTICS_HOST'] or 'localhost'}:3050"
+		linkedUrlProxy:
+			url: process.env['LINKED_URL_PROXY']
 
 	templates:
 		user_id: process.env.TEMPLATES_USER_ID or "5395eb7aad1f29a88756c7f2"
diff --git a/services/web/docker-compose.yml b/services/web/docker-compose.yml
index 5862790191..0f228d528c 100644
--- a/services/web/docker-compose.yml
+++ b/services/web/docker-compose.yml
@@ -15,6 +15,7 @@ services:
       MONGO_URL: "mongodb://mongo/sharelatex"
       SHARELATEX_ALLOW_PUBLIC_ACCESS: 'true'
       PROJECT_HISTORY_ENABLED: 'true'
+      LINKED_URL_PROXY: 'http://localhost:6543'
     depends_on:
       - redis
       - mongo
diff --git a/services/web/test/acceptance/coffee/LinkedFilesTests.coffee b/services/web/test/acceptance/coffee/LinkedFilesTests.coffee
index 2c16a0f0e0..77af535eb5 100644
--- a/services/web/test/acceptance/coffee/LinkedFilesTests.coffee
+++ b/services/web/test/acceptance/coffee/LinkedFilesTests.coffee
@@ -3,18 +3,23 @@ expect = require("chai").expect
 _ = require 'underscore'
 
 MockFileStoreApi = require './helpers/MockFileStoreApi'
-MockURLSource	= require './helpers/MockURLSource'
 request = require "./helpers/request"
 User = require "./helpers/User"
 
-MockURLSource.app.get "/foo", (req, res, next) =>
-	res.send('foo foo foo')
-MockURLSource.app.get "/bar", (req, res, next) =>
-	res.send('bar bar bar')
+
+express = require("express")
+LinkedUrlProxy = express()
+LinkedUrlProxy.get "/", (req, res, next) =>
+	if req.query.url == 'http://example.com/foo'
+		res.send('foo foo foo')
+	else if req.query.url == 'http://example.com/bar'
+		res.send('bar bar bar')
+	else
+		res.sendStatus(404)
 
 describe "LinkedFiles", ->
 	before (done) ->
-		MockURLSource.run (error) =>
+		LinkedUrlProxy.listen 6543, (error) =>
 			return done(error) if error?
 			@owner = new User() 
 			@owner.login done
@@ -36,7 +41,7 @@ describe "LinkedFiles", ->
 				json:
 					provider: 'url'
 					data: {
-						url: "http://localhost:6543/foo"
+						url: 'http://example.com/foo'
 					}
 					parent_folder_id: @root_folder_id
 					name: 'url-test-file-1'
@@ -48,7 +53,7 @@ describe "LinkedFiles", ->
 					file = project.rootFolder[0].fileRefs[0]
 					expect(file.linkedFileData).to.deep.equal({
 						provider: 'url'
-						url: "http://localhost:6543/foo"
+						url: 'http://example.com/foo'
 					})
 					@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
 						throw error if error?
@@ -62,7 +67,7 @@ describe "LinkedFiles", ->
 				json:
 					provider: 'url'
 					data: {
-						url: "http://localhost:6543/foo"
+						url: 'http://example.com/foo'
 					}
 					parent_folder_id: @root_folder_id
 					name: 'url-test-file-2'
@@ -74,7 +79,7 @@ describe "LinkedFiles", ->
 					json:
 						provider: 'url'
 						data: {
-							url: "http://localhost:6543/bar"
+							url: 'http://example.com/bar'
 						}
 						parent_folder_id: @root_folder_id
 						name: 'url-test-file-2'
@@ -86,7 +91,7 @@ describe "LinkedFiles", ->
 						file = project.rootFolder[0].fileRefs[1]
 						expect(file.linkedFileData).to.deep.equal({
 							provider: 'url'
-							url: "http://localhost:6543/bar"
+							url: 'http://example.com/bar'
 						})
 						@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
 							throw error if error?
@@ -100,7 +105,7 @@ describe "LinkedFiles", ->
 				json:
 					provider: 'url'
 					data: {
-						url: "http://localhost:6543/does-not-exist"
+						url: 'http://example.com/does-not-exist'
 					}
 					parent_folder_id: @root_folder_id
 					name: 'url-test-file-3'
@@ -154,7 +159,7 @@ describe "LinkedFiles", ->
 				json:
 					provider: 'url'
 					data: {
-						url: "http://localhost:6543/foo"
+						url: 'example.com/foo'
 					}
 					parent_folder_id: @root_folder_id
 					name: 'url-test-file-6'
@@ -167,7 +172,7 @@ describe "LinkedFiles", ->
 						file.name == 'url-test-file-6'
 					expect(file.linkedFileData).to.deep.equal({
 						provider: 'url'
-						url: "http://localhost:6543/foo"
+						url: 'http://example.com/foo'
 					})
 					@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
 						throw error if error?
diff --git a/services/web/test/acceptance/coffee/helpers/MockURLSource.coffee b/services/web/test/acceptance/coffee/helpers/MockURLSource.coffee
deleted file mode 100644
index 1071a0e20f..0000000000
--- a/services/web/test/acceptance/coffee/helpers/MockURLSource.coffee
+++ /dev/null
@@ -1,7 +0,0 @@
-express = require("express")
-app = express()
-
-module.exports = MockUrlSource =
-	app: app
-	run: (callback) ->
-		app.listen 6543, callback

From d4025908b7171141ebe99001a3681d211d12961b Mon Sep 17 00:00:00 2001
From: James Allen <james@sharelatex.com>
Date: Thu, 1 Mar 2018 10:17:12 +0000
Subject: [PATCH 8/9] Add in enabledLinkedFileTypes setting

---
 .../coffee/Features/LinkedFiles/LinkedFilesController.coffee | 5 +++++
 services/web/app/views/project/editor.pug                    | 3 ++-
 services/web/config/settings.defaults.coffee                 | 2 ++
 services/web/docker-compose.yml                              | 1 +
 .../ide/file-tree/controllers/FileTreeController.coffee      | 3 +++
 5 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee
index f2566beb5c..e2320d148a 100644
--- a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee
+++ b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee
@@ -1,5 +1,7 @@
 AuthenticationController = require '../Authentication/AuthenticationController'
 EditorController = require '../Editor/EditorController'
+Settings = require 'settings-sharelatex'
+logger = require 'logger-sharelatex'
 
 module.exports = LinkedFilesController = {
 	Agents: {
@@ -10,9 +12,12 @@ module.exports = LinkedFilesController = {
 		{project_id} = req.params
 		{name, provider, data, parent_folder_id} = req.body
 		user_id = AuthenticationController.getLoggedInUserId(req)
+		logger.log {project_id, name, provider, data, parent_folder_id, user_id}, 'create linked file request'
 
 		if !LinkedFilesController.Agents.hasOwnProperty(provider)
 			return res.send(400)
+		unless provider in Settings.enabledLinkedFileTypes
+			return res.send(400)
 		Agent = LinkedFilesController.Agents[provider]
 
 		linkedFileData = Agent.sanitizeData(data)
diff --git a/services/web/app/views/project/editor.pug b/services/web/app/views/project/editor.pug
index 88ca3364b8..fed8f4beec 100644
--- a/services/web/app/views/project/editor.pug
+++ b/services/web/app/views/project/editor.pug
@@ -106,7 +106,7 @@ block requirejs
 	//- We need to do .replace(/\//g, '\\/') do that '</script>' -> '<\/script>'
 	//- and doesn't prematurely end the script tag.
 	script#data(type="application/json").
-		!{JSON.stringify({userSettings: userSettings, user: user, trackChangesState: trackChangesState, useV2History: useV2History}).replace(/\//g, '\\/')}
+		!{JSON.stringify({userSettings: userSettings, user: user, trackChangesState: trackChangesState, useV2History: useV2History, enabledLinkedFileTypes: settings.enabledLinkedFileTypes}).replace(/\//g, '\\/')}
 
 	script(type="text/javascript").
 		window.data = JSON.parse($("#data").text());
@@ -115,6 +115,7 @@ block requirejs
 		var data = JSON.parse($("#data").text());
 		window.userSettings = data.userSettings;
 		window.user = data.user;
+		window.enabledLinkedFiles = data.enabledLinkedFiles;
 		window.csrfToken = "!{csrfToken}";
 		window.anonymous = #{anonymous};
 		window.anonymousAccessToken = "#{anonymousAccessToken}";
diff --git a/services/web/config/settings.defaults.coffee b/services/web/config/settings.defaults.coffee
index 303743731d..1a057d4be8 100644
--- a/services/web/config/settings.defaults.coffee
+++ b/services/web/config/settings.defaults.coffee
@@ -216,6 +216,8 @@ module.exports = settings =
 
 	enableSubscriptions:false
 
+	enabledLinkedFileTypes: (process.env['ENABLED_LINKED_FILE_TYPES'] or '').split(',')
+
 	# i18n
 	# ------
 	#
diff --git a/services/web/docker-compose.yml b/services/web/docker-compose.yml
index 0f228d528c..b6649b65d3 100644
--- a/services/web/docker-compose.yml
+++ b/services/web/docker-compose.yml
@@ -15,6 +15,7 @@ services:
       MONGO_URL: "mongodb://mongo/sharelatex"
       SHARELATEX_ALLOW_PUBLIC_ACCESS: 'true'
       PROJECT_HISTORY_ENABLED: 'true'
+      ENABLED_LINKED_FILE_TYPES: 'url'
       LINKED_URL_PROXY: 'http://localhost:6543'
     depends_on:
       - redis
diff --git a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee
index 08f26862cb..3d4077b2dd 100644
--- a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee
+++ b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee
@@ -31,6 +31,9 @@ define [
 			)
 
 		$scope.openLinkedFileModal = window.openLinkedFileModal = () ->
+			unless 'url' in window.data.enabledLinkedFileTypes
+				console.warn("Url linked files are not enabled")
+				return
 			$modal.open(
 				templateUrl: "linkedFileModalTemplate"
 				controller:  "LinkedFileModalController"

From d92c3e8b55b58ca95f3296074a5a486ca2407ba3 Mon Sep 17 00:00:00 2001
From: James Allen <james@sharelatex.com>
Date: Thu, 1 Mar 2018 11:32:05 +0000
Subject: [PATCH 9/9] Logging and error message tweaks

---
 .../Features/LinkedFiles/LinkedFilesController.coffee       | 1 +
 .../web/app/coffee/Features/LinkedFiles/UrlAgent.coffee     | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee
index e2320d148a..aaf4172cf4 100644
--- a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee
+++ b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee
@@ -24,6 +24,7 @@ module.exports = LinkedFilesController = {
 		linkedFileData.provider = provider
 		Agent.writeIncomingFileToDisk project_id, linkedFileData, user_id, (error, fsPath) ->
 			if error?
+				logger.error {err: error, project_id, name, linkedFileData, parent_folder_id, user_id}, 'error writing linked file to disk'
 				return Agent.handleError(error, req, res, next)
 			EditorController.upsertFile project_id, parent_folder_id, name, fsPath, linkedFileData, "upload", user_id, (error) ->
 				return next(error) if error?
diff --git a/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee
index cb80108826..ad96aa628f 100644
--- a/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee
+++ b/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee
@@ -31,15 +31,15 @@ module.exports = UrlAgent = {
 		callback = _.once(callback)
 		url = data.url
 		if !urlValidator.isWebUri(url)
-			return callback(new InvalidUrlError())
-		url = @._wrapWithProxy(url)
+			return callback(new InvalidUrlError("invalid url: #{url}"))
+		url = UrlAgent._wrapWithProxy(url)
 		readStream = request.get(url)
 		readStream.on "error", callback
 		readStream.on "response", (response) ->
 			if 200 <= response.statusCode < 300
 				FileWriter.writeStreamToDisk project_id, readStream, callback
 			else
-				error = new UrlFetchFailedError()
+				error = new UrlFetchFailedError("url fetch failed: #{url}")
 				error.statusCode = response.statusCode
 				callback(error)