From 1e4ef0cc5b01aca8017d36e7266dd8234337bd71 Mon Sep 17 00:00:00 2001 From: Michael Walker Date: Fri, 12 Jan 2018 16:47:13 +0000 Subject: [PATCH] Fully implement max file size - add `repoStore.maxFileSize` key to config - use maxFileSize in ResourceCache on both header path and blob download path - make failures during commit less fragile --- services/git-bridge/conf/example_config.json | 3 + .../application/config/Config.java | 12 +++ .../uk/ac/ic/wlgitbridge/bridge/Bridge.java | 51 +++++++--- .../bridge/repo/FSGitRepoStore.java | 44 +++++++-- .../bridge/repo/GitProjectRepo.java | 24 +++-- .../wlgitbridge/bridge/repo/ProjectRepo.java | 7 +- .../ic/wlgitbridge/bridge/repo/RepoStore.java | 5 + .../bridge/repo/RepoStoreConfig.java | 21 ++++ .../bridge/repo/WalkOverrideGitRepo.java | 95 +++++++++++++++++++ .../bridge/resource/ResourceCache.java | 7 +- .../bridge/resource/UrlResourceCache.java | 51 ++++++++-- .../ac/ic/wlgitbridge/data/ServletFile.java | 5 + .../data/filestore/RawDirectory.java | 3 + .../wlgitbridge/data/filestore/RawFile.java | 2 + .../data/filestore/RepositoryFile.java | 5 + .../git/handler/WLReceivePackFactory.java | 8 +- .../git/handler/hook/WriteLatexPutHook.java | 20 ++-- .../wlgitbridge/git/servlet/WLGitServlet.java | 4 +- .../git/util/RepositoryObjectTreeWalker.java | 14 +-- .../wlgitbridge/server/GitBridgeServer.java | 15 ++- .../snapshot/getforversion/SnapshotFile.java | 5 + .../uk/ac/ic/wlgitbridge/util/Instance.java | 8 +- .../application/config/ConfigTest.java | 5 +- .../ac/ic/wlgitbridge/bridge/BridgeTest.java | 13 +++ .../bridge/repo/FSGitRepoStoreTest.java | 3 +- .../bridge/repo/GitProjectRepoTest.java | 4 +- .../bridge/swap/job/SwapJobImplTest.java | 1 + .../data/model/ResourceFetcherTest.java | 15 +-- 28 files changed, 367 insertions(+), 83 deletions(-) create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStoreConfig.java create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/WalkOverrideGitRepo.java diff --git a/services/git-bridge/conf/example_config.json b/services/git-bridge/conf/example_config.json index c7f1193811..3b844ef099 100644 --- a/services/git-bridge/conf/example_config.json +++ b/services/git-bridge/conf/example_config.json @@ -11,6 +11,9 @@ "oauth2ClientSecret": "asdf", "oauth2Server": "https://localhost" }, + "repoStore": { + "maxFileSize": 52428800 + }, "swapStore": { "type": "s3", "awsAccessKey": "asdf", diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java index 8a53ccdf2f..a6853e937f 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java @@ -4,6 +4,7 @@ import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import uk.ac.ic.wlgitbridge.application.exception.ConfigFileException; +import uk.ac.ic.wlgitbridge.bridge.repo.RepoStoreConfig; import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig; import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStoreConfig; import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource; @@ -30,6 +31,7 @@ public class Config implements JSONSource { config.postbackURL, config.serviceName, Oauth2.asSanitised(config.oauth2), + config.repoStore, SwapStoreConfig.sanitisedCopy(config.swapStore), config.swapJob ); @@ -45,6 +47,8 @@ public class Config implements JSONSource { @Nullable private Oauth2 oauth2; @Nullable + private RepoStoreConfig repoStore; + @Nullable private SwapStoreConfig swapStore; @Nullable private SwapJobConfig swapJob; @@ -69,6 +73,7 @@ public class Config implements JSONSource { String postbackURL, String serviceName, Oauth2 oauth2, + RepoStoreConfig repoStore, SwapStoreConfig swapStore, SwapJobConfig swapJob ) { @@ -80,6 +85,7 @@ public class Config implements JSONSource { this.postbackURL = postbackURL; this.serviceName = serviceName; this.oauth2 = oauth2; + this.repoStore = repoStore; this.swapStore = swapStore; this.swapJob = swapJob; } @@ -108,6 +114,8 @@ public class Config implements JSONSource { postbackURL += "/"; } oauth2 = new Gson().fromJson(configObject.get("oauth2"), Oauth2.class); + repoStore = new Gson().fromJson( + configObject.get("repoStore"), RepoStoreConfig.class); swapStore = new Gson().fromJson( configObject.get("swapStore"), SwapStoreConfig.class @@ -161,6 +169,10 @@ public class Config implements JSONSource { return oauth2; } + public Optional getRepoStore() { + return Optional.ofNullable(repoStore); + } + public Optional getSwapStore() { return Optional.ofNullable(swapStore); } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java index 31796c2ebc..c7c27564f2 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java @@ -2,6 +2,7 @@ package uk.ac.ic.wlgitbridge.bridge; import com.google.api.client.auth.oauth2.Credential; import org.eclipse.jgit.errors.RepositoryNotFoundException; +import uk.ac.ic.wlgitbridge.application.config.Config; import uk.ac.ic.wlgitbridge.bridge.db.DBStore; import uk.ac.ic.wlgitbridge.bridge.db.ProjectState; import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore; @@ -9,20 +10,16 @@ import uk.ac.ic.wlgitbridge.bridge.gc.GcJob; import uk.ac.ic.wlgitbridge.bridge.gc.GcJobImpl; import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard; import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock; -import uk.ac.ic.wlgitbridge.bridge.repo.FSGitRepoStore; -import uk.ac.ic.wlgitbridge.bridge.repo.GitProjectRepo; -import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo; -import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; +import uk.ac.ic.wlgitbridge.bridge.repo.*; import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache; import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache; import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotApi; import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi; +import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApiFacade; import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob; -import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig; import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobImpl; import uk.ac.ic.wlgitbridge.bridge.swap.store.S3SwapStore; import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore; -import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApiFacade; import uk.ac.ic.wlgitbridge.data.CandidateSnapshot; import uk.ac.ic.wlgitbridge.data.ProjectLockImpl; import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents; @@ -118,6 +115,7 @@ import java.util.*; * * 6. The Snapshot API, which provides data from the Overleaf app. * + * @see SnapshotApiFacade - wraps a concrete instance of the Snapshot API. * @see SnapshotApi - the interface for the Snapshot API. * @see NetSnapshotApi - the default concrete implementation * @@ -139,6 +137,8 @@ import java.util.*; */ public class Bridge { + private final Config config; + private final ProjectLock lock; private final RepoStore repoStore; @@ -157,29 +157,31 @@ public class Bridge { * swap store, and the swap job config. * * This should be the method used to create a Bridge. + * @param config The config to use * @param repoStore The repo store to use * @param dbStore The db store to use * @param swapStore The swap store to use - * @param swapJobConfig The swap config to use, or empty for no-op + * @param snapshotApi The snapshot api to use * @return The constructed Bridge. */ public static Bridge make( + Config config, RepoStore repoStore, DBStore dbStore, SwapStore swapStore, - Optional swapJobConfig, SnapshotApi snapshotApi ) { ProjectLock lock = new ProjectLockImpl((int threads) -> Log.info("Waiting for " + threads + " projects...") ); return new Bridge( + config, lock, repoStore, dbStore, swapStore, SwapJob.fromConfig( - swapJobConfig, + config.getSwapJob(), lock, repoStore, dbStore, @@ -205,6 +207,7 @@ public class Bridge { * @param resourceCache the {@link ResourceCache} to use */ Bridge( + Config config, ProjectLock lock, RepoStore repoStore, DBStore dbStore, @@ -214,6 +217,7 @@ public class Bridge { SnapshotApiFacade snapshotAPI, ResourceCache resourceCache ) { + this.config = config; this.lock = lock; this.repoStore = repoStore; this.dbStore = dbStore; @@ -621,6 +625,11 @@ public class Bridge { makeCommitsFromSnapshots(repo, snapshots); + // TODO: in case crashes around here, add an + // "updating_from_commit" column to the DB as a way to rollback the + // any failed partial updates before re-trying + // Also need to consider the empty state (a new git init'd repo being + // the rollback target) if (!snapshots.isEmpty()) { dbStore.setLatestVersionForProject( projectName, @@ -635,7 +644,7 @@ public class Bridge { * Performs the actual Git commits on the disk. * * Each commit adds files to the db store - * ({@link ResourceCache#get(String, String, String, Map, Map)}, + * ({@link ResourceCache#get(String, String, String, Map, Map, Optional)}, * and then removes any files that were deleted. * @param repo The repository to commit to * @param snapshots The snapshots to commit @@ -647,10 +656,25 @@ public class Bridge { Collection snapshots ) throws IOException, GitUserException { String name = repo.getProjectName(); + Optional maxSize = config + .getRepoStore() + .flatMap(RepoStoreConfig::getMaxFileSize); for (Snapshot snapshot : snapshots) { - Map fileTable = repo.getFiles(); - List files = new LinkedList<>(); + RawDirectory directory = repo.getDirectory(); + Map fileTable = directory.getFileTable(); + List files = new ArrayList<>(); files.addAll(snapshot.getSrcs()); + for (RawFile file : files) { + long size = file.size(); + /* Can't throw in ifPresent... */ + if (maxSize.isPresent()) { + long maxSize_ = maxSize.get(); + if (size >= maxSize_) { + throw new SizeLimitExceededException( + Optional.of(file.getPath()), size, maxSize_); + } + } + } Map fetchedUrls = new HashMap<>(); for (SnapshotAttachment snapshotAttachment : snapshot.getAtts()) { files.add( @@ -659,7 +683,8 @@ public class Bridge { snapshotAttachment.getUrl(), snapshotAttachment.getPath(), fileTable, - fetchedUrls + fetchedUrls, + maxSize ) ); } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java index eef1a92f86..a3bfe30928 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java @@ -2,6 +2,8 @@ package uk.ac.ic.wlgitbridge.bridge.repo; import com.google.api.client.repackaged.com.google.common.base.Preconditions; import org.apache.commons.io.FileUtils; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; import uk.ac.ic.wlgitbridge.util.Project; import uk.ac.ic.wlgitbridge.util.Tar; @@ -12,6 +14,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Optional; import java.util.function.Function; import static uk.ac.ic.wlgitbridge.util.Util.deleteInDirectoryApartFrom; @@ -21,17 +24,35 @@ import static uk.ac.ic.wlgitbridge.util.Util.deleteInDirectoryApartFrom; */ public class FSGitRepoStore implements RepoStore { + private static final long DEFAULT_MAX_FILE_SIZE = 50 * 1024 * 1024; + private final String repoStorePath; + private final File rootDirectory; + + private final long maxFileSize; + private final Function fsSizer; - public FSGitRepoStore(String repoStorePath) { - this(repoStorePath, d -> d.getTotalSpace() - d.getFreeSpace()); + public FSGitRepoStore( + String repoStorePath, + Optional maxFileSize + ) { + this( + repoStorePath, + maxFileSize.orElse(DEFAULT_MAX_FILE_SIZE), + d -> d.getTotalSpace() - d.getFreeSpace() + ); } - public FSGitRepoStore(String repoStorePath, Function fsSizer) { + public FSGitRepoStore( + String repoStorePath, + long maxFileSize, + Function fsSizer + ) { this.repoStorePath = repoStorePath; rootDirectory = initRootGitDirectory(repoStorePath); + this.maxFileSize = maxFileSize; this.fsSizer = fsSizer; } @@ -47,16 +68,25 @@ public class FSGitRepoStore implements RepoStore { @Override public ProjectRepo initRepo(String project) throws IOException { - GitProjectRepo ret = new GitProjectRepo(project); + GitProjectRepo ret = GitProjectRepo.fromName(project); ret.initRepo(this); - return ret; + return new WalkOverrideGitRepo( + ret, Optional.of(maxFileSize), Optional.empty()); } @Override public ProjectRepo getExistingRepo(String project) throws IOException { - GitProjectRepo ret = new GitProjectRepo(project); + GitProjectRepo ret = GitProjectRepo.fromName(project); ret.useExistingRepository(this); - return ret; + return new WalkOverrideGitRepo( + ret, Optional.of(maxFileSize), Optional.empty()); + } + + @Override + public ProjectRepo useJGitRepo(Repository repo, ObjectId commitId) { + GitProjectRepo ret = GitProjectRepo.fromJGitRepo(repo); + return new WalkOverrideGitRepo( + ret, Optional.of(maxFileSize), Optional.of(commitId)); } /* TODO: Perhaps we should just delete bad directories on the fly. */ diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepo.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepo.java index dd83169598..6f020d79ad 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepo.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepo.java @@ -9,7 +9,7 @@ import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.storage.file.FileRepositoryBuilder; import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents; -import uk.ac.ic.wlgitbridge.data.filestore.RawFile; +import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory; import uk.ac.ic.wlgitbridge.git.exception.GitUserException; import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker; import uk.ac.ic.wlgitbridge.util.Log; @@ -43,10 +43,19 @@ public class GitProjectRepo implements ProjectRepo { private final String projectName; private Optional repository; - public GitProjectRepo(String projectName) { + public static GitProjectRepo fromJGitRepo(Repository repo) { + return new GitProjectRepo( + repo.getWorkTree().getName(), Optional.of(repo)); + } + + public static GitProjectRepo fromName(String projectName) { + return new GitProjectRepo(projectName, Optional.empty()); + } + + GitProjectRepo(String projectName, Optional repository) { Preconditions.checkArgument(Project.isValidProjectName(projectName)); this.projectName = projectName; - repository = Optional.empty(); + this.repository = repository; } @Override @@ -61,7 +70,10 @@ public class GitProjectRepo implements ProjectRepo { initRepositoryField(repoStore); Preconditions.checkState(repository.isPresent()); Repository repo = this.repository.get(); - Preconditions.checkState(!repo.getObjectDatabase().exists()); + // TODO: assert that this is a fresh repo. At the moment, we can't be + // sure whether the repo to be init'd doesn't exist or is just fresh + // and we crashed / aborted while committing + if (repo.getObjectDatabase().exists()) return; repo.create(); } @@ -77,12 +89,12 @@ public class GitProjectRepo implements ProjectRepo { } @Override - public Map getFiles() + public RawDirectory getDirectory() throws IOException, GitUserException { Preconditions.checkState(repository.isPresent()); return new RepositoryObjectTreeWalker( repository.get() - ).getDirectoryContents().getFileTable(); + ).getDirectoryContents(Optional.empty()); } @Override diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/ProjectRepo.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/ProjectRepo.java index dc875f416b..1e77d45010 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/ProjectRepo.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/ProjectRepo.java @@ -2,13 +2,12 @@ package uk.ac.ic.wlgitbridge.bridge.repo; import org.eclipse.jgit.lib.Repository; import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents; -import uk.ac.ic.wlgitbridge.data.filestore.RawFile; +import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory; import uk.ac.ic.wlgitbridge.git.exception.GitUserException; import java.io.File; import java.io.IOException; import java.util.Collection; -import java.util.Map; /** * Created by winston on 20/08/2016. @@ -25,12 +24,12 @@ public interface ProjectRepo { RepoStore repoStore ) throws IOException; - Map getFiles( + RawDirectory getDirectory( ) throws IOException, GitUserException; Collection commitAndGetMissing( GitDirectoryContents gitDirectoryContents - ) throws IOException; + ) throws IOException, GitUserException; void runGC() throws IOException; diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java index 000ae25ca2..a5eb289942 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java @@ -1,5 +1,8 @@ package uk.ac.ic.wlgitbridge.bridge.repo; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; + import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -21,6 +24,8 @@ public interface RepoStore { ProjectRepo getExistingRepo(String project) throws IOException; + ProjectRepo useJGitRepo(Repository repo, ObjectId commitId); + void purgeNonexistentProjects( Collection existingProjectNames ); diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStoreConfig.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStoreConfig.java new file mode 100644 index 0000000000..3bfd308dcd --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStoreConfig.java @@ -0,0 +1,21 @@ +package uk.ac.ic.wlgitbridge.bridge.repo; + +import javax.annotation.Nullable; +import java.util.Optional; + +/** + * Created by winston on 02/07/2017. + */ +public class RepoStoreConfig { + + @Nullable + private final Long maxFileSize; + + public RepoStoreConfig(Long maxFileSize) { + this.maxFileSize = maxFileSize; + } + + public Optional getMaxFileSize() { + return Optional.ofNullable(maxFileSize); + } +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/WalkOverrideGitRepo.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/WalkOverrideGitRepo.java new file mode 100644 index 0000000000..c240001c7c --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/WalkOverrideGitRepo.java @@ -0,0 +1,95 @@ +package uk.ac.ic.wlgitbridge.bridge.repo; + +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; +import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents; +import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory; +import uk.ac.ic.wlgitbridge.git.exception.GitUserException; +import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.Optional; + +/** + * This class takes a GitProjectRepo and delegates all calls to it. + * + * The purpose is to insert a file size check in {@link #getDirectory()}. + * + * We delegate instead of subclass because we can't override the static + * constructors in {@link GitProjectRepo}. + */ +public class WalkOverrideGitRepo implements ProjectRepo { + + private final GitProjectRepo gitRepo; + + private final Optional maxFileSize; + + private final Optional commitId; + + public WalkOverrideGitRepo( + GitProjectRepo gitRepo, + Optional maxFileSize, + Optional commitId + ) { + this.gitRepo = gitRepo; + this.maxFileSize = maxFileSize; + this.commitId = commitId; + } + + @Override + public String getProjectName() { + return gitRepo.getProjectName(); + } + + @Override + public void initRepo(RepoStore repoStore) throws IOException { + gitRepo.initRepo(repoStore); + } + + @Override + public void useExistingRepository(RepoStore repoStore) throws IOException { + gitRepo.useExistingRepository(repoStore); + } + + @Override + public RawDirectory getDirectory() throws IOException, GitUserException { + Repository repo = gitRepo.getJGitRepository(); + RepositoryObjectTreeWalker walker; + if (commitId.isPresent()) { + walker = new RepositoryObjectTreeWalker(repo, commitId.get()); + } else { + walker = new RepositoryObjectTreeWalker(repo); + } + return walker.getDirectoryContents(maxFileSize); + } + + @Override + public Collection commitAndGetMissing( + GitDirectoryContents gitDirectoryContents + ) throws GitUserException, IOException { + return gitRepo.commitAndGetMissing(gitDirectoryContents); + } + + @Override + public void runGC() throws IOException { + gitRepo.runGC(); + } + + @Override + public void deleteIncomingPacks() throws IOException { + gitRepo.deleteIncomingPacks(); + } + + @Override + public File getProjectDir() { + return gitRepo.getProjectDir(); + } + + @Override + public Repository getJGitRepository() { + return gitRepo.getJGitRepository(); + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/ResourceCache.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/ResourceCache.java index 7744c33386..7a520bb3f8 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/ResourceCache.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/ResourceCache.java @@ -1,9 +1,11 @@ package uk.ac.ic.wlgitbridge.bridge.resource; import uk.ac.ic.wlgitbridge.data.filestore.RawFile; +import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException; import java.io.IOException; import java.util.Map; +import java.util.Optional; /** * Created by winston on 20/08/2016. @@ -15,7 +17,8 @@ public interface ResourceCache { String url, String newPath, Map fileTable, - Map fetchedUrls - ) throws IOException; + Map fetchedUrls, + Optional maxFileSize + ) throws IOException, SizeLimitExceededException; } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/UrlResourceCache.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/UrlResourceCache.java index 8f18d58525..bde1d1f4b9 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/UrlResourceCache.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/UrlResourceCache.java @@ -1,18 +1,19 @@ package uk.ac.ic.wlgitbridge.bridge.resource; -import com.ning.http.client.AsyncCompletionHandler; -import com.ning.http.client.HttpResponseBodyPart; -import com.ning.http.client.Response; +import com.ning.http.client.*; import uk.ac.ic.wlgitbridge.bridge.db.DBStore; import uk.ac.ic.wlgitbridge.data.filestore.RawFile; import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile; +import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException; import uk.ac.ic.wlgitbridge.snapshot.base.Request; import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException; import uk.ac.ic.wlgitbridge.util.Log; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutionException; /** @@ -32,13 +33,14 @@ public class UrlResourceCache implements ResourceCache { String url, String newPath, Map fileTable, - Map fetchedUrls - ) throws IOException { + Map fetchedUrls, + Optional maxFileSize + ) throws IOException, SizeLimitExceededException { String path = dbStore.getPathForURLInProject(projectName, url); byte[] contents; if (path == null) { path = newPath; - contents = fetch(projectName, url, path); + contents = fetch(projectName, url, path, maxFileSize); fetchedUrls.put(url, contents); } else { Log.info("Found (" + projectName + "): " + url); @@ -54,7 +56,7 @@ public class UrlResourceCache implements ResourceCache { + "File url is: " + url ); - contents = fetch(projectName, url, path); + contents = fetch(projectName, url, path, maxFileSize); } else { contents = rawFile.getContents(); } @@ -66,8 +68,9 @@ public class UrlResourceCache implements ResourceCache { private byte[] fetch( String projectName, final String url, - String path - ) throws FailedConnectionException { + String path, + Optional maxFileSize + ) throws FailedConnectionException, SizeLimitExceededException { byte[] contents; Log.info("GET -> " + url); try { @@ -76,6 +79,28 @@ public class UrlResourceCache implements ResourceCache { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + @Override + public STATE onHeadersReceived( + HttpResponseHeaders headers + ) throws SizeLimitExceededException { + List contentLengths + = headers.getHeaders().get("Content-Length"); + if (!maxFileSize.isPresent()) { + return STATE.CONTINUE; + } + if (contentLengths.isEmpty()) { + return STATE.CONTINUE; + } + long contentLength = Long.parseLong(contentLengths.get(0)); + long maxFileSize_ = maxFileSize.get(); + if (contentLength <= maxFileSize_) { + return STATE.CONTINUE; + } + throw new SizeLimitExceededException( + Optional.of(path), contentLength, maxFileSize_ + ); + } + @Override public STATE onBodyPartReceived( HttpResponseBodyPart bodyPart @@ -115,6 +140,10 @@ public class UrlResourceCache implements ResourceCache { ); throw new FailedConnectionException(); } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof SizeLimitExceededException) { + throw (SizeLimitExceededException) cause; + } Log.warn( "ExecutionException when fetching project: " + projectName + @@ -126,6 +155,10 @@ public class UrlResourceCache implements ResourceCache { ); throw new FailedConnectionException(); } + if (maxFileSize.isPresent() && contents.length > maxFileSize.get()) { + throw new SizeLimitExceededException( + Optional.of(path), contents.length, maxFileSize.get()); + } dbStore.addURLIndexForProject(projectName, url, path); return contents; } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ServletFile.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ServletFile.java index bc42ba8c9b..ed3216ae4d 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ServletFile.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ServletFile.java @@ -25,6 +25,11 @@ public class ServletFile extends RawFile { return file.getContents(); } + @Override + public long size() { + return getContents().length; + } + public boolean isChanged() { return changed; } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawDirectory.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawDirectory.java index 2589519386..db8a554282 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawDirectory.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawDirectory.java @@ -1,6 +1,9 @@ package uk.ac.ic.wlgitbridge.data.filestore; +import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException; + import java.util.Map; +import java.util.Optional; /** * Created by Winston on 16/11/14. diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawFile.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawFile.java index 141c820260..d799843c6a 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawFile.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawFile.java @@ -17,6 +17,8 @@ public abstract class RawFile { public abstract byte[] getContents(); + public abstract long size(); + public final void writeToDisk(File directory) throws IOException { File file = new File(directory, getPath()); file.getParentFile().mkdirs(); diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RepositoryFile.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RepositoryFile.java index 751c672d23..e7cc331aec 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RepositoryFile.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RepositoryFile.java @@ -23,4 +23,9 @@ public class RepositoryFile extends RawFile { return contents; } + @Override + public long size() { + return contents.length; + } + } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java index 70bf90de74..c6f0444fba 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java @@ -5,6 +5,7 @@ import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.transport.ReceivePack; import org.eclipse.jgit.transport.resolver.ReceivePackFactory; import uk.ac.ic.wlgitbridge.bridge.Bridge; +import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi; import uk.ac.ic.wlgitbridge.git.handler.hook.WriteLatexPutHook; import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet; @@ -28,9 +29,12 @@ import java.util.Optional; public class WLReceivePackFactory implements ReceivePackFactory { + private final RepoStore repoStore; + private final Bridge bridge; - public WLReceivePackFactory(Bridge bridge) { + public WLReceivePackFactory(RepoStore repoStore, Bridge bridge) { + this.repoStore = repoStore; this.bridge = bridge; } @@ -69,7 +73,7 @@ public class WLReceivePackFactory hostname = httpServletRequest.getLocalName(); } receivePack.setPreReceiveHook( - new WriteLatexPutHook(bridge, hostname, oauth2) + new WriteLatexPutHook(repoStore, bridge, hostname, oauth2) ); return receivePack; } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/WriteLatexPutHook.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/WriteLatexPutHook.java index b1da5ed31c..d5b8b8f1f4 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/WriteLatexPutHook.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/WriteLatexPutHook.java @@ -7,12 +7,12 @@ import org.eclipse.jgit.transport.ReceiveCommand; import org.eclipse.jgit.transport.ReceiveCommand.Result; import org.eclipse.jgit.transport.ReceivePack; import uk.ac.ic.wlgitbridge.bridge.Bridge; +import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory; import uk.ac.ic.wlgitbridge.git.exception.GitUserException; import uk.ac.ic.wlgitbridge.git.handler.WLReceivePackFactory; import uk.ac.ic.wlgitbridge.git.handler.hook.exception.ForcedPushException; import uk.ac.ic.wlgitbridge.git.handler.hook.exception.WrongBranchException; -import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker; import uk.ac.ic.wlgitbridge.snapshot.push.exception.InternalErrorException; import uk.ac.ic.wlgitbridge.snapshot.push.exception.OutOfDateException; import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException; @@ -33,6 +33,8 @@ import java.util.Optional; */ public class WriteLatexPutHook implements PreReceiveHook { + private final RepoStore repoStore; + private final Bridge bridge; private final String hostname; private final Optional oauth2; @@ -41,15 +43,18 @@ public class WriteLatexPutHook implements PreReceiveHook { * The constructor to use, which provides the hook with the {@link Bridge}, * the hostname (used to construct a URL to give to Overleaf to postback), * and the oauth2 (used to authenticate with the Snapshot API). + * @param repoStore * @param bridge the {@link Bridge} * @param hostname the hostname used for postback from the Snapshot API * @param oauth2 used to authenticate with the snapshot API, or null */ public WriteLatexPutHook( + RepoStore repoStore, Bridge bridge, String hostname, Optional oauth2 ) { + this.repoStore = repoStore; this.bridge = bridge; this.hostname = hostname; this.oauth2 = oauth2; @@ -152,18 +157,17 @@ public class WriteLatexPutHook implements PreReceiveHook { Repository repository, ReceiveCommand receiveCommand ) throws IOException, GitUserException { - return new RepositoryObjectTreeWalker( - repository, - receiveCommand.getNewId() - ).getDirectoryContents(); + return repoStore + .useJGitRepo(repository, receiveCommand.getNewId()) + .getDirectory(); } private RawDirectory getOldDirectoryContents( Repository repository ) throws IOException, GitUserException { - return new RepositoryObjectTreeWalker( - repository - ).getDirectoryContents(); + return repoStore + .useJGitRepo(repository, repository.resolve("HEAD")) + .getDirectory(); } } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/servlet/WLGitServlet.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/servlet/WLGitServlet.java index 7a6f39ce69..da491046ed 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/servlet/WLGitServlet.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/servlet/WLGitServlet.java @@ -3,6 +3,7 @@ package uk.ac.ic.wlgitbridge.git.servlet; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jgit.http.server.GitServlet; import uk.ac.ic.wlgitbridge.bridge.Bridge; +import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; import uk.ac.ic.wlgitbridge.git.handler.WLReceivePackFactory; import uk.ac.ic.wlgitbridge.git.handler.WLRepositoryResolver; import uk.ac.ic.wlgitbridge.git.handler.WLUploadPackFactory; @@ -39,10 +40,11 @@ public class WLGitServlet extends GitServlet { */ public WLGitServlet( ServletContextHandler ctxHandler, + RepoStore repoStore, Bridge bridge ) throws ServletException { setRepositoryResolver(new WLRepositoryResolver(bridge)); - setReceivePackFactory(new WLReceivePackFactory(bridge)); + setReceivePackFactory(new WLReceivePackFactory(repoStore, bridge)); setUploadPackFactory(new WLUploadPackFactory()); init(new WLGitServletConfig(ctxHandler)); } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/util/RepositoryObjectTreeWalker.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/util/RepositoryObjectTreeWalker.java index 836b0b319d..f7c9f555da 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/util/RepositoryObjectTreeWalker.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/util/RepositoryObjectTreeWalker.java @@ -47,19 +47,7 @@ public class RepositoryObjectTreeWalker { this(repository, repository.resolve("HEAD~" + fromHead)); } - public RawDirectory getDirectoryContents( - ) throws IOException, SizeLimitExceededException, InvalidGitRepository { - return getDirectoryContents(Optional.empty()); - } - - public RawDirectory getDirectoryContents(long maxFileSize) - throws InvalidGitRepository, - SizeLimitExceededException, - IOException { - return getDirectoryContents(Optional.of(maxFileSize)); - } - - private RawDirectory getDirectoryContents(Optional maxFileSize) + public RawDirectory getDirectoryContents(Optional maxFileSize) throws IOException, SizeLimitExceededException, InvalidGitRepository { diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java index 1984f056eb..3008ed1028 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java @@ -13,6 +13,7 @@ import uk.ac.ic.wlgitbridge.bridge.db.DBStore; import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore; import uk.ac.ic.wlgitbridge.bridge.repo.FSGitRepoStore; import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; +import uk.ac.ic.wlgitbridge.bridge.repo.RepoStoreConfig; import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotApi; import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi; import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore; @@ -52,7 +53,10 @@ public class GitBridgeServer { org.eclipse.jetty.util.log.Log.setLog(new NullLogger()); this.port = config.getPort(); this.rootGitDirectoryPath = config.getRootGitDirectory(); - RepoStore repoStore = new FSGitRepoStore(rootGitDirectoryPath); + RepoStore repoStore = new FSGitRepoStore( + rootGitDirectoryPath, + config.getRepoStore().flatMap(RepoStoreConfig::getMaxFileSize) + ); DBStore dbStore = new SqliteDBStore( Paths.get( repoStore.getRootDirectory().getAbsolutePath() @@ -61,14 +65,14 @@ public class GitBridgeServer { SwapStore swapStore = SwapStore.fromConfig(config.getSwapStore()); SnapshotApi snapshotApi = new NetSnapshotApi(); bridge = Bridge.make( + config, repoStore, dbStore, swapStore, - config.getSwapJob(), snapshotApi ); jettyServer = new Server(port); - configureJettyServer(config, snapshotApi); + configureJettyServer(config, repoStore, snapshotApi); SnapshotAPIRequest.setBasicAuth( config.getUsername(), config.getPassword() @@ -110,11 +114,12 @@ public class GitBridgeServer { private void configureJettyServer( Config config, + RepoStore repoStore, SnapshotApi snapshotApi ) throws ServletException { HandlerCollection handlers = new HandlerList(); handlers.addHandler(initApiHandler()); - handlers.addHandler(initGitHandler(config, snapshotApi)); + handlers.addHandler(initGitHandler(config, repoStore, snapshotApi)); jettyServer.setHandler(handlers); } @@ -136,6 +141,7 @@ public class GitBridgeServer { private Handler initGitHandler( Config config, + RepoStore repoStore, SnapshotApi snapshotApi ) throws ServletException { final ServletContextHandler servletContextHandler = @@ -153,6 +159,7 @@ public class GitBridgeServer { new ServletHolder( new WLGitServlet( servletContextHandler, + repoStore, bridge ) ), diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/getforversion/SnapshotFile.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/getforversion/SnapshotFile.java index 5c41960475..a74ec53eb3 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/getforversion/SnapshotFile.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/getforversion/SnapshotFile.java @@ -44,6 +44,11 @@ public class SnapshotFile extends RawFile implements JSONSource { return contents; } + @Override + public long size() { + return contents.length; + } + /* Mock server */ public SnapshotFile(String contents, String path) { diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Instance.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Instance.java index c89e1c5526..75e72924aa 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Instance.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Instance.java @@ -21,9 +21,11 @@ public class Instance { public static final JsonFactory jsonFactory = new GsonFactory(); - public static final Gson prettyGson = - new GsonBuilder( - ).setPrettyPrinting().disableHtmlEscaping().create(); + public static final Gson prettyGson = new GsonBuilder() + .setPrettyPrinting() + .serializeNulls() + .disableHtmlEscaping() + .create(); public static final Gson gson = new Gson(); diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java index 98d42251f8..096efe9609 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java @@ -95,7 +95,10 @@ public class ConfigTest { " \"oauth2ClientID\": \"\",\n" + " \"oauth2ClientSecret\": \"\",\n" + " \"oauth2Server\": \"https://www.overleaf.com\"\n" + - " }\n" + + " },\n" + + " \"repoStore\": null,\n" + + " \"swapStore\": null,\n" + + " \"swapJob\": null\n" + "}"; assertEquals( "sanitised config did not hide sensitive fields", diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java index ad05047661..4d6df351ea 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java @@ -2,6 +2,7 @@ package uk.ac.ic.wlgitbridge.bridge; import org.junit.Before; import org.junit.Test; +import uk.ac.ic.wlgitbridge.application.config.Config; import uk.ac.ic.wlgitbridge.bridge.db.DBStore; import uk.ac.ic.wlgitbridge.bridge.db.ProjectState; import uk.ac.ic.wlgitbridge.bridge.gc.GcJob; @@ -50,6 +51,18 @@ public class BridgeTest { swapJob = mock(SwapJob.class); gcJob = mock(GcJob.class); bridge = new Bridge( + new Config( + 0, + "", + "", + "", + "", + "", + "", + null, + null, + null, + null), lock, repoStore, dbStore, diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest.java index c5291610e8..fae03e50a4 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest.java @@ -10,6 +10,7 @@ import java.io.*; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.Optional; import static org.junit.Assert.*; @@ -42,7 +43,7 @@ public class FSGitRepoStoreTest { File tmp = makeTempRepoDir(tmpFolder, "rootdir"); original = tmpFolder.newFolder("original"); FileUtils.copyDirectory(tmp, original); - repoStore = new FSGitRepoStore(tmp.getAbsolutePath()); + repoStore = new FSGitRepoStore(tmp.getAbsolutePath(), Optional.empty()); } @Test diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepoTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepoTest.java index b888f28c1f..3f3504b784 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepoTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepoTest.java @@ -56,7 +56,7 @@ public class GitProjectRepoTest { @Before public void setup() throws IOException { rootdir = makeTempRepoDir(tmpFolder, "rootdir"); - repoStore = new FSGitRepoStore(rootdir.getAbsolutePath()); + repoStore = new FSGitRepoStore(rootdir.getAbsolutePath(), Optional.empty()); repo = fromExistingDir("repo"); badGitignore = fromExistingDir("badgitignore"); incoming = fromExistingDir("incoming"); @@ -64,7 +64,7 @@ public class GitProjectRepoTest { } private GitProjectRepo fromExistingDir(String dir) throws IOException { - GitProjectRepo ret = new GitProjectRepo(dir); + GitProjectRepo ret = GitProjectRepo.fromName(dir); ret.useExistingRepository(repoStore); return ret; } diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java index 7f4fc3f284..5eb3196456 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java @@ -45,6 +45,7 @@ public class SwapJobImplTest { tmpFolder, "repostore" ).getAbsolutePath(), + 100_000, FileUtils::sizeOfDirectory ); dbStore = new SqliteDBStore(tmpFolder.newFile()); diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcherTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcherTest.java index 9ee9282c44..740220ca8f 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcherTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcherTest.java @@ -9,7 +9,6 @@ import org.mockserver.client.server.MockServerClient; import org.mockserver.junit.MockServerRule; import uk.ac.ic.wlgitbridge.bridge.db.DBStore; import uk.ac.ic.wlgitbridge.bridge.repo.FSGitRepoStore; -import uk.ac.ic.wlgitbridge.bridge.repo.GitProjectRepo; import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo; import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache; @@ -20,6 +19,7 @@ import uk.ac.ic.wlgitbridge.git.exception.GitUserException; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.mockserver.model.HttpRequest.request; @@ -67,12 +67,13 @@ public class ResourceFetcherTest { TemporaryFolder repositoryFolder = new TemporaryFolder(); repositoryFolder.create(); String repoStorePath = repositoryFolder.getRoot().getAbsolutePath(); - RepoStore repoStore = new FSGitRepoStore(repoStorePath); - ProjectRepo repo = new GitProjectRepo("repo"); - repo.initRepo(repoStore); - Map fileTable = repo.getFiles(); - Map fetchedUrls = new HashMap(); - resources.get(testProjectName, testUrl, newTestPath, fileTable, fetchedUrls); + RepoStore repoStore = new FSGitRepoStore(repoStorePath, Optional.empty()); + ProjectRepo repo = repoStore.initRepo("repo"); + Map fileTable = repo.getDirectory().getFileTable(); + Map fetchedUrls = new HashMap<>(); + resources.get( + testProjectName, testUrl, newTestPath, + fileTable, fetchedUrls, Optional.empty()); // We don't bother caching in this case, at present. assertEquals(0, fetchedUrls.size());