From c459cd57af16e31b8b27ef87f7fa2451f30ec28e Mon Sep 17 00:00:00 2001 From: Winston Li Date: Wed, 24 Aug 2016 15:15:34 +0100 Subject: [PATCH] Implement and test SwapJobImpl --- .../application/config/Config.java | 17 ++ .../uk/ac/ic/wlgitbridge/bridge/Bridge.java | 16 +- .../ac/ic/wlgitbridge/bridge/db/DBStore.java | 2 + .../bridge/db/sqlite/SqliteDBStore.java | 5 + .../sqlite/query/GetNumUnswappedProjects.java | 31 ++++ .../ic/wlgitbridge/bridge/lock/LockGuard.java | 10 ++ .../wlgitbridge/bridge/lock/ProjectLock.java | 8 + .../wlgitbridge/bridge/repo/FSRepoStore.java | 7 +- .../ic/wlgitbridge/bridge/repo/RepoStore.java | 16 +- .../ic/wlgitbridge/bridge/swap/SwapJob.java | 14 -- .../bridge/swap/SwapJobConfig.java | 33 ---- .../wlgitbridge/bridge/swap/SwapJobImpl.java | 73 -------- .../ic/wlgitbridge/bridge/swap/SwapStore.java | 21 --- .../bridge/swap/job/NoopSwapJob.java | 30 ++++ .../wlgitbridge/bridge/swap/job/SwapJob.java | 42 +++++ .../bridge/swap/job/SwapJobConfig.java | 44 +++++ .../bridge/swap/job/SwapJobImpl.java | 163 ++++++++++++++++++ .../swap/{ => store}/InMemorySwapStore.java | 2 +- .../bridge/swap/store/NoopSwapStore.java | 35 ++++ .../bridge/swap/{ => store}/S3SwapStore.java | 12 +- .../bridge/swap/store/SwapStore.java | 43 +++++ .../bridge/swap/store/SwapStoreConfig.java | 63 +++++++ .../wlgitbridge/server/GitBridgeServer.java | 60 ++++--- .../java/uk/ac/ic/wlgitbridge/util/Tar.java | 11 +- .../ac/ic/wlgitbridge/bridge/BridgeTest.java | 10 +- .../bridge/db/sqlite/SqliteDBStoreTest.java | 15 ++ .../DeleteFilesForProjectSQLUpdateTest.java | 14 +- .../bridge/repo/FSRepoStoreTest.java | 22 ++- .../bridge/swap/SwapJobImplTest.java | 58 ------- .../bridge/swap/job/SwapJobImplTest.java | 119 +++++++++++++ .../{ => store}/InMemorySwapStoreTest.java | 6 +- .../swap/{ => store}/S3SwapStoreTest.java | 2 +- .../src/test/resources/logback-test.xml | 6 +- 33 files changed, 745 insertions(+), 265 deletions(-) create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumUnswappedProjects.java create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/LockGuard.java delete mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java delete mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobConfig.java delete mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java delete mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/NoopSwapJob.java create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java rename services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/{ => store}/InMemorySwapStore.java (95%) create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/NoopSwapStore.java rename services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/{ => store}/S3SwapStore.java (86%) create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStore.java create mode 100644 services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStoreConfig.java delete mode 100644 services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImplTest.java create mode 100644 services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java rename services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/{ => store}/InMemorySwapStoreTest.java (93%) rename services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/{ => store}/S3SwapStoreTest.java (95%) diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java index f434437606..26ffa495fd 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java @@ -4,12 +4,16 @@ import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import uk.ac.ic.wlgitbridge.application.exception.ConfigFileException; +import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig; +import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStoreConfig; import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource; import uk.ac.ic.wlgitbridge.util.Instance; +import javax.annotation.Nullable; import java.io.FileReader; import java.io.IOException; import java.io.Reader; +import java.util.Optional; /** * Created by Winston on 05/12/14. @@ -36,7 +40,12 @@ public class Config implements JSONSource { private String apiBaseURL; private String postbackURL; private String serviceName; + @Nullable private Oauth2 oauth2; + @Nullable + private SwapStoreConfig swapStore; + @Nullable + private SwapJobConfig swapJob; public Config(String configFilePath) throws ConfigFileException, IOException { @@ -134,6 +143,14 @@ public class Config implements JSONSource { return oauth2; } + public Optional getSwapStore() { + return Optional.ofNullable(swapStore); + } + + public Optional getSwapJob() { + return Optional.ofNullable(swapJob); + } + private JsonElement getElement(JsonObject configObject, String name) { JsonElement element = configObject.get(name); if (element == null) { diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java index a3646c1b7b..0f3d014e2f 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java @@ -9,10 +9,9 @@ import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache; import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache; import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotAPI; import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI; -import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob; -import uk.ac.ic.wlgitbridge.bridge.swap.SwapJobConfig; -import uk.ac.ic.wlgitbridge.bridge.swap.SwapJobImpl; -import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore; +import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob; +import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig; +import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore; import uk.ac.ic.wlgitbridge.data.CandidateSnapshot; import uk.ac.ic.wlgitbridge.data.ProjectLockImpl; import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents; @@ -31,7 +30,6 @@ import uk.ac.ic.wlgitbridge.snapshot.push.exception.*; import uk.ac.ic.wlgitbridge.util.Log; import java.io.IOException; -import java.time.Duration; import java.util.*; /** @@ -56,7 +54,7 @@ public class Bridge { RepoStore repoStore, DBStore dbStore, SwapStore swapStore, - SwapJobConfig swapJobConfig + Optional swapJobConfig ) { ProjectLock lock = new ProjectLockImpl((int threads) -> Log.info("Waiting for " + threads + " projects...") @@ -66,7 +64,7 @@ public class Bridge { repoStore, dbStore, swapStore, - new SwapJobImpl( + SwapJob.fromConfig( swapJobConfig, lock, repoStore, @@ -108,8 +106,8 @@ public class Bridge { Log.info("Bye"); } - public void startSwapJob(Duration interval) { - swapJob.start(interval); + public void startSwapJob() { + swapJob.start(); } /* TODO: Remove these when WLBridged is moved into RepoStore */ diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java index 79267596c8..0f5d40868a 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java @@ -24,6 +24,8 @@ public interface DBStore { String getOldestUnswappedProject(); + int getNumUnswappedProjects(); + /** * Sets the last accessed time for the given project name. * @param projectName the project's name diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java index 1b4566b54d..8cc7f1f6e7 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java @@ -86,6 +86,11 @@ public class SqliteDBStore implements DBStore { return query(new GetOldestProjectName()); } + @Override + public int getNumUnswappedProjects() { + return query(new GetNumUnswappedProjects()); + } + @Override public void setLastAccessedTime( String projectName, diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumUnswappedProjects.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumUnswappedProjects.java new file mode 100644 index 0000000000..ce1c609395 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumUnswappedProjects.java @@ -0,0 +1,31 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery; + +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Created by winston on 24/08/2016. + */ +public class GetNumUnswappedProjects implements SQLQuery { + + private static final String GET_NUM_UNSWAPPED_PROJECTS = + "SELECT COUNT(*)\n" + + " FROM `swap_table`\n" + + " WHERE `last_accessed` IS NOT NULL"; + + @Override + public String getSQL() { + return GET_NUM_UNSWAPPED_PROJECTS; + } + + @Override + public Integer processResultSet(ResultSet resultSet) throws SQLException { + while (resultSet.next()) { + return resultSet.getInt("COUNT(*)"); + } + throw new IllegalStateException("Count always returns results"); + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/LockGuard.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/LockGuard.java new file mode 100644 index 0000000000..73c8d253c1 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/LockGuard.java @@ -0,0 +1,10 @@ +package uk.ac.ic.wlgitbridge.bridge.lock; + +/** + * Created by winston on 24/08/2016. + */ +public interface LockGuard extends AutoCloseable { + + void close(); + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/ProjectLock.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/ProjectLock.java index a8157e86b6..624004946f 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/ProjectLock.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/ProjectLock.java @@ -4,9 +4,17 @@ package uk.ac.ic.wlgitbridge.bridge.lock; * Created by winston on 20/08/2016. */ public interface ProjectLock { + void lockAll(); void lockForProject(String projectName); void unlockForProject(String projectName); + + /* RAII hahaha */ + default LockGuard lockGuard(String projectName) { + lockForProject(projectName); + return () -> unlockForProject(projectName); + } + } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java index a12dbf2ec2..bc9bca2027 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java @@ -58,9 +58,12 @@ public class FSRepoStore implements RepoStore { } @Override - public InputStream bzip2Project(String projectName) throws IOException { + public InputStream bzip2Project( + String projectName, + long[] sizePtr + ) throws IOException { Preconditions.checkArgument(Project.isValidProjectName(projectName)); - return Tar.bz2.zip(getDotGitForProject(projectName)); + return Tar.bz2.zip(getDotGitForProject(projectName), sizePtr); } @Override diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java index 5701f5aa09..ed328079a4 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java @@ -25,7 +25,16 @@ public interface RepoStore { * IOException if the project doesn't exist. The returned stream is a copy * of the original .git directory, which must be deleted using remove(). */ - InputStream bzip2Project(String projectName) throws IOException; + InputStream bzip2Project( + String projectName, + long[] sizePtr + ) throws IOException; + + default InputStream bzip2Project( + String projectName + ) throws IOException { + return bzip2Project(projectName, null); + } void remove(String projectName) throws IOException; @@ -36,6 +45,9 @@ public interface RepoStore { * @param projectName the name of the project, e.g. abc123 * @param dataStream the data stream containing the bzipped contents. */ - void unbzip2Project(String projectName, InputStream dataStream) throws IOException; + void unbzip2Project( + String projectName, + InputStream dataStream + ) throws IOException; } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java deleted file mode 100644 index 313696af9c..0000000000 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java +++ /dev/null @@ -1,14 +0,0 @@ -package uk.ac.ic.wlgitbridge.bridge.swap; - -import java.time.Duration; - -/** - * Created by winston on 20/08/2016. - */ -public interface SwapJob { - - void start(Duration interval); - - void stop(); - -} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobConfig.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobConfig.java deleted file mode 100644 index 5a6d37302a..0000000000 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobConfig.java +++ /dev/null @@ -1,33 +0,0 @@ -package uk.ac.ic.wlgitbridge.bridge.swap; - -/** - * Created by winston on 23/08/2016. - */ -public class SwapJobConfig { - - public static final SwapJobConfig DEFAULT = - new SwapJobConfig(1, 1, 2); - - private final int minProjects; - private final long lowGiB; - private final long highGiB; - - public SwapJobConfig(int minProjects, long lowGiB, long highGiB) { - this.minProjects = minProjects; - this.lowGiB = lowGiB; - this.highGiB = highGiB; - } - - public int getMinProjects() { - return minProjects; - } - - public long getLowGiB() { - return lowGiB; - } - - public long getHighGiB() { - return highGiB; - } - -} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java deleted file mode 100644 index f7b179cb4c..0000000000 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java +++ /dev/null @@ -1,73 +0,0 @@ -package uk.ac.ic.wlgitbridge.bridge.swap; - -import uk.ac.ic.wlgitbridge.bridge.db.DBStore; -import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock; -import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; -import uk.ac.ic.wlgitbridge.util.Log; - -import java.time.Duration; -import java.util.Timer; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Created by winston on 20/08/2016. - */ -public class SwapJobImpl implements SwapJob { - - private final int minProjects; - private final long lowGiB; - private final long highGiB; - - private final ProjectLock lock; - private final RepoStore repoStore; - private final SwapStore swapStore; - private final DBStore dbStore; - - private final Timer timer; - - final AtomicInteger swaps; - - public SwapJobImpl( - SwapJobConfig cfg, - ProjectLock lock, - RepoStore repoStore, - DBStore dbStore, - SwapStore swapStore - ) { - minProjects = cfg.getMinProjects(); - lowGiB = cfg.getLowGiB(); - highGiB = cfg.getHighGiB(); - this.lock = lock; - this.repoStore = repoStore; - this.swapStore = swapStore; - this.dbStore = dbStore; - timer = new Timer(); - swaps = new AtomicInteger(0); - } - - @Override - public void start(Duration interval) { - timer.scheduleAtFixedRate( - uk.ac.ic.wlgitbridge.util.Timer.makeTimerTask(this::doSwap), - 0, - interval.toMillis() - ); - } - - @Override - public void stop() { - timer.cancel(); - } - - private void doSwap() { - Log.info("Running {}th swap", swaps.getAndIncrement()); - while (repoStore.totalSize() > lowGiB) { - doEvict(); - } - } - - private void doEvict() { - dbStore.getOldestUnswappedProject(); - } - -} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java deleted file mode 100644 index dc82c35706..0000000000 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java +++ /dev/null @@ -1,21 +0,0 @@ -package uk.ac.ic.wlgitbridge.bridge.swap; - -import java.io.IOException; -import java.io.InputStream; - -/** - * Created by winston on 20/08/2016. - */ -public interface SwapStore { - - void upload( - String projectName, - InputStream uploadStream, - long contentLength - ) throws IOException; - - InputStream openDownloadStream(String projectName); - - void remove(String projectName); - -} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/NoopSwapJob.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/NoopSwapJob.java new file mode 100644 index 0000000000..0825e80dc4 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/NoopSwapJob.java @@ -0,0 +1,30 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.job; + +import java.io.IOException; + +/** + * Created by winston on 24/08/2016. + */ +public class NoopSwapJob implements SwapJob { + + @Override + public void start() { + + } + + @Override + public void stop() { + + } + + @Override + public void evict(String projName) throws IOException { + + } + + @Override + public void restore(String projName) throws IOException { + + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java new file mode 100644 index 0000000000..1147488ff7 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java @@ -0,0 +1,42 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.job; + +import uk.ac.ic.wlgitbridge.bridge.db.DBStore; +import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock; +import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; +import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore; + +import java.io.IOException; +import java.util.Optional; + +/** + * Created by winston on 20/08/2016. + */ +public interface SwapJob { + + static SwapJob fromConfig( + Optional cfg, + ProjectLock lock, + RepoStore repoStore, + DBStore dbStore, + SwapStore swapStore + ) { + if (cfg.isPresent()) { + return new SwapJobImpl( + cfg.get(), + lock, + repoStore, + dbStore, + swapStore + ); + } + return new NoopSwapJob(); + } + + void start(); + + void stop(); + + void evict(String projName) throws IOException; + + void restore(String projName) throws IOException; +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java new file mode 100644 index 0000000000..dfacc0001e --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java @@ -0,0 +1,44 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.job; + +/** + * Created by winston on 23/08/2016. + */ +public class SwapJobConfig { + + public static final SwapJobConfig DEFAULT = + new SwapJobConfig(1, 1, 2, 3600000); + + private final int minProjects; + private final int lowGiB; + private final int highGiB; + private final long intervalMillis; + + public SwapJobConfig( + int minProjects, + int lowGiB, + int highGiB, + long intervalMillis + ) { + this.minProjects = minProjects; + this.lowGiB = lowGiB; + this.highGiB = highGiB; + this.intervalMillis = intervalMillis; + } + + public int getMinProjects() { + return minProjects; + } + + public int getLowGiB() { + return lowGiB; + } + + public int getHighGiB() { + return highGiB; + } + + public long getIntervalMillis() { + return intervalMillis; + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java new file mode 100644 index 0000000000..9556412162 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java @@ -0,0 +1,163 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.job; + +import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import uk.ac.ic.wlgitbridge.bridge.db.DBStore; +import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard; +import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock; +import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; +import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore; +import uk.ac.ic.wlgitbridge.util.Log; + +import java.io.IOException; +import java.io.InputStream; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.Timer; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Created by winston on 20/08/2016. + */ +public class SwapJobImpl implements SwapJob { + + private static final long GiB = (1l << 30); + + int minProjects; + long lowWatermarkBytes; + long highWatermarkBytes; + Duration interval; + + private final ProjectLock lock; + private final RepoStore repoStore; + private final DBStore dbStore; + private final SwapStore swapStore; + + private final Timer timer; + + final AtomicInteger swaps; + + public SwapJobImpl( + SwapJobConfig cfg, + ProjectLock lock, + RepoStore repoStore, + DBStore dbStore, + SwapStore swapStore + ) { + this( + cfg.getMinProjects(), + GiB * cfg.getLowGiB(), + GiB * cfg.getHighGiB(), + Duration.ofMillis(cfg.getIntervalMillis()), + lock, + repoStore, + dbStore, + swapStore + ); + } + + SwapJobImpl( + int minProjects, + long lowWatermarkBytes, + long highWatermarkBytes, + Duration interval, + ProjectLock lock, + RepoStore repoStore, + DBStore dbStore, + SwapStore swapStore + ) { + this.minProjects = minProjects; + this.lowWatermarkBytes = lowWatermarkBytes; + this.highWatermarkBytes = highWatermarkBytes; + this.interval = interval; + this.lock = lock; + this.repoStore = repoStore; + this.dbStore = dbStore; + this.swapStore = swapStore; + timer = new Timer(); + swaps = new AtomicInteger(0); + } + + @Override + public void start() { + timer.scheduleAtFixedRate( + uk.ac.ic.wlgitbridge.util.Timer.makeTimerTask(this::doSwap), + 0, + interval.toMillis() + ); + } + + @Override + public void stop() { + timer.cancel(); + } + + private void doSwap() { + Log.info("Running swap number {}", swaps.get() + 1); + long totalSize = repoStore.totalSize(); + Log.info("Size is {}/{} (high)", totalSize, highWatermarkBytes); + if (totalSize < highWatermarkBytes) { + Log.info("No need to swap."); + swaps.incrementAndGet(); + return; + } + int numProjects = dbStore.getNumProjects(); + while ( + (totalSize = repoStore.totalSize()) > lowWatermarkBytes && + (numProjects = dbStore.getNumUnswappedProjects()) > minProjects + ) { + try { + evict(dbStore.getOldestUnswappedProject()); + } catch (IOException e) { + Log.warn("Exception while swapping, giving up", e); + } + } + if (totalSize > lowWatermarkBytes) { + Log.warn( + "Finished swapping, but total size is still too high." + ); + } + Log.info( + "Size: {}/{} (low), " + + "{} (high), " + + "projects on disk: {}/{}, " + + "min projects on disk: {}", + totalSize, + lowWatermarkBytes, + highWatermarkBytes, + numProjects, + dbStore.getNumProjects(), + minProjects + ); + swaps.incrementAndGet(); + } + + @Override + public void evict(String projName) throws IOException { + Preconditions.checkNotNull(projName); + Log.info("Evicting project: {}", projName); + try (LockGuard __ = lock.lockGuard(projName)) { + long[] sizePtr = new long[1]; + InputStream bzipped = repoStore.bzip2Project(projName, sizePtr); + swapStore.upload(projName, bzipped, sizePtr[0]); + dbStore.setLastAccessedTime(projName, null); + repoStore.remove(projName); + } + Log.info("Evicted project: {}", projName); + } + + @Override + public void restore(String projName) throws IOException { + try (LockGuard __ = lock.lockGuard(projName)) { + repoStore.unbzip2Project( + projName, + swapStore.openDownloadStream(projName) + ); + dbStore.setLastAccessedTime( + projName, + Timestamp.valueOf(LocalDateTime.now()) + ); + } + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/InMemorySwapStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStore.java similarity index 95% rename from services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/InMemorySwapStore.java rename to services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStore.java index 05a919bb26..d4c5a11136 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/InMemorySwapStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStore.java @@ -1,4 +1,4 @@ -package uk.ac.ic.wlgitbridge.bridge.swap; +package uk.ac.ic.wlgitbridge.bridge.swap.store; import org.apache.commons.io.IOUtils; diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/NoopSwapStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/NoopSwapStore.java new file mode 100644 index 0000000000..72d3c38e12 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/NoopSwapStore.java @@ -0,0 +1,35 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.store; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * Created by winston on 24/08/2016. + */ +public class NoopSwapStore implements SwapStore { + + public NoopSwapStore(SwapStoreConfig config) { + + } + + @Override + public void upload( + String projectName, + InputStream uploadStream, + long contentLength + ) throws IOException { + + } + + @Override + public InputStream openDownloadStream(String projectName) { + return new ByteArrayInputStream(new byte[0]); + } + + @Override + public void remove(String projectName) { + + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStore.java similarity index 86% rename from services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStore.java rename to services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStore.java index 639e25779d..ed1e379a7b 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStore.java @@ -1,4 +1,4 @@ -package uk.ac.ic.wlgitbridge.bridge.swap; +package uk.ac.ic.wlgitbridge.bridge.swap.store; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.s3.AmazonS3; @@ -16,7 +16,15 @@ public class S3SwapStore implements SwapStore { private final String bucketName; - public S3SwapStore( + public S3SwapStore(SwapStoreConfig cfg) { + this( + cfg.getAwsAccessKey(), + cfg.getAwsSecret(), + cfg.getS3BucketName() + ); + } + + S3SwapStore( String accessKey, String secret, String bucketName diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStore.java new file mode 100644 index 0000000000..1f582230bb --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStore.java @@ -0,0 +1,43 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.store; + +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +/** + * Created by winston on 20/08/2016. + */ +public interface SwapStore { + + Map> swapStores = + new HashMap>() { + + { + put("noop", NoopSwapStore::new); + put("s3", S3SwapStore::new); + } + + }; + + static SwapStore fromConfig( + Optional cfg + ) { + SwapStoreConfig cfg_ = cfg.orElse(SwapStoreConfig.NOOP); + String type = cfg_.getType(); + return swapStores.get(type).apply(cfg_); + } + + void upload( + String projectName, + InputStream uploadStream, + long contentLength + ) throws IOException; + + InputStream openDownloadStream(String projectName); + + void remove(String projectName); + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStoreConfig.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStoreConfig.java new file mode 100644 index 0000000000..de10c5d506 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStoreConfig.java @@ -0,0 +1,63 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.store; + +/** + * Created by winston on 24/08/2016. + */ +public class SwapStoreConfig { + + public static final SwapStoreConfig NOOP = new SwapStoreConfig( + "noop", + null, + null, + null + ); + + private String type; + private String awsAccessKey; + private String awsSecret; + private String s3BucketName; + + public SwapStoreConfig() {} + + public SwapStoreConfig( + String awsAccessKey, + String awsSecret, + String s3BucketName + ) { + this( + "s3", + awsAccessKey, + awsSecret, + s3BucketName + ); + } + + SwapStoreConfig( + String type, + String awsAccessKey, + String awsSecret, + String s3BucketName + ) { + this.type = type; + this.awsAccessKey = awsAccessKey; + this.awsSecret = awsSecret; + this.s3BucketName = s3BucketName; + } + + public String getType() { + return type; + } + + public String getAwsAccessKey() { + return awsAccessKey; + } + + public String getAwsSecret() { + return awsSecret; + } + + public String getS3BucketName() { + return s3BucketName; + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java index f62847be6e..a1990fe66b 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java @@ -13,8 +13,7 @@ import uk.ac.ic.wlgitbridge.bridge.db.DBStore; import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore; import uk.ac.ic.wlgitbridge.bridge.repo.FSRepoStore; import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; -import uk.ac.ic.wlgitbridge.bridge.swap.SwapJobConfig; -import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore; +import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore; import uk.ac.ic.wlgitbridge.git.exception.InvalidRootDirectoryPathException; import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet; import uk.ac.ic.wlgitbridge.snapshot.base.SnapshotAPIRequest; @@ -25,7 +24,6 @@ import javax.servlet.DispatcherType; import javax.servlet.Filter; import javax.servlet.ServletException; import java.io.File; -import java.io.InputStream; import java.net.BindException; import java.nio.file.Paths; import java.util.EnumSet; @@ -47,7 +45,9 @@ public class GitBridgeServer { private String rootGitDirectoryPath; private String apiBaseURL; - public GitBridgeServer(Config config) throws ServletException, InvalidRootDirectoryPathException { + public GitBridgeServer( + Config config + ) throws ServletException, InvalidRootDirectoryPathException { org.eclipse.jetty.util.log.Log.setLog(new NullLogger()); this.port = config.getPort(); this.rootGitDirectoryPath = config.getRootGitDirectory(); @@ -57,31 +57,19 @@ public class GitBridgeServer { repoStore.getRootDirectory().getAbsolutePath() ).resolve(".wlgb").resolve("wlgb.db").toFile() ); - SwapStore swapStore = new SwapStore() { - @Override - public void upload(String projectName, InputStream uploadStream, long contentLength) { - - } - - @Override - public InputStream openDownloadStream(String projectName) { - return null; - } - - @Override - public void remove(String projectName) { - - } - }; + SwapStore swapStore = SwapStore.fromConfig(config.getSwapStore()); bridgeAPI = Bridge.make( repoStore, dbStore, swapStore, - SwapJobConfig.DEFAULT + config.getSwapJob() ); jettyServer = new Server(port); configureJettyServer(config); - SnapshotAPIRequest.setBasicAuth(config.getUsername(), config.getPassword()); + SnapshotAPIRequest.setBasicAuth( + config.getUsername(), + config.getPassword() + ); apiBaseURL = config.getAPIBaseURL(); SnapshotAPIRequest.setBaseURL(apiBaseURL); Util.setServiceName(config.getServiceName()); @@ -115,7 +103,9 @@ public class GitBridgeServer { } } - private void configureJettyServer(Config config) throws ServletException, InvalidRootDirectoryPathException { + private void configureJettyServer( + Config config + ) throws ServletException, InvalidRootDirectoryPathException { HandlerCollection handlers = new HandlerList(); handlers.addHandler(initApiHandler()); handlers.addHandler(initGitHandler(config)); @@ -135,16 +125,28 @@ public class GitBridgeServer { return api; } - private Handler initGitHandler(Config config) throws ServletException, InvalidRootDirectoryPathException { - final ServletContextHandler servletContextHandler = new ServletContextHandler(ServletContextHandler.SESSIONS); + private Handler initGitHandler( + Config config + ) throws ServletException, InvalidRootDirectoryPathException { + final ServletContextHandler servletContextHandler = + new ServletContextHandler(ServletContextHandler.SESSIONS); if (config.isUsingOauth2()) { Filter filter = new Oauth2Filter(config.getOauth2()); - servletContextHandler.addFilter(new FilterHolder(filter), "/*", EnumSet.of(DispatcherType.REQUEST)); + servletContextHandler.addFilter( + new FilterHolder(filter), + "/*", + EnumSet.of(DispatcherType.REQUEST) + ); } servletContextHandler.setContextPath("/"); servletContextHandler.addServlet( new ServletHolder( - new WLGitServlet(servletContextHandler, bridgeAPI, rootGitDirectoryPath)), + new WLGitServlet( + servletContextHandler, + bridgeAPI, + rootGitDirectoryPath + ) + ), "/*" ); return servletContextHandler; @@ -152,7 +154,9 @@ public class GitBridgeServer { private Handler initResourceHandler() { ResourceHandler resourceHandler = new FileHandler(bridgeAPI); - resourceHandler.setResourceBase(new File(rootGitDirectoryPath, ".wlgb/atts").getAbsolutePath()); + resourceHandler.setResourceBase( + new File(rootGitDirectoryPath, ".wlgb/atts").getAbsolutePath() + ); return resourceHandler; } } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java index 1326aedc5c..7ed57368de 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java @@ -23,11 +23,21 @@ public class Tar { public static InputStream zip( File fileOrDir + ) throws IOException { + return zip(fileOrDir, null); + } + + public static InputStream zip( + File fileOrDir, + long[] sizePtr ) throws IOException { ByteArrayOutputStream target = new ByteArrayOutputStream(); try (OutputStream bzip2 = new BZip2CompressorOutputStream(target)) { tarTo(fileOrDir, bzip2); } + if (sizePtr != null) { + sizePtr[0] = target.size(); + } return target.toInputStream(); } @@ -60,7 +70,6 @@ public class Tar { Paths.get(fileOrDir.getParentFile().getAbsolutePath()), fileOrDir ); - tout.close(); } } diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java index dbd6c17283..ebe3b31fc0 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java @@ -7,10 +7,8 @@ import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock; import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache; import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI; -import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob; -import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore; - -import java.time.Duration; +import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob; +import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -52,9 +50,9 @@ public class BridgeTest { @Test public void shutdownStopsSwapJob() { - bridge.startSwapJob(Duration.ofSeconds(1)); + bridge.startSwapJob(); bridge.doShutdown(); - verify(swapJob).start(Duration.ofSeconds(1)); + verify(swapJob).start(); verify(swapJob).stop(); } diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java index 64ab6d2ffa..0fcbf545df 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java @@ -132,4 +132,19 @@ public class SqliteDBStoreTest { assertEquals("older", dbStore.getOldestUnswappedProject()); } + @Test + public void testGetNumUnswappedProjects() { + dbStore.setLatestVersionForProject("asdf", 1); + dbStore.setLastAccessedTime( + "asdf", + Timestamp.valueOf(LocalDateTime.now()) + ); + assertEquals(1, dbStore.getNumUnswappedProjects()); + dbStore.setLastAccessedTime( + "asdf", + null + ); + assertEquals(0, dbStore.getNumUnswappedProjects()); + } + } \ No newline at end of file diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteFilesForProjectSQLUpdateTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteFilesForProjectSQLUpdateTest.java index 973dd2ca11..4cf2c21d57 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteFilesForProjectSQLUpdateTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteFilesForProjectSQLUpdateTest.java @@ -7,8 +7,18 @@ public class DeleteFilesForProjectSQLUpdateTest { @Test public void testGetSQL() { - DeleteFilesForProjectSQLUpdate update = new DeleteFilesForProjectSQLUpdate("projname", "path1", "path2"); - assertEquals("DELETE FROM `url_index_store` WHERE `project_name` = ? AND path IN (?, ?);\n", update.getSQL()); + DeleteFilesForProjectSQLUpdate update = + new DeleteFilesForProjectSQLUpdate( + "projname", + "path1", + "path2" + ); + assertEquals( + "DELETE FROM `url_index_store` " + + "WHERE `project_name` = ? " + + "AND path IN (?, ?);\n", + update.getSQL() + ); } } \ No newline at end of file diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest.java index 363237ea07..99d502d04a 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest.java @@ -21,6 +21,20 @@ import static org.junit.Assert.assertEquals; */ public class FSRepoStoreTest { + public static File makeTempRepoDir( + TemporaryFolder tmpFolder, + String name + ) throws IOException { + File tmp = tmpFolder.newFolder(name); + Path rootdir = Paths.get( + "src/test/resources/uk/ac/ic/wlgitbridge/" + + "bridge/repo/FSRepoStoreTest/rootdir" + ); + FileUtils.copyDirectory(rootdir.toFile(), tmp); + Files.renameAll(tmp, "DOTgit", ".git"); + return tmp; + } + private FSRepoStore repoStore; private File original; @@ -28,13 +42,7 @@ public class FSRepoStoreTest { public void setup() throws IOException { TemporaryFolder tmpFolder = new TemporaryFolder(); tmpFolder.create(); - File tmp = tmpFolder.newFolder("repostore"); - Path rootdir = Paths.get( - "src/test/resources/uk/ac/ic/wlgitbridge/" - + "bridge/repo/FSRepoStoreTest/rootdir" - ); - FileUtils.copyDirectory(rootdir.toFile(), tmp); - Files.renameAll(tmp, "DOTgit", ".git"); + File tmp = makeTempRepoDir(tmpFolder, "rootdir"); original = tmpFolder.newFolder("original"); FileUtils.copyDirectory(tmp, original); repoStore = new FSRepoStore(tmp.getAbsolutePath()); diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImplTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImplTest.java deleted file mode 100644 index c8ab595a28..0000000000 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImplTest.java +++ /dev/null @@ -1,58 +0,0 @@ -package uk.ac.ic.wlgitbridge.bridge.swap; - -import org.junit.Before; -import org.junit.Test; -import uk.ac.ic.wlgitbridge.bridge.db.DBStore; -import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock; -import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; - -import java.time.Duration; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; - -/** - * Created by winston on 20/08/2016. - */ -public class SwapJobImplTest { - - private SwapJobImpl swapJob; - - private ProjectLock lock; - private RepoStore repoStore; - private DBStore dbStore; - private SwapStore swapStore; - - @Before - public void setup() { - lock = mock(ProjectLock.class); - repoStore = mock(RepoStore.class); - dbStore = mock(DBStore.class); - swapStore = mock(SwapStore.class); - swapJob = new SwapJobImpl( - SwapJobConfig.DEFAULT, - lock, - repoStore, - dbStore, - swapStore - ); - } - - @Test - public void startingTimerAlwaysCausesASwap() { - assertEquals(0, swapJob.swaps.get()); - swapJob.start(Duration.ofHours(1)); - while (swapJob.swaps.get() <= 0); - assertTrue(swapJob.swaps.get() > 0); - } - - @Test - public void swapsHappenEveryInterval() { - assertEquals(0, swapJob.swaps.get()); - swapJob.start(Duration.ofMillis(1)); - while (swapJob.swaps.get() <= 1); - assertTrue(swapJob.swaps.get() > 1); - } - -} \ No newline at end of file diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java new file mode 100644 index 0000000000..f00651d977 --- /dev/null +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java @@ -0,0 +1,119 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.job; + +import org.junit.Before; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import uk.ac.ic.wlgitbridge.bridge.db.DBStore; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore; +import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock; +import uk.ac.ic.wlgitbridge.bridge.repo.FSRepoStore; +import uk.ac.ic.wlgitbridge.bridge.repo.FSRepoStoreTest; +import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; +import uk.ac.ic.wlgitbridge.bridge.swap.store.InMemorySwapStore; +import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore; +import uk.ac.ic.wlgitbridge.data.ProjectLockImpl; + +import java.io.IOException; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Created by winston on 20/08/2016. + */ +public class SwapJobImplTest { + + private SwapJobImpl swapJob; + + private ProjectLock lock; + private RepoStore repoStore; + private DBStore dbStore; + private SwapStore swapStore; + + @Before + public void setup() throws IOException { + TemporaryFolder tmpFolder = new TemporaryFolder(); + tmpFolder.create(); + lock = new ProjectLockImpl(); + repoStore = new FSRepoStore( + FSRepoStoreTest.makeTempRepoDir( + tmpFolder, + "repostore" + ).getAbsolutePath() + ); + dbStore = new SqliteDBStore(tmpFolder.newFile()); + dbStore.setLatestVersionForProject("proj1", 0); + dbStore.setLatestVersionForProject("proj2", 0); + dbStore.setLastAccessedTime( + "proj1", + Timestamp.valueOf(LocalDateTime.now()) + ); + dbStore.setLastAccessedTime( + "proj2", + Timestamp.valueOf( + LocalDateTime.now().minus(1, ChronoUnit.SECONDS) + ) + ); + swapStore = new InMemorySwapStore(); + swapJob = new SwapJobImpl( + 1, + 15000, + 30000, + Duration.ofMillis(100), + lock, + repoStore, + dbStore, + swapStore + ); + } + + @Test + public void startingTimerAlwaysCausesASwap() { + swapJob.lowWatermarkBytes = 16384; + swapJob.interval = Duration.ofHours(1); + assertEquals(0, swapJob.swaps.get()); + swapJob.start(); + while (swapJob.swaps.get() <= 0); + assertTrue(swapJob.swaps.get() > 0); + } + + @Test + public void swapsHappenEveryInterval() { + swapJob.lowWatermarkBytes = 16384; + assertEquals(0, swapJob.swaps.get()); + swapJob.start(); + while (swapJob.swaps.get() <= 1); + assertTrue(swapJob.swaps.get() > 1); + } + + @Test + public void noProjectsGetSwappedWhenUnderHighWatermark() { + swapJob.highWatermarkBytes = 65536; + assertEquals(2, dbStore.getNumUnswappedProjects()); + swapJob.start(); + while (swapJob.swaps.get() < 1); + assertEquals(2, dbStore.getNumUnswappedProjects()); + } + + @Test + public void correctProjGetSwappedWhenOverHighWatermark( + ) throws IOException { + swapJob.lowWatermarkBytes = 16384; + assertEquals(2, dbStore.getNumUnswappedProjects()); + assertEquals("proj2", dbStore.getOldestUnswappedProject()); + swapJob.start(); + while (swapJob.swaps.get() < 1); + assertEquals(1, dbStore.getNumUnswappedProjects()); + assertEquals("proj1", dbStore.getOldestUnswappedProject()); + swapJob.restore("proj2"); + int numSwaps = swapJob.swaps.get(); + while (swapJob.swaps.get() <= numSwaps); + assertEquals(1, dbStore.getNumUnswappedProjects()); + assertEquals("proj2", dbStore.getOldestUnswappedProject()); + } + +} \ No newline at end of file diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/InMemorySwapStoreTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStoreTest.java similarity index 93% rename from services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/InMemorySwapStoreTest.java rename to services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStoreTest.java index 34553ae287..7e257f2305 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/InMemorySwapStoreTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStoreTest.java @@ -1,9 +1,10 @@ -package uk.ac.ic.wlgitbridge.bridge.swap; +package uk.ac.ic.wlgitbridge.bridge.swap.store; import org.apache.commons.io.IOUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import uk.ac.ic.wlgitbridge.bridge.swap.store.InMemorySwapStore; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -51,7 +52,8 @@ public class InMemorySwapStoreTest { } @Test - public void uploadingForTheSameProjectOverwritesTheFile() throws IOException { + public void uploadingForTheSameProjectOverwritesTheFile( + ) throws IOException { byte[] proj1Contents = "helloproj1".getBytes(); byte[] proj1NewContents = "goodbyeproj1".getBytes(); swapStore.upload( diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStoreTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStoreTest.java similarity index 95% rename from services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStoreTest.java rename to services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStoreTest.java index c5c4720be5..f8a7ec59e1 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStoreTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStoreTest.java @@ -1,4 +1,4 @@ -package uk.ac.ic.wlgitbridge.bridge.swap; +package uk.ac.ic.wlgitbridge.bridge.swap.store; import org.junit.Before; diff --git a/services/git-bridge/src/test/resources/logback-test.xml b/services/git-bridge/src/test/resources/logback-test.xml index 4e1539cf06..455a379541 100644 --- a/services/git-bridge/src/test/resources/logback-test.xml +++ b/services/git-bridge/src/test/resources/logback-test.xml @@ -1,7 +1,7 @@ - - ${java.io.tmpdir}/git-bridge-test.log + + System.err %d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%thread] %logger{0}: %msg%n @@ -12,6 +12,6 @@ - +