diff --git a/services/git-bridge/pom.xml b/services/git-bridge/pom.xml index 5666ad925b..07db011f45 100644 --- a/services/git-bridge/pom.xml +++ b/services/git-bridge/pom.xml @@ -140,5 +140,29 @@ mockito-core 1.10.19 + + + com.amazonaws + aws-java-sdk + 1.11.28 + + + + org.apache.httpcomponents + httpclient + 4.5.2 + + + + commons-io + commons-io + 2.5 + + + + org.apache.commons + commons-compress + 1.12 + diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java index 294a5df50a..d9b0a57f58 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java @@ -30,6 +30,7 @@ import uk.ac.ic.wlgitbridge.snapshot.push.exception.*; import uk.ac.ic.wlgitbridge.util.Log; import java.io.IOException; +import java.time.Duration; import java.util.*; /** @@ -104,8 +105,8 @@ public class Bridge { Log.info("Bye"); } - public void startSwapJob(int intervalMillis) { - swapJob.start(intervalMillis); + public void startSwapJob(Duration interval) { + swapJob.start(interval); } /* TODO: Remove these when WLBridged is moved into RepoStore */ diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java index 33075444cf..b9dccd1469 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java @@ -1,5 +1,6 @@ package uk.ac.ic.wlgitbridge.bridge.db; +import java.sql.Timestamp; import java.util.List; /** @@ -19,4 +20,13 @@ public interface DBStore { String getPathForURLInProject(String projectName, String url); + String getOldestUnswappedProject(); + + /** + * Sets the last accessed time for the given project name. + * @param projectName the project's name + * @param time the time, or null if the project is to be swapped + */ + void setLastAccessedTime(String projectName, Timestamp time); + } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/SqliteDBStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/SqliteDBStore.java index 4838479b8d..c241d87272 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/SqliteDBStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/SqliteDBStore.java @@ -5,6 +5,7 @@ import uk.ac.ic.wlgitbridge.util.Log; import java.io.File; import java.sql.SQLException; +import java.sql.Timestamp; import java.util.Arrays; import java.util.List; @@ -84,4 +85,14 @@ public class SqliteDBStore implements DBStore { } } + @Override + public String getOldestUnswappedProject() { + throw new UnsupportedOperationException(); + } + + @Override + public void setLastAccessedTime(String projectName, Timestamp time) { + throw new UnsupportedOperationException(); + } + } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java index 8f431fce07..7ab8a95e64 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java @@ -1,12 +1,20 @@ package uk.ac.ic.wlgitbridge.bridge.repo; -import uk.ac.ic.wlgitbridge.util.Util; +import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import org.apache.commons.io.FileUtils; +import uk.ac.ic.wlgitbridge.util.Project; +import uk.ac.ic.wlgitbridge.util.Tar; import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import static uk.ac.ic.wlgitbridge.util.Util.deleteInDirectoryApartFrom; + /** * Created by winston on 20/08/2016. */ @@ -30,6 +38,7 @@ public class FSRepoStore implements RepoStore { return rootDirectory; } + /* TODO: Perhaps we should just delete bad directories on the fly. */ @Override public void purgeNonexistentProjects( Collection existingProjectNames @@ -37,15 +46,63 @@ public class FSRepoStore implements RepoStore { List excludedFromDeletion = new ArrayList<>(existingProjectNames); excludedFromDeletion.add(".wlgb"); - Util.deleteInDirectoryApartFrom( + deleteInDirectoryApartFrom( rootDirectory, excludedFromDeletion.toArray(new String[] {}) ); } + @Override + public long totalSize() { + return FileUtils.sizeOfDirectory(rootDirectory); + } + + @Override + public InputStream bzip2Project(String projectName) throws IOException { + Preconditions.checkArgument(Project.isValidProjectName(projectName)); + return Tar.tar(getDotGitForProject(projectName)); + } + + @Override + public void remove(String projectName) throws IOException { + Preconditions.checkArgument(Project.isValidProjectName(projectName)); + FileUtils.deleteDirectory(new File(rootDirectory, projectName)); + } + + @Override + public void unbzip2Project( + String projectName, + InputStream dataStream + ) throws IOException { + Preconditions.checkArgument(Project.isValidProjectName(projectName)); + Preconditions.checkState(getDirForProject(projectName).mkdirs()); + Tar.untar(dataStream, getDirForProject(projectName)); + } + + private File getDirForProject(String projectName) { + Preconditions.checkArgument(Project.isValidProjectName(projectName)); + return Paths.get( + rootDirectory.getAbsolutePath() + ).resolve( + projectName + ).toFile(); + } + + private File getDotGitForProject(String projectName) { + Preconditions.checkArgument(Project.isValidProjectName(projectName)); + return Paths.get( + rootDirectory.getAbsolutePath() + ).resolve( + projectName + ).resolve( + ".git" + ).toFile(); + } + private File initRootGitDirectory(String rootGitDirectoryPath) { File rootGitDirectory = new File(rootGitDirectoryPath); rootGitDirectory.mkdirs(); + Preconditions.checkArgument(rootGitDirectory.isDirectory()); return rootGitDirectory; } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java index 9d9d6482d7..5701f5aa09 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java @@ -1,8 +1,8 @@ package uk.ac.ic.wlgitbridge.bridge.repo; -import uk.ac.ic.wlgitbridge.bridge.db.DBStore; - import java.io.File; +import java.io.IOException; +import java.io.InputStream; import java.util.Collection; /** @@ -14,9 +14,28 @@ public interface RepoStore { File getRootDirectory(); - void purgeNonexistentProjects( Collection existingProjectNames ); + long totalSize(); + + /* + * Tars and bzip2s the .git directory of the given project. Throws an + * IOException if the project doesn't exist. The returned stream is a copy + * of the original .git directory, which must be deleted using remove(). + */ + InputStream bzip2Project(String projectName) throws IOException; + + void remove(String projectName) throws IOException; + + /** + * Unbzip2s the given data stream into a .git directory for projectName. + * Creates the project directory. + * If projectName already exists, throws an IOException. + * @param projectName the name of the project, e.g. abc123 + * @param dataStream the data stream containing the bzipped contents. + */ + void unbzip2Project(String projectName, InputStream dataStream) throws IOException; + } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStore.java new file mode 100644 index 0000000000..639e25779d --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStore.java @@ -0,0 +1,64 @@ +package uk.ac.ic.wlgitbridge.bridge.swap; + +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.*; + +import java.io.InputStream; + +/** + * Created by winston on 21/08/2016. + */ +public class S3SwapStore implements SwapStore { + + private final AmazonS3 s3; + + private final String bucketName; + + public S3SwapStore( + String accessKey, + String secret, + String bucketName + ) { + s3 = new AmazonS3Client(new BasicAWSCredentials(accessKey, secret)); + this.bucketName = bucketName; + } + + @Override + public void upload( + String projectName, + InputStream uploadStream, + long contentLength + ) { + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentLength(contentLength); + PutObjectRequest put = new PutObjectRequest( + bucketName, + projectName, + uploadStream, + metadata + ); + PutObjectResult res = s3.putObject(put); + } + + @Override + public InputStream openDownloadStream(String projectName) { + GetObjectRequest get = new GetObjectRequest( + bucketName, + projectName + ); + S3Object res = s3.getObject(get); + return res.getObjectContent(); + } + + @Override + public void remove(String projectName) { + DeleteObjectRequest del = new DeleteObjectRequest( + bucketName, + projectName + ); + s3.deleteObject(del); + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java index 04a52e4108..313696af9c 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java @@ -1,11 +1,13 @@ package uk.ac.ic.wlgitbridge.bridge.swap; +import java.time.Duration; + /** * Created by winston on 20/08/2016. */ public interface SwapJob { - void start(int intervalMillis); + void start(Duration interval); void stop(); diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java index ea26962dd5..06209df6d7 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java @@ -3,9 +3,10 @@ package uk.ac.ic.wlgitbridge.bridge.swap; import uk.ac.ic.wlgitbridge.bridge.db.DBStore; import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock; import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; -import uk.ac.ic.wlgitbridge.util.Util; +import java.time.Duration; import java.util.Timer; +import java.util.concurrent.atomic.AtomicInteger; /** * Created by winston on 20/08/2016. @@ -19,10 +20,13 @@ public class SwapJobImpl implements SwapJob { private final Timer timer; + final AtomicInteger swaps; + public SwapJobImpl( ProjectLock lock, RepoStore repoStore, - DBStore dbStore, SwapStore swapStore + DBStore dbStore, + SwapStore swapStore ) { this.lock = lock; @@ -30,14 +34,15 @@ public class SwapJobImpl implements SwapJob { this.swapStore = swapStore; this.dbStore = dbStore; timer = new Timer(); + swaps = new AtomicInteger(0); } @Override - public void start(int intervalMillis) { + public void start(Duration interval) { timer.scheduleAtFixedRate( - Util.makeTimerTask(this::doSwap), + uk.ac.ic.wlgitbridge.util.Timer.makeTimerTask(this::doSwap), 0, - intervalMillis + interval.toMillis() ); } @@ -47,7 +52,7 @@ public class SwapJobImpl implements SwapJob { } private void doSwap() { - throw new UnsupportedOperationException(); + swaps.incrementAndGet(); } } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java index d230de00ad..4394adca11 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java @@ -1,7 +1,16 @@ package uk.ac.ic.wlgitbridge.bridge.swap; +import java.io.InputStream; + /** * Created by winston on 20/08/2016. */ public interface SwapStore { + + void upload(String projectName, InputStream uploadStream, long contentLength); + + InputStream openDownloadStream(String projectName); + + void remove(String projectName); + } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java index 739fa9130f..a2e69e52c9 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java @@ -24,6 +24,7 @@ import javax.servlet.DispatcherType; import javax.servlet.Filter; import javax.servlet.ServletException; import java.io.File; +import java.io.InputStream; import java.net.BindException; import java.util.EnumSet; @@ -50,7 +51,22 @@ public class GitBridgeServer { this.rootGitDirectoryPath = config.getRootGitDirectory(); RepoStore repoStore = new FSRepoStore(rootGitDirectoryPath); DBStore dbStore = new SqliteDBStore(repoStore.getRootDirectory()); - SwapStore swapStore = new SwapStore() {}; + SwapStore swapStore = new SwapStore() { + @Override + public void upload(String projectName, InputStream uploadStream, long contentLength) { + + } + + @Override + public InputStream openDownloadStream(String projectName) { + return null; + } + + @Override + public void remove(String projectName) { + + } + }; bridgeAPI = Bridge.make( repoStore, dbStore, diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Files.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Files.java new file mode 100644 index 0000000000..ef198a9e08 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Files.java @@ -0,0 +1,127 @@ +package uk.ac.ic.wlgitbridge.util; + +import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.TrueFileFilter; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Created by winston on 23/08/2016. + */ +public class Files { + + private Files() {} + + public static boolean contentsAreEqual( + File f0, + File f1 + ) throws IOException { + try { + return uncheckedContentsAreEqual(f0, f1); + } catch (UncheckedIOException e) { + throw e.getCause(); + } + } + + public static void renameAll( + File fileOrDir, + String from, + String to + ) { + if (fileOrDir.isDirectory()) { + File f = doRename(fileOrDir, from, to); + for (File c : f.listFiles()) { + renameAll(c, from, to); + } + } else if (fileOrDir.isFile()) { + doRename(fileOrDir, from, to); + } else { + throw new IllegalArgumentException( + "not a file or dir: " + fileOrDir + ); + } + } + + private static File doRename(File fileOrDir, String from, String to) { + if (!fileOrDir.getName().equals(from)) { + return fileOrDir; + } + File renamed = new File(fileOrDir.getParent(), to); + Preconditions.checkState(fileOrDir.renameTo(renamed)); + return renamed; + } + + private static boolean uncheckedContentsAreEqual(File f0, File f1) throws IOException { + if (f0.equals(f1)) { + return true; + } + if (!f0.isDirectory() || !f1.isDirectory()) { + return !f0.isDirectory() && !f1.isDirectory() && + Arrays.equals( + FileUtils.readFileToByteArray(f0), + FileUtils.readFileToByteArray(f1) + ); + } + Path f0Base = Paths.get(f0.getAbsolutePath()); + Path f1Base = Paths.get(f1.getAbsolutePath()); + Set children0 = getChildren(f0, f0Base); + Set children1 = getChildren(f1, f1Base); + if (children0.size() != children1.size()) { + return false; + } + return children0.stream( + ).allMatch(c0 -> + children1.contains(c0) && childEquals(c0, f0Base, f1Base) + ); + } + + private static Set getChildren(File f0, Path f0Base) { + return FileUtils.listFilesAndDirs( + f0, + TrueFileFilter.TRUE, + TrueFileFilter.TRUE + ).stream( + ).map( + File::getAbsolutePath + ).map( + Paths::get + ).map(p -> + f0Base.relativize(p) + ).filter(p -> + !p.toString().isEmpty() + ).collect( + Collectors.toSet() + ); + } + + private static boolean childEquals( + Path child, + Path f0Base, + Path f1Base + ) throws UncheckedIOException { + File c0 = f0Base.resolve(child).toFile(); + File c1 = f1Base.resolve(child).toFile(); + boolean c0IsDir = c0.isDirectory(); + boolean c1IsDir = c1.isDirectory(); + if (c0IsDir || c1IsDir) { + return c0IsDir && c1IsDir; + } + try { + return c0.isFile() && c1.isFile() && Arrays.equals( + FileUtils.readFileToByteArray(c0), + FileUtils.readFileToByteArray(c1) + ); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Project.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Project.java new file mode 100644 index 0000000000..3c3b4239a1 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Project.java @@ -0,0 +1,13 @@ +package uk.ac.ic.wlgitbridge.util; + +/** + * Created by winston on 23/08/2016. + */ +public class Project { + + public static boolean isValidProjectName(String projectName) { + return projectName != null && !projectName.isEmpty() + && !projectName.startsWith("."); + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java new file mode 100644 index 0000000000..ec8083aa40 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java @@ -0,0 +1,106 @@ +package uk.ac.ic.wlgitbridge.util; + +import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.output.ByteArrayOutputStream; + +import java.io.*; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Created by winston on 23/08/2016. + */ +public class Tar { + + private Tar() {} + + public static InputStream tar(File fileOrDir) throws IOException { + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + TarArchiveOutputStream tout = new TarArchiveOutputStream(bout); + addTarEntry( + tout, + Paths.get(fileOrDir.getParentFile().getAbsolutePath()), + fileOrDir + ); + tout.close(); + return new ByteArrayInputStream(bout.toByteArray()); + } + + private static void addTarEntry( + TarArchiveOutputStream tout, + Path base, + File fileOrDir + ) throws IOException { + if (fileOrDir.isDirectory()) { + addTarDir(tout, base, fileOrDir); + } else if (fileOrDir.isFile()) { + addTarFile(tout, base, fileOrDir); + } else { + throw new IllegalArgumentException( + "invalid file or dir: " + fileOrDir + ); + } + } + + private static void addTarDir( + TarArchiveOutputStream tout, + Path base, + File dir + ) throws IOException { + Preconditions.checkArgument(dir.isDirectory()); + String name = base.relativize( + Paths.get(dir.getAbsolutePath()) + ).toString(); + ArchiveEntry entry = tout.createArchiveEntry(dir, name); + tout.putArchiveEntry(entry); + tout.closeArchiveEntry(); + for (File f : dir.listFiles()) { + addTarEntry(tout, base, f); + } + } + + private static void addTarFile( + TarArchiveOutputStream tout, + Path base, + File file + ) throws IOException { + Preconditions.checkArgument(file.isFile()); + String name = base.relativize( + Paths.get(file.getAbsolutePath()) + ).toString(); + ArchiveEntry entry = tout.createArchiveEntry(file, name); + tout.putArchiveEntry(entry); + tout.write(FileUtils.readFileToByteArray(file)); + tout.closeArchiveEntry(); + } + + public static void untar(InputStream tar, File parentDir) throws IOException { + TarArchiveInputStream tin = new TarArchiveInputStream(tar); + ArchiveEntry e; + while ((e = tin.getNextEntry()) != null) { + File f = new File(parentDir, e.getName()); + f.setLastModified(e.getLastModifiedDate().getTime()); + f.getParentFile().mkdirs(); + if (e.isDirectory()) { + f.mkdir(); + continue; + } + long size = e.getSize(); + Preconditions.checkArgument( + size > 0 && size < Integer.MAX_VALUE, + "file too big: tar should have thrown an IOException" + ); + try (OutputStream out = new FileOutputStream(f)) { + /* TarInputStream pretends each + entry's EOF is the stream's EOF */ + IOUtils.copy(tin, out); + } + } + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Timer.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Timer.java new file mode 100644 index 0000000000..b2dd212707 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Timer.java @@ -0,0 +1,19 @@ +package uk.ac.ic.wlgitbridge.util; + +import java.util.TimerTask; + +/** + * Created by winston on 23/08/2016. + */ +public class Timer { + + public static TimerTask makeTimerTask(Runnable lamb) { + return new TimerTask() { + @Override + public void run() { + lamb.run(); + } + }; + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Util.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Util.java index 3654561f9f..97440a9ff5 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Util.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Util.java @@ -19,15 +19,6 @@ public class Util { private static String POSTBACK_URL; private static final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSS"); - public static TimerTask makeTimerTask(Runnable lamb) { - return new TimerTask() { - @Override - public void run() { - lamb.run(); - } - }; - } - public static String entries(int entries) { if (entries == 1) { return "entry"; diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java index 68236e3735..464e5b7331 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java @@ -10,6 +10,8 @@ import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI; import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob; import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore; +import java.time.Duration; + import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -35,6 +37,7 @@ public class BridgeTest { dbStore = mock(DBStore.class); swapStore = mock(SwapStore.class); snapshotAPI = mock(SnapshotAPI.class); + resourceCache = mock(ResourceCache.class); swapJob = mock(SwapJob.class); bridge = new Bridge( lock, @@ -49,9 +52,9 @@ public class BridgeTest { @Test public void shutdownStopsSwapJob() { - bridge.startSwapJob(1000); + bridge.startSwapJob(Duration.ofSeconds(1)); bridge.doShutdown(); - verify(swapJob).start(1000); + verify(swapJob).start(Duration.ofSeconds(1)); verify(swapJob).stop(); } diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest.java new file mode 100644 index 0000000000..363237ea07 --- /dev/null +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest.java @@ -0,0 +1,75 @@ +package uk.ac.ic.wlgitbridge.bridge.repo; + +import org.apache.commons.io.FileUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import uk.ac.ic.wlgitbridge.util.Files; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; + +/** + * Created by winston on 23/08/2016. + */ +public class FSRepoStoreTest { + + private FSRepoStore repoStore; + private File original; + + @Before + public void setup() throws IOException { + TemporaryFolder tmpFolder = new TemporaryFolder(); + tmpFolder.create(); + File tmp = tmpFolder.newFolder("repostore"); + Path rootdir = Paths.get( + "src/test/resources/uk/ac/ic/wlgitbridge/" + + "bridge/repo/FSRepoStoreTest/rootdir" + ); + FileUtils.copyDirectory(rootdir.toFile(), tmp); + Files.renameAll(tmp, "DOTgit", ".git"); + original = tmpFolder.newFolder("original"); + FileUtils.copyDirectory(tmp, original); + repoStore = new FSRepoStore(tmp.getAbsolutePath()); + } + + @Test + public void testPurgeNonexistentProjects() { + File toDelete = new File(repoStore.getRootDirectory(), "idontexist"); + File wlgb = new File(repoStore.getRootDirectory(), ".wlgb"); + Assert.assertTrue(toDelete.exists()); + Assert.assertTrue(wlgb.exists()); + repoStore.purgeNonexistentProjects(Arrays.asList("proj1", "proj2")); + Assert.assertFalse(toDelete.exists()); + Assert.assertTrue(wlgb.exists()); + } + + @Test + public void testTotalSize() { + assertEquals(31860, repoStore.totalSize()); + } + + @Test + public void zipAndUnzipShouldBeTheSame() throws IOException { + long beforeSize = repoStore.totalSize(); + InputStream zipped = repoStore.bzip2Project("proj1"); + repoStore.remove("proj1"); + Assert.assertTrue(beforeSize > repoStore.totalSize()); + repoStore.unbzip2Project("proj1", zipped); + Assert.assertEquals(beforeSize, repoStore.totalSize()); + Assert.assertTrue( + Files.contentsAreEqual( + original, + repoStore.getRootDirectory() + ) + ); + } + +} \ No newline at end of file diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStoreTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStoreTest.java new file mode 100644 index 0000000000..c5c4720be5 --- /dev/null +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStoreTest.java @@ -0,0 +1,41 @@ +package uk.ac.ic.wlgitbridge.bridge.swap; + +import org.junit.Before; + +/** + * Created by winston on 21/08/2016. + */ +public class S3SwapStoreTest { + + private static final String accessKey = null; + private static final String secret = null; + private static final String bucketName = "com.overleaf.testbucket"; + + private S3SwapStore s3; + + @Before + public void setup() { + if (accessKey == null || secret == null) { + s3 = null; + return; + } + s3 = new S3SwapStore(accessKey, secret, bucketName); + } + +// @Ignore +// @Test +// public void testUploadDownloadDelete() throws Exception { +// assumeNotNull(s3); +// String projName = "abc123"; +// byte[] contents = "hello".getBytes(); +// s3.upload( +// projName, +// new ByteArrayInputStream(contents), +// contents.length +// ); +// InputStream down = s3.openDownloadStream(projName); +// s3.remove(projName); +// assertArrayEquals(contents, IOUtils.toByteArray(down)); +// } + +} \ No newline at end of file diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImplTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImplTest.java new file mode 100644 index 0000000000..7114026d00 --- /dev/null +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImplTest.java @@ -0,0 +1,57 @@ +package uk.ac.ic.wlgitbridge.bridge.swap; + +import org.junit.Before; +import org.junit.Test; +import uk.ac.ic.wlgitbridge.bridge.db.DBStore; +import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock; +import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; + +import java.time.Duration; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * Created by winston on 20/08/2016. + */ +public class SwapJobImplTest { + + private SwapJobImpl swapJob; + + private ProjectLock lock; + private RepoStore repoStore; + private DBStore dbStore; + private SwapStore swapStore; + + @Before + public void setup() { + lock = mock(ProjectLock.class); + repoStore = mock(RepoStore.class); + dbStore = mock(DBStore.class); + swapStore = mock(SwapStore.class); + swapJob = new SwapJobImpl( + lock, + repoStore, + dbStore, + swapStore + ); + } + + @Test + public void startingTimerAlwaysCausesASwap() { + assertEquals(0, swapJob.swaps.get()); + swapJob.start(Duration.ofHours(1)); + while (swapJob.swaps.get() <= 0); + assertTrue(swapJob.swaps.get() > 0); + } + + @Test + public void swapsHappenEveryInterval() { + assertEquals(0, swapJob.swaps.get()); + swapJob.start(Duration.ofMillis(1)); + while (swapJob.swaps.get() <= 1); + assertTrue(swapJob.swaps.get() > 1); + } + +} \ No newline at end of file diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/ProjectTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/ProjectTest.java new file mode 100644 index 0000000000..23d664e4ed --- /dev/null +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/ProjectTest.java @@ -0,0 +1,18 @@ +package uk.ac.ic.wlgitbridge.util; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Created by winston on 23/08/2016. + */ +public class ProjectTest { + + @Test + public void testValidProjectNames() { + Assert.assertFalse(Project.isValidProjectName(null)); + Assert.assertFalse(Project.isValidProjectName("")); + Assert.assertFalse(Project.isValidProjectName(".wlgb")); + } + +} diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TarTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TarTest.java new file mode 100644 index 0000000000..74b6fcf109 --- /dev/null +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TarTest.java @@ -0,0 +1,45 @@ +package uk.ac.ic.wlgitbridge.util; + +import org.apache.commons.io.FileUtils; +import org.junit.Before; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.Assert.assertTrue; + +/** + * Created by winston on 23/08/2016. + */ +public class TarTest { + + private File testDir; + + @Before + public void setup() throws IOException { + TemporaryFolder tmpFolder = new TemporaryFolder(); + tmpFolder.create(); + testDir = tmpFolder.newFolder("testdir"); + Path resdir = Paths.get( + "src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir" + ); + FileUtils.copyDirectory(resdir.toFile(), testDir); + } + + @Test + public void tarAndUntarProducesTheSameResult() throws IOException { + InputStream tar = Tar.tar(testDir); + TemporaryFolder tmpF = new TemporaryFolder(); + tmpF.create(); + File parentDir = tmpF.newFolder(); + Tar.untar(tar, parentDir); + File untarred = new File(parentDir, "testdir"); + assertTrue(Files.contentsAreEqual(testDir, untarred)); + } + +} \ No newline at end of file diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TimerTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TimerTest.java new file mode 100644 index 0000000000..487c7d8bf5 --- /dev/null +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TimerTest.java @@ -0,0 +1,19 @@ +package uk.ac.ic.wlgitbridge.util; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Created by winston on 23/08/2016. + */ +public class TimerTest { + + @Test + public void testMakeTimerTask() { + int[] iPtr = new int[] { 3 }; + Timer.makeTimerTask(() -> iPtr[0] = 5).run(); + assertEquals(5, iPtr[0]); + } + +} diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest/rootdir/proj1 b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest/rootdir/proj1 new file mode 160000 index 0000000000..e5fc0d2678 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest/rootdir/proj1 @@ -0,0 +1 @@ +Subproject commit e5fc0d2678ec7b9bacf0bf514bac035fa371cb6e diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest/rootdir/proj2 b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest/rootdir/proj2 new file mode 160000 index 0000000000..6c12c073e5 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest/rootdir/proj2 @@ -0,0 +1 @@ +Subproject commit 6c12c073e5702530a9d06b83840d62f8a6621764 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/file1 b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/file1 new file mode 100644 index 0000000000..e2129701f1 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/file1 @@ -0,0 +1 @@ +file1 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/file2 b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/file2 new file mode 100644 index 0000000000..6c493ff740 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/file2 @@ -0,0 +1 @@ +file2 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file1 b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file1 new file mode 100644 index 0000000000..5c441e6377 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file1 @@ -0,0 +1 @@ +nest1/file1 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file2 b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file2 new file mode 100644 index 0000000000..2b7361bb7e --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file2 @@ -0,0 +1 @@ +nest1file2 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file3 b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file3 new file mode 100644 index 0000000000..ac69189828 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file3 @@ -0,0 +1 @@ +nest1/file3 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/nest2/file1 b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/nest2/file1 new file mode 100644 index 0000000000..5e92e63c44 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/nest2/file1 @@ -0,0 +1 @@ +nest1/nest2/file1 diff --git a/services/git-bridge/writelatex-git-bridge.iml b/services/git-bridge/writelatex-git-bridge.iml index 87a4714796..f5496dc370 100644 --- a/services/git-bridge/writelatex-git-bridge.iml +++ b/services/git-bridge/writelatex-git-bridge.iml @@ -37,10 +37,6 @@ - - - - @@ -80,9 +76,9 @@ - - - + + + @@ -97,7 +93,6 @@ - @@ -109,5 +104,84 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file