agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-05-25 02:00:10 +02:00
Code Issues Packages Projects Releases Wiki Activity

Implementation and tests of GC, GcJob, S3 files

Browse Source
This commit is contained in:
Winston Li
2017-02-17 11:22:11 +00:00
parent 8a8d308365
commit ee61d72e2e
72 changed files with 1668 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java
View File

@@ -4,6 +4,7 @@ import org.junit.Before;
import org.junit.Test;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
import uk.ac.ic.wlgitbridge.bridge.gc.GcJob;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
@@ -20,9 +21,7 @@ import java.util.ArrayDeque;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.*;
/**
* Created by winston on 20/08/2016.
@@ -38,6 +37,7 @@ public class BridgeTest {
private SnapshotAPI snapshotAPI;
private ResourceCache resourceCache;
private SwapJob swapJob;
private GcJob gcJob;
@Before
public void setup() {
@@ -48,23 +48,27 @@ public class BridgeTest {
snapshotAPI = mock(SnapshotAPI.class);
resourceCache = mock(ResourceCache.class);
swapJob = mock(SwapJob.class);
gcJob = mock(GcJob.class);
bridge = new Bridge(
lock,
repoStore,
dbStore,
swapStore,
swapJob,
gcJob,
snapshotAPI,
resourceCache
);
}
@Test
public void shutdownStopsSwapJob() {
bridge.startSwapJob();
bridge.doShutdown();
public void shutdownStopsSwapAndGcJobs() {
bridge.startBackgroundJobs();
verify(swapJob).start();
verify(gcJob).start();
bridge.doShutdown();
verify(swapJob).stop();
verify(gcJob).stop();
}
@Test

107
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/gc/GcJobImplTest.java Normal file
View File

@@ -0,0 +1,107 @@
package uk.ac.ic.wlgitbridge.bridge.gc;
import org.junit.Test;
import org.mockito.stubbing.OngoingStubbing;
import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.data.ProjectLockImpl;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static org.junit.Assert.assertFalse;
import static org.mockito.Mockito.*;
/**
* Created by winston on 16/02/2017.
*/
public class GcJobImplTest {
RepoStore repoStore = mock(RepoStore.class);
ProjectLock locks = new ProjectLockImpl();
GcJobImpl gcJob = new GcJobImpl(repoStore, locks, 5);
@Test
public void addedProjectsAreAllEventuallyGcedOnce() throws Exception {
int numProjects = 5;
/* Make the mocks, make expectations, and keep a reference to them */
final OngoingStubbing<ProjectRepo>[] o = new OngoingStubbing[] {
when(repoStore.getExistingRepo(anyString()))
};
List<ProjectRepo> mockRepos = IntStream.range(
0, numProjects
).mapToObj(i ->
String.valueOf((char) ('a' + i))
).map(proj -> {
gcJob.queueForGc(proj);
ProjectRepo mockRepo = mock(ProjectRepo.class);
o[0] = o[0].thenReturn(mockRepo);
return mockRepo;
}).collect(Collectors.toList());
CompletableFuture<Void> fut = gcJob.waitForRun();
gcJob.start();
fut.join();
for (ProjectRepo mock : mockRepos) {
verify(mock).runGC();
verify(mock).deleteIncomingPacks();
}
/* Nothing should happen on the next run */
when(repoStore.getExistingRepo(anyString())).thenThrow(
new IllegalStateException()
);
gcJob.waitForRun().join();
}
@Test
public void cannotOverlapGcRuns() throws Exception {
CompletableFuture<Void> runningForever = new CompletableFuture<>();
gcJob.onPostGc(() -> {
try {
/* Pretend the GC is taking forever */
runningForever.join();
} catch (Throwable e) {
runningForever.completeExceptionally(e);
}
});
CompletableFuture<Void> fut = gcJob.waitForRun();
gcJob.start();
fut.join();
CompletableFuture<Void> ranAgain = new CompletableFuture<>();
gcJob.onPreGc(() -> ranAgain.complete(null));
/* Should not run again any time soon */
for (int i = 0; i < 50; ++i) {
assertFalse(ranAgain.isDone());
/* The gc interval is 5 ms, so 50 1ms sleeps should be more than
enough without making the test slow */
Thread.sleep(1);
}
assertFalse(runningForever.isCompletedExceptionally());
}
@Test
public void willNotGcProjectUntilItIsUnlocked()
throws InterruptedException, IOException {
ProjectRepo repo = mock(ProjectRepo.class);
when(repoStore.getExistingRepo(anyString())).thenReturn(repo);
gcJob.onPostGc(gcJob::stop);
gcJob.queueForGc("a");
CompletableFuture<Void> fut = gcJob.waitForRun();
try (LockGuard __ = locks.lockGuard("a")) {
gcJob.start();
for (int i = 0; i < 50; ++i) {
assertFalse(fut.isDone());
Thread.sleep(1);
}
}
/* Now that we've released the lock, fut should complete */
fut.join();
}
}

4
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest.java
View File

@@ -47,7 +47,9 @@ public class FSGitRepoStoreTest {
@Test
public void testPurgeNonexistentProjects() {
File toDelete = new File(repoStore.getRootDirectory(), "idontexist");
File toDelete = new File(
repoStore.getRootDirectory(), "idontexist"
);
File wlgb = new File(repoStore.getRootDirectory(), ".wlgb");
assertTrue(toDelete.exists());
assertTrue(wlgb.exists());

74
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepoTest.java
View File

@@ -3,11 +3,13 @@ package uk.ac.ic.wlgitbridge.bridge.repo;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile;
import uk.ac.ic.wlgitbridge.snapshot.servermock.util.FileUtil;
import uk.ac.ic.wlgitbridge.util.Files;
import java.io.File;
@@ -16,8 +18,11 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.function.Supplier;
import static org.junit.Assert.assertEquals;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.lessThan;
import static org.junit.Assert.*;
/**
* Created by winston on 08/10/2016.
@@ -42,17 +47,26 @@ public class GitProjectRepoTest {
FSGitRepoStore repoStore;
GitProjectRepo repo;
GitProjectRepo badGitignore;
GitProjectRepo incoming;
GitProjectRepo withoutIncoming;
@Rule
public TemporaryFolder tmpFolder = new TemporaryFolder();
@Before
public void setup() throws IOException {
TemporaryFolder tmpFolder = new TemporaryFolder();
tmpFolder.create();
rootdir = makeTempRepoDir(tmpFolder, "rootdir");
repoStore = new FSGitRepoStore(rootdir.getAbsolutePath());
repo = new GitProjectRepo("repo");
repo.useExistingRepository(repoStore);
badGitignore = new GitProjectRepo("badgitignore");
badGitignore.useExistingRepository(repoStore);
repo = fromExistingDir("repo");
badGitignore = fromExistingDir("badgitignore");
incoming = fromExistingDir("incoming");
withoutIncoming = fromExistingDir("without_incoming");
}
private GitProjectRepo fromExistingDir(String dir) throws IOException {
GitProjectRepo ret = new GitProjectRepo(dir);
ret.useExistingRepository(repoStore);
return ret;
}
private GitDirectoryContents makeDirContents(
@@ -88,7 +102,7 @@ public class GitProjectRepoTest {
);
repo.commitAndGetMissing(contents);
repo.resetHard();
File dir = repo.getDirectory();
File dir = repo.getDotGitDir();
assertEquals(
new HashSet<String>(Arrays.asList(".git", ".gitignore")),
new HashSet<String>(Arrays.asList(dir.list()))
@@ -120,7 +134,7 @@ public class GitProjectRepoTest {
"file2.txt",
"added.ignored"
)),
new HashSet<String>(Arrays.asList(repo.getDirectory().list()))
new HashSet<String>(Arrays.asList(repo.getDotGitDir().list()))
);
}
@@ -141,4 +155,46 @@ public class GitProjectRepoTest {
badGitignore.commitAndGetMissing(contents);
}
private static long repoSize(ProjectRepo repo) {
return FileUtils.sizeOfDirectory(repo.getProjectDir());
}
@Test
public void runGCReducesTheSizeOfARepoWithGarbage() throws IOException {
long beforeSize = repoSize(repo);
repo.runGC();
long afterSize = repoSize(repo);
assertThat(beforeSize, lessThan(afterSize));
}
@Test
public void runGCDoesNothingOnARepoWithoutGarbage() throws IOException {
repo.runGC();
long beforeSize = repoSize(repo);
repo.runGC();
long afterSize = repoSize(repo);
assertThat(beforeSize, equalTo(afterSize));
}
@Test
public void deleteIncomingPacksDeletesIncomingPacks() throws IOException {
Supplier<Boolean> dirsAreEq = () -> FileUtil.directoryDeepEquals(
incoming.getProjectDir(), withoutIncoming.getProjectDir()
);
assertFalse(dirsAreEq.get());
incoming.deleteIncomingPacks();
assertTrue(dirsAreEq.get());
}
@Test
public void deleteIncomingPacksOnDirWithoutIncomingPacksDoesNothing()
throws IOException {
File actual = withoutIncoming.getProjectDir();
File expected = tmpFolder.newFolder();
FileUtils.copyDirectory(actual, expected);
withoutIncoming.deleteIncomingPacks();
assertTrue(FileUtil.directoryDeepEquals(actual, expected));
}
}

18
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TarTest.java
View File

@@ -35,18 +35,20 @@ public class TarTest {
@Test
public void tarAndUntarProducesTheSameResult() throws IOException {
InputStream tar = Tar.tar(testDir);
Tar.untar(tar, tmpDir);
File untarred = new File(tmpDir, "testdir");
assertTrue(Files.contentsAreEqual(testDir, untarred));
try (InputStream tar = Tar.tar(testDir)) {
Tar.untar(tar, tmpDir);
File untarred = new File(tmpDir, "testdir");
assertTrue(Files.contentsAreEqual(testDir, untarred));
}
}
@Test
public void tarbz2AndUntarbz2ProducesTheSameResult() throws IOException {
InputStream tarbz2 = Tar.bz2.zip(testDir);
Tar.bz2.unzip(tarbz2, tmpDir);
File unzipped = new File(tmpDir, "testdir");
assertTrue(Files.contentsAreEqual(testDir, unzipped));
try (InputStream tarbz2 = Tar.bz2.zip(testDir)) {
Tar.bz2.unzip(tarbz2, tmpDir);
File unzipped = new File(tmpDir, "testdir");
assertTrue(Files.contentsAreEqual(testDir, unzipped));
}
}
}

4
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TimerTest.java → services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TimerUtilsTest.java
View File

@@ -7,12 +7,12 @@ import static org.junit.Assert.assertEquals;
/**
* Created by winston on 23/08/2016.
*/
public class TimerTest {
public class TimerUtilsTest {
@Test
public void testMakeTimerTask() {
int[] iPtr = new int[] { 3 };
Timer.makeTimerTask(() -> iPtr[0] = 5).run();
TimerUtils.makeTimerTask(() -> iPtr[0] = 5).run();
assertEquals(5, iPtr[0]);
}