diff --git a/services/git-bridge/bin/wlgb b/services/git-bridge/bin/wlgb index 44f289b7cb..963a528b38 100755 --- a/services/git-bridge/bin/wlgb +++ b/services/git-bridge/bin/wlgb @@ -56,7 +56,7 @@ do_start() echo "Starting WriteLatex-Git Bridge..." start-stop-daemon --start --quiet --make-pidfile --pidfile $PIDFILE \ --exec /usr/bin/env LANG="C.UTF-8" $DAEMON -- \ - $DAEMON_ARGS >> /var/log/wlgb/out.log 2>> /var/log/err.log & + $DAEMON_ARGS >> /var/log/wlgb/out.log 2>> /var/log/wlgb/err.log & echo "WriteLatex-Git Bridge started." echo "Config file at /etc/wlgb/config.json" echo "Log file at /var/log/wlgb/out.log" diff --git a/services/git-bridge/pom.xml b/services/git-bridge/pom.xml index 6b23bad943..8443da4f9c 100644 --- a/services/git-bridge/pom.xml +++ b/services/git-bridge/pom.xml @@ -49,18 +49,14 @@ junit junit - 4.10 + 4.12 test org.jmock jmock-junit4 2.6.0 - - - org.jmock - jmock-legacy - 2.6.0 + test org.eclipse.jetty.aggregate @@ -122,6 +118,12 @@ commons-lang 2.6 + + org.mock-server + mockserver-netty + 3.10.1 + test + \ No newline at end of file diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/DataStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/DataStore.java index 404f4c50d5..2fdc3673dd 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/DataStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/DataStore.java @@ -12,6 +12,7 @@ import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory; import uk.ac.ic.wlgitbridge.data.filestore.RawFile; import uk.ac.ic.wlgitbridge.data.model.db.PersistentStore; import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException; +import uk.ac.ic.wlgitbridge.data.model.db.SqlitePersistentStore; import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException; import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment; import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException; @@ -33,7 +34,7 @@ public class DataStore { public DataStore(String rootGitDirectoryPath) { rootGitDirectory = initRootGitDirectory(rootGitDirectoryPath); - persistentStore = new PersistentStore(rootGitDirectory); + persistentStore = new SqlitePersistentStore(rootGitDirectory); List excludedFromDeletion = persistentStore.getProjectNames(); excludedFromDeletion.add(".wlgb"); Util.deleteInDirectoryApartFrom(rootGitDirectory, excludedFromDeletion.toArray(new String[] {})); @@ -44,10 +45,12 @@ public class DataStore { public void updateProjectWithName(Credential oauth2, String name, Repository repository) throws IOException, SnapshotPostException, GitAPIException, ForbiddenException { LinkedList snapshots = snapshotFetcher.getSnapshotsForProjectAfterVersion(oauth2, name, persistentStore.getLatestVersionForProject(name)); + + makeCommitsFromSnapshots(name, repository, snapshots); + if (!snapshots.isEmpty()) { persistentStore.setLatestVersionForProject(name, snapshots.getLast().getVersionID()); } - makeCommitsFromSnapshots(name, repository, snapshots); } private void makeCommitsFromSnapshots(String name, Repository repository, List snapshots) throws IOException, GitAPIException { diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcher.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcher.java index 4bc0c277d9..836bc2f42f 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcher.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcher.java @@ -42,9 +42,15 @@ public class ResourceFetcher { if (contents == null) { RawFile rawFile = new RepositoryObjectTreeWalker(repository).getDirectoryContents().getFileTable().get(path); if (rawFile == null) { - throw new IllegalStateException("file was not in the current commit, or the git tree, yet path was not null"); + Util.sout( + "WARNING: " + + "File " + path + " was not in the current commit, or the git tree, yet path was not null. " + + "File url is: " + url + ); + contents = fetch(projectName, url, path); + } else { + contents = rawFile.getContents(); } - contents = rawFile.getContents(); } } return new RepositoryFile(newPath, contents); @@ -83,5 +89,4 @@ public class ResourceFetcher { persistentStore.addURLIndexForProject(projectName, url, path); return contents; } - } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/PersistentStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/PersistentStore.java index 310a75f099..59cf8103c2 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/PersistentStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/PersistentStore.java @@ -1,72 +1,20 @@ package uk.ac.ic.wlgitbridge.data.model.db; -import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLiteWLDatabase; - -import java.io.File; -import java.sql.SQLException; import java.util.List; /** - * Created by Winston on 19/11/14. + * Created by m on 20/11/15. */ -public class PersistentStore { +public interface PersistentStore { + List getProjectNames(); - private final SQLiteWLDatabase database; + void setLatestVersionForProject(String project, int versionID); - public PersistentStore(File rootGitDirectory) { - try { - database = new SQLiteWLDatabase(rootGitDirectory); - } catch (SQLException e) { - throw new RuntimeException(e); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } - } + int getLatestVersionForProject(String project); - public List getProjectNames() { - try { - return database.getProjectNames(); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - public void setLatestVersionForProject(String project, int versionID) { - try { - database.setVersionIDForProject(project, versionID); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } + void addURLIndexForProject(String projectName, String url, String path); - public int getLatestVersionForProject(String project) { - try { - return database.getVersionIDForProjectName(project); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - public void addURLIndexForProject(String projectName, String url, String path) { - try { - database.addURLIndex(projectName, url, path); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - public void deleteFilesForProject(String project, String... files) { - try { - database.deleteFilesForProject(project, files); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - public String getPathForURLInProject(String projectName, String url) { - try { - return database.getPathForURLInProject(projectName, url); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } + void deleteFilesForProject(String project, String... files); + String getPathForURLInProject(String projectName, String url); } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/SqlitePersistentStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/SqlitePersistentStore.java new file mode 100644 index 0000000000..7a56d6111f --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/SqlitePersistentStore.java @@ -0,0 +1,78 @@ +package uk.ac.ic.wlgitbridge.data.model.db; + +import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLiteWLDatabase; + +import java.io.File; +import java.sql.SQLException; +import java.util.List; + +/** + * Created by Winston on 19/11/14. + */ +public class SqlitePersistentStore implements PersistentStore { + + private final SQLiteWLDatabase database; + + public SqlitePersistentStore(File rootGitDirectory) { + try { + database = new SQLiteWLDatabase(rootGitDirectory); + } catch (SQLException e) { + throw new RuntimeException(e); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + @Override + public List getProjectNames() { + try { + return database.getProjectNames(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + @Override + public void setLatestVersionForProject(String project, int versionID) { + try { + database.setVersionIDForProject(project, versionID); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public int getLatestVersionForProject(String project) { + try { + return database.getVersionIDForProjectName(project); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void addURLIndexForProject(String projectName, String url, String path) { + try { + database.addURLIndex(projectName, url, path); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void deleteFilesForProject(String project, String... files) { + try { + database.deleteFilesForProject(project, files); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + @Override + public String getPathForURLInProject(String projectName, String url) { + try { + return database.getPathForURLInProject(projectName, url); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/getdoc/GetDocResult.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/getdoc/GetDocResult.java index 012e0af807..63c864bc3a 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/getdoc/GetDocResult.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/getdoc/GetDocResult.java @@ -67,6 +67,7 @@ public class GetDocResult extends Result { JsonObject jsonObject = json.getAsJsonObject(); if (jsonObject.has("status")) { switch (jsonObject.get("status").getAsInt()) { + case 401: case 403: forbidden = new ForbiddenException(); break; diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcherTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcherTest.java new file mode 100644 index 0000000000..66bd4c87db --- /dev/null +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcherTest.java @@ -0,0 +1,74 @@ +package uk.ac.ic.wlgitbridge.data.model; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.mockserver.model.HttpRequest.request; +import static org.mockserver.model.HttpResponse.response; + +import org.eclipse.jgit.lib.*; +import org.eclipse.jgit.storage.file.FileRepositoryBuilder; +import org.jmock.Expectations; +import org.jmock.Mockery; +import org.hamcrest.TypeSafeMatcher; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockserver.client.server.MockServerClient; +import org.mockserver.junit.MockServerRule; +import uk.ac.ic.wlgitbridge.data.model.db.PersistentStore; + + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Created by m on 20/11/15. + */ +public class ResourceFetcherTest { + @Rule + public MockServerRule mockServerRule = new MockServerRule(this); + + private MockServerClient mockServerClient; + + @Test + public void fetchesFilesThatAreMissingFromUrlStoreCache() throws IOException { + final String testProjectName = "123abc"; + final String testUrl = "http://localhost:" + mockServerRule.getHttpPort() + "/123abc"; + final String oldTestPath = "testPath"; + final String newTestPath = "missingPath"; + + mockServerClient.when( + request() + .withMethod("GET") + .withPath("/123abc") + ) + .respond( + response() + .withStatusCode(200) + .withBody("content") + ); + + final Mockery context = new Mockery(); + final PersistentStore persistentStore = context.mock(PersistentStore.class); + context.checking(new Expectations() {{ + // It should fetch the file once it finds it is missing. + oneOf(persistentStore).getPathForURLInProject(testProjectName, testUrl); + will(returnValue(oldTestPath)); + + // It should update the URL index store once it has fetched; at present, it does not actually change the stored path. + oneOf(persistentStore).addURLIndexForProject(testProjectName, testUrl, oldTestPath); + }}); + + ResourceFetcher resourceFetcher = new ResourceFetcher(persistentStore); + TemporaryFolder repositoryFolder = new TemporaryFolder(); + repositoryFolder.create(); + Repository repository = new FileRepositoryBuilder().setWorkTree(repositoryFolder.getRoot()).build(); + Map fetchedUrls = new HashMap(); + resourceFetcher.get(testProjectName, testUrl, newTestPath, repository, fetchedUrls); + + // We don't bother caching in this case, at present. + assertEquals(0, fetchedUrls.size()); + } +} \ No newline at end of file