agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-05-23 09:09:36 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #18402 from overleaf/msm-git-bridge-pom

Browse Source 
* [git-bridge] Extracted pom versions to properties

Managing versions as properties is a best practice
with pom files. An advantage in our case is ensuring
dependency groups (as simpleclient) are upgraded
together.

* [git-bridge] Remove '_' as keyword

* [git-bridge] Added formatting with 'fmt-maven-plugin'

* Remove java from .editorconfig

No longer needed as a formatter is now available

* Replace javadoc comments with multiline comments

Replaces "/**" with "/*", which then prevents the formatter
from adding HTML elements to comments

* [git-bridge] Formatted .java files

GitOrigin-RevId: 0997b838cee88c290d824a74444295e26392f26b
This commit is contained in:
Miguel Serrano
2024-06-14 12:42:59 +02:00
committed by Copybot
parent b34be6bea4
commit da5846209f
203 changed files with 9548 additions and 10742 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
services/git-bridge/Makefile
View File

@@ -16,6 +16,15 @@ $(MVN_TARGET): $(shell find src -type f) pom.xml
build: $(MVN_TARGET)
format:
mvn $(MVN_OPTS) com.spotify.fmt:fmt-maven-plugin:check
format_fix:
mvn $(MVN_OPTS) com.spotify.fmt:fmt-maven-plugin:format
test:
mvn $(MVN_OPTS) test

97
services/git-bridge/pom.xml
View File

@@ -8,19 +8,43 @@
<artifactId>writelatex-git-bridge</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<!--<maven.test.skip>true</maven.test.skip>-->
<maven.compiler.plugin.version>3.7.0</maven.compiler.plugin.version>
<maven.surefire.plugin.version>2.12.4</maven.surefire.plugin.version>
<maven.assembly.plugin.version>3.1.0</maven.assembly.plugin.version>
<fmt.plugin.version>2.23</fmt.plugin.version>
<junit.version>4.13.2</junit.version>
<jmock.junit4.version>2.8.4</jmock.junit4.version>
<jetty.servlet.version>9.4.51.v20230217</jetty.servlet.version>
<gson.version>2.9.0</gson.version>
<async.http.client.version>2.12.3</async.http.client.version>
<jgit.version>6.6.1.202309021850-r</jgit.version>
<sqlite.jdbc.version>3.41.2.2</sqlite.jdbc.version>
<joda.time.version>2.9.9</joda.time.version>
<google.oauth.client.version>1.34.1</google.oauth.client.version>
<google.http.client.version>1.23.0</google.http.client.version>
<commons.lang3.version>3.12.0</commons.lang3.version>
<logback.classic.version>1.2.3</logback.classic.version>
<mockserver.version>5.12.0</mockserver.version>
<mockito.version>3.11.1</mockito.version>
<aws.java.sdk.version>1.11.274</aws.java.sdk.version>
<jakarta.xml.bind.api.version>${jaxb.runtime.version}</jakarta.xml.bind.api.version>
<jaxb.runtime.version>2.3.2</jaxb.runtime.version>
<httpclient.version>4.5.14</httpclient.version>
<commons.io.version>2.10.0</commons.io.version>
<commons.compress.version>1.24.0</commons.compress.version>
<simpleclient.version>0.10.0</simpleclient.version>
</properties>
<build>
<plugins>
<!-- https://mvnrepository.com/artifact/org.apache.maven.plugins/maven-compiler-plugin -->
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
<version>${maven.compiler.plugin.version}</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<compilerArgument></compilerArgument>
<source>${java.version}</source>
<target>${java.version}</target>
</configuration>
</plugin>
<!-- Workaround, test loader crashes without this configuration option -->
@@ -28,7 +52,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.12.4</version>
<version>${maven.surefire.plugin.version}</version>
<configuration>
<argLine>-Djdk.net.URLClassPath.disableClassPathURLCheck=true</argLine>
</configuration>
@@ -36,7 +60,7 @@
<!-- https://mvnrepository.com/artifact/org.apache.maven.plugins/maven-assembly-plugin -->
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.1.0</version>
<version>${maven.assembly.plugin.version}</version>
<executions>
<execution>
<phase>package</phase>
@@ -56,6 +80,11 @@
</descriptorRefs>
</configuration>
</plugin>
<plugin>
<groupId>com.spotify.fmt</groupId>
<artifactId>fmt-maven-plugin</artifactId>
<version>${fmt.plugin.version}</version>
</plugin>
</plugins>
</build>
<dependencies>
@@ -63,169 +92,169 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.2</version>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.jmock/jmock-junit4 -->
<dependency>
<groupId>org.jmock</groupId>
<artifactId>jmock-junit4</artifactId>
<version>2.8.4</version>
<version>${jmock.junit4.version}</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.eclipse.jetty/jetty-servlet -->
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
<version>9.4.51.v20230217</version>
<version>${jetty.servlet.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.eclipse.jetty/jetty-server -->
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>9.4.51.v20230217</version>
<version>${jetty.servlet.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.9.0</version>
<version>${gson.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.asynchttpclient/async-http-client -->
<dependency>
<groupId>org.asynchttpclient</groupId>
<artifactId>async-http-client</artifactId>
<version>2.12.3</version>
<version>${async.http.client.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.eclipse.jgit/org.eclipse.jgit -->
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
<version>6.6.1.202309021850-r</version>
<version>${jgit.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.eclipse.jgit/org.eclipse.jgit.http.server -->
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit.http.server</artifactId>
<version>6.6.1.202309021850-r</version>
<version>${jgit.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.xerial/sqlite-jdbc -->
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
<version>3.41.2.2</version>
<version>${sqlite.jdbc.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/joda-time/joda-time -->
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.9.9</version>
<version>${joda.time.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.oauth-client/google-oauth-client -->
<dependency>
<groupId>com.google.oauth-client</groupId>
<artifactId>google-oauth-client</artifactId>
<version>1.34.1</version>
<version>${google.oauth.client.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.http-client/google-http-client -->
<dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client</artifactId>
<version>1.23.0</version>
<version>${google.http.client.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.http-client/google-http-client-gson -->
<dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client-gson</artifactId>
<version>1.23.0</version>
<version>${google.http.client.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
<version>${commons.lang3.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/ch.qos.logback/logback-classic -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.2.3</version>
<version>${logback.classic.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.mock-server/mockserver-netty -->
<dependency>
<groupId>org.mock-server</groupId>
<artifactId>mockserver-netty</artifactId>
<version>5.12.0</version>
<version>${mockserver.version}</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.mock-server/mockserver-junit-rule -->
<dependency>
<groupId>org.mock-server</groupId>
<artifactId>mockserver-junit-rule</artifactId>
<version>5.12.0</version>
<version>${mockserver.version}</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.mockito/mockito-core -->
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>3.11.1</version>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.11.274</version>
<version>${aws.java.sdk.version}</version>
</dependency>
<!-- API, java.xml.bind module -->
<dependency>
<groupId>jakarta.xml.bind</groupId>
<artifactId>jakarta.xml.bind-api</artifactId>
<version>2.3.2</version>
<version>${jakarta.xml.bind.api.version}</version>
</dependency>
<!-- Runtime, com.sun.xml.bind module -->
<dependency>
<groupId>org.glassfish.jaxb</groupId>
<artifactId>jaxb-runtime</artifactId>
<version>2.3.2</version>
<version>${jaxb.runtime.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.14</version>
<version>${httpclient.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.10.0</version>
<version>${commons.io.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-compress -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.24.0</version>
<version>${commons.compress.version}</version>
</dependency>
<!-- prometheus metrics -->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient</artifactId>
<version>0.10.0</version>
<version>${simpleclient.version}</version>
</dependency>
<!-- Hotspot JVM metrics -->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_hotspot</artifactId>
<version>0.10.0</version>
<version>${simpleclient.version}</version>
</dependency>
<!-- Expose metrics via a servlet -->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_servlet</artifactId>
<version>0.10.0</version>
<version>${simpleclient.version}</version>
</dependency>
</dependencies>
</project>

34
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/Main.java
View File

@@ -1,17 +1,14 @@
package uk.ac.ic.wlgitbridge;
import java.util.Arrays;
import uk.ac.ic.wlgitbridge.application.GitBridgeApp;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.server.GitBridgeServer;
import uk.ac.ic.wlgitbridge.util.Log;
import java.util.Arrays;
/**
/*
* Created by Winston on 01/11/14.
*/
/**
/*
* This is the entry point into the Git Bridge.
*
* It is responsible for creating the {@link GitBridgeApp} and then running it.
@@ -29,21 +26,14 @@ import java.util.Arrays;
*/
public class Main {
public static void main(String[] args) {
Log.info(
"Git Bridge started with args: "
+ Arrays.toString(args)
);
try {
new GitBridgeApp(args).run();
} catch (Throwable t) {
/* So that we get a timestamp */
Log.error(
"Fatal exception thrown to top level, exiting: ",
t
);
System.exit(1);
}
public static void main(String[] args) {
Log.info("Git Bridge started with args: " + Arrays.toString(args));
try {
new GitBridgeApp(args).run();
} catch (Throwable t) {
/* So that we get a timestamp */
Log.error("Fatal exception thrown to top level, exiting: ", t);
System.exit(1);
}
}
}

143
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/GitBridgeApp.java
View File

@@ -1,102 +1,93 @@
package uk.ac.ic.wlgitbridge.application;
import java.io.IOException;
import javax.servlet.ServletException;
import uk.ac.ic.wlgitbridge.application.config.Config;
import uk.ac.ic.wlgitbridge.application.exception.ArgsException;
import uk.ac.ic.wlgitbridge.application.exception.ConfigFileException;
import uk.ac.ic.wlgitbridge.server.GitBridgeServer;
import uk.ac.ic.wlgitbridge.util.Log;
import javax.servlet.ServletException;
import java.io.IOException;
/**
/*
* Created by Winston on 02/11/14.
*/
/**
/*
* Class that represents the application. Parses arguments and gives them to the
* server, or dies with a usage message.
*/
public class GitBridgeApp implements Runnable {
public static final int EXIT_CODE_FAILED = 1;
private static final String USAGE_MESSAGE =
"usage: writelatex-git-bridge [config_file]";
public static final int EXIT_CODE_FAILED = 1;
private static final String USAGE_MESSAGE = "usage: writelatex-git-bridge [config_file]";
private String configFilePath;
Config config;
private GitBridgeServer server;
private String configFilePath;
Config config;
private GitBridgeServer server;
/**
* Constructs an instance of the WriteLatex-Git Bridge application.
* @param args args from main, which should be in the format [config_file]
*/
public GitBridgeApp(String[] args) {
try {
parseArguments(args);
loadConfigFile();
Log.info("Config loaded: {}", config.getSanitisedString());
} catch (ArgsException e) {
printUsage();
System.exit(EXIT_CODE_FAILED);
} catch (ConfigFileException e) {
Log.error(
"The property for " +
e.getMissingMember() +
" is invalid. Check your config file."
);
System.exit(EXIT_CODE_FAILED);
} catch (IOException e) {
Log.error("Invalid config file. Check the file path.");
System.exit(EXIT_CODE_FAILED);
}
try {
server = new GitBridgeServer(config);
} catch (ServletException e) {
Log.error(
"Servlet exception when instantiating GitBridgeServer",
e
);
}
/*
* Constructs an instance of the WriteLatex-Git Bridge application.
* @param args args from main, which should be in the format [config_file]
*/
public GitBridgeApp(String[] args) {
try {
parseArguments(args);
loadConfigFile();
Log.info("Config loaded: {}", config.getSanitisedString());
} catch (ArgsException e) {
printUsage();
System.exit(EXIT_CODE_FAILED);
} catch (ConfigFileException e) {
Log.error(
"The property for " + e.getMissingMember() + " is invalid. Check your config file.");
System.exit(EXIT_CODE_FAILED);
} catch (IOException e) {
Log.error("Invalid config file. Check the file path.");
System.exit(EXIT_CODE_FAILED);
}
/**
* Starts the server with the port number and root directory path given in
* the command-line arguments.
*/
@Override
public void run() {
server.start();
try {
server = new GitBridgeServer(config);
} catch (ServletException e) {
Log.error("Servlet exception when instantiating GitBridgeServer", e);
}
}
public void stop() {
server.stop();
/*
* Starts the server with the port number and root directory path given in
* the command-line arguments.
*/
@Override
public void run() {
server.start();
}
public void stop() {
server.stop();
}
/* Helper methods */
private void parseArguments(String[] args) throws ArgsException {
checkArgumentsLength(args);
parseConfigFilePath(args);
}
private void checkArgumentsLength(String[] args) throws ArgsException {
if (args.length < 1) {
throw new ArgsException();
}
}
/* Helper methods */
private void parseConfigFilePath(String[] args) throws ArgsException {
configFilePath = args[0];
}
private void parseArguments(String[] args) throws ArgsException {
checkArgumentsLength(args);
parseConfigFilePath(args);
}
private void checkArgumentsLength(String[] args) throws ArgsException {
if (args.length < 1) {
throw new ArgsException();
}
}
private void parseConfigFilePath(String[] args) throws ArgsException {
configFilePath = args[0];
}
private void loadConfigFile() throws ConfigFileException, IOException {
Log.info("Loading config file at path: " + configFilePath);
config = new Config(configFilePath);
}
private void printUsage() {
System.err.println(USAGE_MESSAGE);
}
private void loadConfigFile() throws ConfigFileException, IOException {
Log.info("Loading config file at path: " + configFilePath);
config = new Config(configFilePath);
}
private void printUsage() {
System.err.println(USAGE_MESSAGE);
}
}

364
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java
View File

@@ -3,6 +3,11 @@ package uk.ac.ic.wlgitbridge.application.config;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.Optional;
import javax.annotation.Nullable;
import uk.ac.ic.wlgitbridge.application.exception.ConfigFileException;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStoreConfig;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig;
@@ -10,211 +15,182 @@ import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStoreConfig;
import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
import uk.ac.ic.wlgitbridge.util.Instance;
import javax.annotation.Nullable;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.Optional;
/**
/*
* Created by Winston on 05/12/14.
*/
public class Config implements JSONSource {
static Config asSanitised(Config config) {
return new Config(
config.port,
config.bindIp,
config.idleTimeout,
config.rootGitDirectory,
config.apiBaseURL,
config.postbackURL,
config.serviceName,
Oauth2.asSanitised(config.oauth2),
config.userPasswordEnabled,
config.repoStore,
SwapStoreConfig.sanitisedCopy(config.swapStore),
config.swapJob,
config.sqliteHeapLimitBytes
);
static Config asSanitised(Config config) {
return new Config(
config.port,
config.bindIp,
config.idleTimeout,
config.rootGitDirectory,
config.apiBaseURL,
config.postbackURL,
config.serviceName,
Oauth2.asSanitised(config.oauth2),
config.userPasswordEnabled,
config.repoStore,
SwapStoreConfig.sanitisedCopy(config.swapStore),
config.swapJob,
config.sqliteHeapLimitBytes);
}
private int port;
private String bindIp;
private int idleTimeout;
private String rootGitDirectory;
private String apiBaseURL;
private String postbackURL;
private String serviceName;
@Nullable private Oauth2 oauth2;
private boolean userPasswordEnabled;
@Nullable private RepoStoreConfig repoStore;
@Nullable private SwapStoreConfig swapStore;
@Nullable private SwapJobConfig swapJob;
private int sqliteHeapLimitBytes = 0;
public Config(String configFilePath) throws ConfigFileException, IOException {
this(new FileReader(configFilePath));
}
Config(Reader reader) {
fromJSON(new Gson().fromJson(reader, JsonElement.class));
}
public Config(
int port,
String bindIp,
int idleTimeout,
String rootGitDirectory,
String apiBaseURL,
String postbackURL,
String serviceName,
Oauth2 oauth2,
boolean userPasswordEnabled,
RepoStoreConfig repoStore,
SwapStoreConfig swapStore,
SwapJobConfig swapJob,
int sqliteHeapLimitBytes) {
this.port = port;
this.bindIp = bindIp;
this.idleTimeout = idleTimeout;
this.rootGitDirectory = rootGitDirectory;
this.apiBaseURL = apiBaseURL;
this.postbackURL = postbackURL;
this.serviceName = serviceName;
this.oauth2 = oauth2;
this.userPasswordEnabled = userPasswordEnabled;
this.repoStore = repoStore;
this.swapStore = swapStore;
this.swapJob = swapJob;
this.sqliteHeapLimitBytes = sqliteHeapLimitBytes;
}
@Override
public void fromJSON(JsonElement json) {
JsonObject configObject = json.getAsJsonObject();
port = getElement(configObject, "port").getAsInt();
bindIp = getElement(configObject, "bindIp").getAsString();
idleTimeout = getElement(configObject, "idleTimeout").getAsInt();
rootGitDirectory = getElement(configObject, "rootGitDirectory").getAsString();
String apiBaseURL = getElement(configObject, "apiBaseUrl").getAsString();
if (!apiBaseURL.endsWith("/")) {
apiBaseURL += "/";
}
private int port;
private String bindIp;
private int idleTimeout;
private String rootGitDirectory;
private String apiBaseURL;
private String postbackURL;
private String serviceName;
@Nullable
private Oauth2 oauth2;
private boolean userPasswordEnabled;
@Nullable
private RepoStoreConfig repoStore;
@Nullable
private SwapStoreConfig swapStore;
@Nullable
private SwapJobConfig swapJob;
private int sqliteHeapLimitBytes = 0;
public Config(
String configFilePath
) throws ConfigFileException,
IOException {
this(new FileReader(configFilePath));
this.apiBaseURL = apiBaseURL;
serviceName = getElement(configObject, "serviceName").getAsString();
postbackURL = getElement(configObject, "postbackBaseUrl").getAsString();
if (!postbackURL.endsWith("/")) {
postbackURL += "/";
}
Config(Reader reader) {
fromJSON(new Gson().fromJson(reader, JsonElement.class));
oauth2 = new Gson().fromJson(configObject.get("oauth2"), Oauth2.class);
userPasswordEnabled = getOptionalString(configObject, "userPasswordEnabled").equals("true");
repoStore = new Gson().fromJson(configObject.get("repoStore"), RepoStoreConfig.class);
swapStore = new Gson().fromJson(configObject.get("swapStore"), SwapStoreConfig.class);
swapJob = new Gson().fromJson(configObject.get("swapJob"), SwapJobConfig.class);
if (configObject.has("sqliteHeapLimitBytes")) {
sqliteHeapLimitBytes = getElement(configObject, "sqliteHeapLimitBytes").getAsInt();
}
}
public Config(
int port,
String bindIp,
int idleTimeout,
String rootGitDirectory,
String apiBaseURL,
String postbackURL,
String serviceName,
Oauth2 oauth2,
boolean userPasswordEnabled,
RepoStoreConfig repoStore,
SwapStoreConfig swapStore,
SwapJobConfig swapJob,
int sqliteHeapLimitBytes
) {
this.port = port;
this.bindIp = bindIp;
this.idleTimeout = idleTimeout;
this.rootGitDirectory = rootGitDirectory;
this.apiBaseURL = apiBaseURL;
this.postbackURL = postbackURL;
this.serviceName = serviceName;
this.oauth2 = oauth2;
this.userPasswordEnabled = userPasswordEnabled;
this.repoStore = repoStore;
this.swapStore = swapStore;
this.swapJob = swapJob;
this.sqliteHeapLimitBytes = sqliteHeapLimitBytes;
public String getSanitisedString() {
return Instance.prettyGson.toJson(Config.asSanitised(this));
}
public int getPort() {
return port;
}
public String getBindIp() {
return bindIp;
}
public int getIdleTimeout() {
return idleTimeout;
}
public String getRootGitDirectory() {
return rootGitDirectory;
}
public int getSqliteHeapLimitBytes() {
return this.sqliteHeapLimitBytes;
}
public String getAPIBaseURL() {
return apiBaseURL;
}
public String getServiceName() {
return serviceName;
}
public String getPostbackURL() {
return postbackURL;
}
public boolean isUsingOauth2() {
return oauth2 != null;
}
public boolean isUserPasswordEnabled() {
return userPasswordEnabled;
}
public Oauth2 getOauth2() {
if (!isUsingOauth2()) {
throw new AssertionError("Getting oauth2 when not using it");
}
return oauth2;
}
@Override
public void fromJSON(JsonElement json) {
JsonObject configObject = json.getAsJsonObject();
port = getElement(configObject, "port").getAsInt();
bindIp = getElement(configObject, "bindIp").getAsString();
idleTimeout = getElement(configObject, "idleTimeout").getAsInt();
rootGitDirectory = getElement(
configObject,
"rootGitDirectory"
).getAsString();
String apiBaseURL = getElement(
configObject,
"apiBaseUrl"
).getAsString();
if (!apiBaseURL.endsWith("/")) {
apiBaseURL += "/";
}
this.apiBaseURL = apiBaseURL;
serviceName = getElement(configObject, "serviceName").getAsString();
postbackURL = getElement(configObject, "postbackBaseUrl").getAsString();
if (!postbackURL.endsWith("/")) {
postbackURL += "/";
}
oauth2 = new Gson().fromJson(configObject.get("oauth2"), Oauth2.class);
userPasswordEnabled = getOptionalString(configObject, "userPasswordEnabled").equals("true");
repoStore = new Gson().fromJson(
configObject.get("repoStore"), RepoStoreConfig.class);
swapStore = new Gson().fromJson(
configObject.get("swapStore"),
SwapStoreConfig.class
);
swapJob = new Gson().fromJson(
configObject.get("swapJob"),
SwapJobConfig.class
);
if (configObject.has("sqliteHeapLimitBytes")) {
sqliteHeapLimitBytes = getElement(configObject, "sqliteHeapLimitBytes").getAsInt();
}
public Optional<RepoStoreConfig> getRepoStore() {
return Optional.ofNullable(repoStore);
}
public Optional<SwapStoreConfig> getSwapStore() {
return Optional.ofNullable(swapStore);
}
public Optional<SwapJobConfig> getSwapJob() {
return Optional.ofNullable(swapJob);
}
private JsonElement getElement(JsonObject configObject, String name) {
JsonElement element = configObject.get(name);
if (element == null) {
throw new RuntimeException(new ConfigFileException(name));
}
return element;
}
public String getSanitisedString() {
return Instance.prettyGson.toJson(Config.asSanitised(this));
private String getOptionalString(JsonObject configObject, String name) {
JsonElement element = configObject.get(name);
if (element == null || !element.isJsonPrimitive()) {
return "";
}
public int getPort() {
return port;
}
public String getBindIp() {
return bindIp;
}
public int getIdleTimeout() {
return idleTimeout;
}
public String getRootGitDirectory() {
return rootGitDirectory;
}
public int getSqliteHeapLimitBytes() {
return this.sqliteHeapLimitBytes;
}
public String getAPIBaseURL() {
return apiBaseURL;
}
public String getServiceName() {
return serviceName;
}
public String getPostbackURL() {
return postbackURL;
}
public boolean isUsingOauth2() {
return oauth2 != null;
}
public boolean isUserPasswordEnabled() {
return userPasswordEnabled;
}
public Oauth2 getOauth2() {
if (!isUsingOauth2()) {
throw new AssertionError("Getting oauth2 when not using it");
}
return oauth2;
}
public Optional<RepoStoreConfig> getRepoStore() {
return Optional.ofNullable(repoStore);
}
public Optional<SwapStoreConfig> getSwapStore() {
return Optional.ofNullable(swapStore);
}
public Optional<SwapJobConfig> getSwapJob() {
return Optional.ofNullable(swapJob);
}
private JsonElement getElement(JsonObject configObject, String name) {
JsonElement element = configObject.get(name);
if (element == null) {
throw new RuntimeException(new ConfigFileException(name));
}
return element;
}
private String getOptionalString(JsonObject configObject, String name) {
JsonElement element = configObject.get(name);
if (element == null || !element.isJsonPrimitive()) {
return "";
}
return element.getAsString();
}
return element.getAsString();
}
}

51
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Oauth2.java
View File

@@ -1,42 +1,33 @@
package uk.ac.ic.wlgitbridge.application.config;
/**
/*
* Created by winston on 25/10/15.
*/
public class Oauth2 {
private final String oauth2ClientID;
private final String oauth2ClientSecret;
private final String oauth2Server;
private final String oauth2ClientID;
private final String oauth2ClientSecret;
private final String oauth2Server;
public Oauth2(
String oauth2ClientID,
String oauth2ClientSecret,
String oauth2Server
) {
this.oauth2ClientID = oauth2ClientID;
this.oauth2ClientSecret = oauth2ClientSecret;
this.oauth2Server = oauth2Server;
}
public Oauth2(String oauth2ClientID, String oauth2ClientSecret, String oauth2Server) {
this.oauth2ClientID = oauth2ClientID;
this.oauth2ClientSecret = oauth2ClientSecret;
this.oauth2Server = oauth2Server;
}
public String getOauth2ClientID() {
return oauth2ClientID;
}
public String getOauth2ClientID() {
return oauth2ClientID;
}
public String getOauth2ClientSecret() {
return oauth2ClientSecret;
}
public String getOauth2ClientSecret() {
return oauth2ClientSecret;
}
public String getOauth2Server() {
return oauth2Server;
}
public static Oauth2 asSanitised(Oauth2 oauth2) {
return new Oauth2(
"<oauth2ClientID>",
"<oauth2ClientSecret>",
oauth2.oauth2Server
);
}
public String getOauth2Server() {
return oauth2Server;
}
public static Oauth2 asSanitised(Oauth2 oauth2) {
return new Oauth2("<oauth2ClientID>", "<oauth2ClientSecret>", oauth2.oauth2Server);
}
}

2
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/exception/ArgsException.java
View File

@@ -1,6 +1,6 @@
package uk.ac.ic.wlgitbridge.application.exception;
/**
/*
* Created by Winston on 03/11/14.
*/
public class ArgsException extends Exception {}

17
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/exception/ConfigFileException.java
View File

@@ -1,18 +1,17 @@
package uk.ac.ic.wlgitbridge.application.exception;
/**
/*
* Created by Winston on 05/12/14.
*/
public class ConfigFileException extends Exception {
private final String missingMember;
private final String missingMember;
public ConfigFileException(String missingMember) {
this.missingMember = missingMember;
}
public String getMissingMember() {
return missingMember;
}
public ConfigFileException(String missingMember) {
this.missingMember = missingMember;
}
public String getMissingMember() {
return missingMember;
}
}

99
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/jetty/NullLogger.java
View File

@@ -2,84 +2,59 @@ package uk.ac.ic.wlgitbridge.application.jetty;
import org.eclipse.jetty.util.log.Logger;
/**
/*
* Created by Winston on 03/11/14.
*/
public class NullLogger implements Logger {
@Override
public String getName() {
return "null_logger";
}
@Override
public String getName() {
return "null_logger";
}
@Override
public void warn(String s, Object... objects) {
@Override
public void warn(String s, Object... objects) {}
}
@Override
public void warn(Throwable throwable) {}
@Override
public void warn(Throwable throwable) {
@Override
public void warn(String s, Throwable throwable) {}
}
@Override
public void info(String s, Object... objects) {}
@Override
public void warn(String s, Throwable throwable) {
@Override
public void info(Throwable throwable) {}
}
@Override
public void info(String s, Throwable throwable) {}
@Override
public void info(String s, Object... objects) {
@Override
public boolean isDebugEnabled() {
return false;
}
}
@Override
public void setDebugEnabled(boolean b) {}
@Override
public void info(Throwable throwable) {
@Override
public void debug(String s, Object... objects) {}
}
@Override
public void debug(String s, long l) {}
@Override
public void info(String s, Throwable throwable) {
@Override
public void debug(Throwable throwable) {}
}
@Override
public void debug(String s, Throwable throwable) {}
@Override
public boolean isDebugEnabled() {
return false;
}
@Override
public void setDebugEnabled(boolean b) {
}
@Override
public void debug(String s, Object... objects) {
}
@Override
public void debug(String s, long l) {
}
@Override
public void debug(Throwable throwable) {
}
@Override
public void debug(String s, Throwable throwable) {
}
@Override
public Logger getLogger(String s) {
return this;
}
@Override
public void ignore(Throwable throwable) {
}
@Override
public Logger getLogger(String s) {
return this;
}
@Override
public void ignore(Throwable throwable) {}
}

1248
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java
View File

File diff suppressed because it is too large Load Diff

21
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBInitException.java
View File

@@ -1,20 +1,19 @@
package uk.ac.ic.wlgitbridge.bridge.db;
/**
/*
* Created by winston on 23/08/2016.
*/
public class DBInitException extends RuntimeException {
public DBInitException(String message) {
super(message);
}
public DBInitException(String message) {
super(message);
}
public DBInitException(String message, Throwable cause) {
super(message, cause);
}
public DBInitException(Throwable cause) {
super(cause);
}
public DBInitException(String message, Throwable cause) {
super(message, cause);
}
public DBInitException(Throwable cause) {
super(cause);
}
}

48
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java
View File

@@ -3,46 +3,46 @@ package uk.ac.ic.wlgitbridge.bridge.db;
import java.sql.Timestamp;
import java.util.List;
/**
/*
* Created by winston on 20/08/2016.
*/
public interface DBStore {
int getNumProjects();
int getNumProjects();
List<String> getProjectNames();
List<String> getProjectNames();
void setLatestVersionForProject(String project, int versionID);
void setLatestVersionForProject(String project, int versionID);
int getLatestVersionForProject(String project);
int getLatestVersionForProject(String project);
void addURLIndexForProject(String projectName, String url, String path);
void addURLIndexForProject(String projectName, String url, String path);
void deleteFilesForProject(String project, String... files);
void deleteFilesForProject(String project, String... files);
String getPathForURLInProject(String projectName, String url);
String getPathForURLInProject(String projectName, String url);
String getOldestUnswappedProject();
String getOldestUnswappedProject();
void swap(String projectName, String compressionMethod);
void swap(String projectName, String compressionMethod);
void restore(String projectName);
void restore(String projectName);
String getSwapCompression(String projectName);
String getSwapCompression(String projectName);
int getNumUnswappedProjects();
int getNumUnswappedProjects();
ProjectState getProjectState(String projectName);
ProjectState getProjectState(String projectName);
/**
* Sets the last accessed time for the given project name.
* @param projectName the project's name
* @param time the time, or null if the project is to be swapped
*/
void setLastAccessedTime(String projectName, Timestamp time);
/*
* Sets the last accessed time for the given project name.
* @param projectName the project's name
* @param time the time, or null if the project is to be swapped
*/
void setLastAccessedTime(String projectName, Timestamp time);
/**
* Delete the metadata associated with the given project.
*/
void deleteProject(String projectName);
/*
* Delete the metadata associated with the given project.
*/
void deleteProject(String projectName);
}

10
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/ProjectState.java
View File

@@ -1,12 +1,10 @@
package uk.ac.ic.wlgitbridge.bridge.db;
/**
/*
* Created by winston on 24/08/2016.
*/
public enum ProjectState {
NOT_PRESENT,
PRESENT,
SWAPPED
NOT_PRESENT,
PRESENT,
SWAPPED
}

104
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/noop/NoopDbStore.java
View File

@@ -1,78 +1,70 @@
package uk.ac.ic.wlgitbridge.bridge.db.noop;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
import java.sql.Timestamp;
import java.util.List;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
public class NoopDbStore implements DBStore {
@Override
public int getNumProjects() {
return 0;
}
@Override
public int getNumProjects() {
return 0;
}
@Override
public List<String> getProjectNames() {
return null;
}
@Override
public List<String> getProjectNames() {
return null;
}
@Override
public void setLatestVersionForProject(String project, int versionID) {
@Override
public void setLatestVersionForProject(String project, int versionID) {}
}
@Override
public int getLatestVersionForProject(String project) {
return 0;
}
@Override
public int getLatestVersionForProject(String project) {
return 0;
}
@Override
public void addURLIndexForProject(String projectName, String url, String path) {}
@Override
public void addURLIndexForProject(String projectName, String url, String path) {
@Override
public void deleteFilesForProject(String project, String... files) {}
}
@Override
public String getPathForURLInProject(String projectName, String url) {
return null;
}
@Override
public void deleteFilesForProject(String project, String... files) {
@Override
public String getOldestUnswappedProject() {
return null;
}
}
@Override
public int getNumUnswappedProjects() {
return 0;
}
@Override
public String getPathForURLInProject(String projectName, String url) {
return null;
}
@Override
public ProjectState getProjectState(String projectName) {
return null;
}
@Override
public String getOldestUnswappedProject() {
return null;
}
@Override
public void setLastAccessedTime(String projectName, Timestamp time) {}
@Override
public int getNumUnswappedProjects() {
return 0;
}
@Override
public void swap(String projectName, String compressionMethod) {}
@Override
public ProjectState getProjectState(String projectName) {
return null;
}
@Override
public void restore(String projectName) {}
@Override
public void setLastAccessedTime(String projectName, Timestamp time) {
}
@Override
public String getSwapCompression(String projectName) {
return null;
}
@Override
public void swap(String projectName, String compressionMethod) {}
@Override
public void restore(String projectName) {}
@Override
public String getSwapCompression(String projectName) {
return null;
}
@Override
public void deleteProject(String projectName) {}
@Override
public void deleteProject(String projectName) {}
}

5
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SQLQuery.java
View File

@@ -3,11 +3,10 @@ package uk.ac.ic.wlgitbridge.bridge.db.sqlite;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
/*
* Created by Winston on 20/11/14.
*/
public interface SQLQuery<T> extends SQLUpdate {
public T processResultSet(ResultSet resultSet) throws SQLException;
public T processResultSet(ResultSet resultSet) throws SQLException;
}

10
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SQLUpdate.java
View File

@@ -3,16 +3,12 @@ package uk.ac.ic.wlgitbridge.bridge.db.sqlite;
import java.sql.PreparedStatement;
import java.sql.SQLException;
/**
/*
* Created by Winston on 20/11/14.
*/
public interface SQLUpdate {
String getSQL();
default void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
}
String getSQL();
default void addParametersToStatement(PreparedStatement statement) throws SQLException {}
}

407
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java
View File

@@ -1,6 +1,10 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite;
import com.google.common.base.Preconditions;
import java.io.File;
import java.sql.*;
import java.util.List;
import java.util.stream.Stream;
import uk.ac.ic.wlgitbridge.bridge.db.DBInitException;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
@@ -10,225 +14,214 @@ import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create.*;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete.*;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert.*;
import java.io.File;
import java.sql.*;
import java.util.List;
import java.util.stream.Stream;
/**
/*
* Created by Winston on 17/11/14.
*/
public class SqliteDBStore implements DBStore {
private final Connection connection;
private int heapLimitBytes = 0;
private final Connection connection;
private int heapLimitBytes = 0;
public SqliteDBStore(File dbFile) {
this(dbFile, 0);
public SqliteDBStore(File dbFile) {
this(dbFile, 0);
}
public SqliteDBStore(File dbFile, int heapLimitBytes) {
this.heapLimitBytes = heapLimitBytes;
try {
connection = openConnectionTo(dbFile);
createTables();
} catch (Throwable t) {
throw new DBInitException(t);
}
}
@Override
public int getNumProjects() {
return query(new GetNumProjects());
}
@Override
public List<String> getProjectNames() {
return query(new GetProjectNamesSQLQuery());
}
@Override
public void setLatestVersionForProject(String projectName, int versionID) {
update(new SetProjectSQLUpdate(projectName, versionID));
}
@Override
public int getLatestVersionForProject(String projectName) {
return query(new GetLatestVersionForProjectSQLQuery(projectName));
}
@Override
public void addURLIndexForProject(String projectName, String url, String path) {
update(new AddURLIndexSQLUpdate(projectName, url, path));
}
@Override
public void deleteFilesForProject(String projectName, String... paths) {
update(new DeleteFilesForProjectSQLUpdate(projectName, paths));
}
@Override
public String getPathForURLInProject(String projectName, String url) {
return query(new GetPathForURLInProjectSQLQuery(projectName, url));
}
@Override
public String getOldestUnswappedProject() {
return query(new GetOldestProjectName());
}
@Override
public int getNumUnswappedProjects() {
return query(new GetNumUnswappedProjects());
}
@Override
public ProjectState getProjectState(String projectName) {
return query(new GetProjectState(projectName));
}
@Override
public void setLastAccessedTime(String projectName, Timestamp lastAccessed) {
update(new SetProjectLastAccessedTime(projectName, lastAccessed));
}
@Override
public void swap(String projectName, String compressionMethod) {
update(new UpdateSwap(projectName, compressionMethod));
}
@Override
public void restore(String projectName) {
update(new UpdateRestore(projectName));
}
@Override
public String getSwapCompression(String projectName) {
return query(new GetSwapCompression(projectName));
}
@Override
public void deleteProject(String projectName) {
update(new DeleteAllFilesInProjectSQLUpdate(projectName));
update(new DeleteProjectSQLUpdate(projectName));
}
private Connection openConnectionTo(File dbFile) {
File parentDir = dbFile.getParentFile();
if (!parentDir.exists() && !parentDir.mkdirs()) {
throw new DBInitException(
parentDir.getAbsolutePath()
+ " directory didn't exist, "
+ "and unable to create. Check your permissions.");
}
try {
Class.forName("org.sqlite.JDBC");
} catch (ClassNotFoundException e) {
throw new DBInitException(e);
}
try {
return DriverManager.getConnection("jdbc:sqlite:" + dbFile.getAbsolutePath());
} catch (SQLException e) {
throw new DBInitException("Unable to connect to DB", e);
}
}
private void createTables() {
/* Migrations */
/* We need to eat exceptions from here */
try {
doUpdate(new SetSoftHeapLimitPragma(this.heapLimitBytes));
} catch (SQLException ignore) {
}
try {
doUpdate(new ProjectsAddLastAccessed());
} catch (SQLException ignore) {
}
try {
doUpdate(new ProjectsAddSwapTime());
} catch (SQLException ignore) {
}
try {
doUpdate(new ProjectsAddRestoreTime());
} catch (SQLException ignore) {
}
try {
doUpdate(new ProjectsAddSwapCompression());
} catch (SQLException ignore) {
}
public SqliteDBStore(File dbFile, int heapLimitBytes) {
this.heapLimitBytes = heapLimitBytes;
try {
connection = openConnectionTo(dbFile);
createTables();
} catch (Throwable t) {
throw new DBInitException(t);
}
/* Create tables (if they don't exist) */
Stream.of(
new CreateProjectsTableSQLUpdate(),
new CreateProjectsIndexLastAccessed(),
new CreateURLIndexStoreSQLUpdate(),
new CreateIndexURLIndexStore())
.forEach(this::update);
/* In the case of needing to change the schema, we need to check that
migrations didn't just fail */
Preconditions.checkState(query(new LastAccessedColumnExists()));
Preconditions.checkState(query(new SwapTimeColumnExists()));
Preconditions.checkState(query(new RestoreTimeColumnExists()));
Preconditions.checkState(query(new SwapCompressionColumnExists()));
}
private void update(SQLUpdate update) {
try {
doUpdate(update);
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
@Override
public int getNumProjects() {
return query(new GetNumProjects());
private <T> T query(SQLQuery<T> query) {
try {
return doQuery(query);
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
@Override
public List<String> getProjectNames() {
return query(new GetProjectNamesSQLQuery());
private void doUpdate(SQLUpdate update) throws SQLException {
PreparedStatement statement = null;
try {
statement = connection.prepareStatement(update.getSQL());
update.addParametersToStatement(statement);
statement.executeUpdate();
} catch (SQLException e) {
throw e;
} finally {
try {
statement.close();
} catch (Throwable t) {
throw new SQLException(t);
}
}
}
@Override
public void setLatestVersionForProject(
String projectName,
int versionID
) {
update(new SetProjectSQLUpdate(projectName, versionID));
private <T> T doQuery(SQLQuery<T> query) throws SQLException {
PreparedStatement statement = null;
ResultSet results = null;
try {
statement = connection.prepareStatement(query.getSQL());
query.addParametersToStatement(statement);
results = statement.executeQuery();
return query.processResultSet(results);
} catch (SQLException e) {
throw e;
} finally {
if (statement != null) {
statement.close();
}
if (results != null) {
results.close();
}
}
@Override
public int getLatestVersionForProject(
String projectName
) {
return query(new GetLatestVersionForProjectSQLQuery(projectName));
}
@Override
public void addURLIndexForProject(
String projectName,
String url,
String path
) {
update(new AddURLIndexSQLUpdate(projectName, url, path));
}
@Override
public void deleteFilesForProject(
String projectName,
String... paths
) {
update(new DeleteFilesForProjectSQLUpdate(projectName, paths));
}
@Override
public String getPathForURLInProject(
String projectName,
String url
) {
return query(new GetPathForURLInProjectSQLQuery(projectName, url));
}
@Override
public String getOldestUnswappedProject() {
return query(new GetOldestProjectName());
}
@Override
public int getNumUnswappedProjects() {
return query(new GetNumUnswappedProjects());
}
@Override
public ProjectState getProjectState(String projectName) {
return query(new GetProjectState(projectName));
}
@Override
public void setLastAccessedTime(
String projectName,
Timestamp lastAccessed
) {
update(new SetProjectLastAccessedTime(projectName, lastAccessed));
}
@Override
public void swap(String projectName, String compressionMethod) {
update(new UpdateSwap(projectName, compressionMethod));
}
@Override
public void restore(String projectName) {
update(new UpdateRestore(projectName));
}
@Override
public String getSwapCompression(String projectName) {
return query(new GetSwapCompression(projectName));
}
@Override
public void deleteProject(String projectName) {
update(new DeleteAllFilesInProjectSQLUpdate(projectName));
update(new DeleteProjectSQLUpdate(projectName));
}
private Connection openConnectionTo(File dbFile) {
File parentDir = dbFile.getParentFile();
if (!parentDir.exists() && !parentDir.mkdirs()) {
throw new DBInitException(
parentDir.getAbsolutePath() + " directory didn't exist, " +
"and unable to create. Check your permissions."
);
}
try {
Class.forName("org.sqlite.JDBC");
} catch (ClassNotFoundException e) {
throw new DBInitException(e);
}
try {
return DriverManager.getConnection(
"jdbc:sqlite:" + dbFile.getAbsolutePath()
);
} catch (SQLException e) {
throw new DBInitException("Unable to connect to DB", e);
}
}
private void createTables() {
/* Migrations */
/* We need to eat exceptions from here */
try { doUpdate(new SetSoftHeapLimitPragma(this.heapLimitBytes)); } catch (SQLException ignore) {}
try { doUpdate(new ProjectsAddLastAccessed()); } catch (SQLException ignore) {}
try { doUpdate(new ProjectsAddSwapTime()); } catch (SQLException ignore) {}
try { doUpdate(new ProjectsAddRestoreTime()); } catch (SQLException ignore) {}
try { doUpdate(new ProjectsAddSwapCompression()); } catch (SQLException ignore) {}
/* Create tables (if they don't exist) */
Stream.of(
new CreateProjectsTableSQLUpdate(),
new CreateProjectsIndexLastAccessed(),
new CreateURLIndexStoreSQLUpdate(),
new CreateIndexURLIndexStore()
).forEach(this::update);
/* In the case of needing to change the schema, we need to check that
migrations didn't just fail */
Preconditions.checkState(query(new LastAccessedColumnExists()));
Preconditions.checkState(query(new SwapTimeColumnExists()));
Preconditions.checkState(query(new RestoreTimeColumnExists()));
Preconditions.checkState(query(new SwapCompressionColumnExists()));
}
private void update(SQLUpdate update) {
try {
doUpdate(update);
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
private <T> T query(SQLQuery<T> query) {
try {
return doQuery(query);
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
private void doUpdate(SQLUpdate update) throws SQLException {
PreparedStatement statement = null;
try {
statement = connection.prepareStatement(update.getSQL());
update.addParametersToStatement(statement);
statement.executeUpdate();
} catch (SQLException e) {
throw e;
} finally {
try {
statement.close();
} catch (Throwable t) {
throw new SQLException(t);
}
}
}
private <T> T doQuery(SQLQuery<T> query) throws SQLException {
PreparedStatement statement = null;
ResultSet results = null;
try {
statement = connection.prepareStatement(query.getSQL());
query.addParametersToStatement(statement);
results = statement.executeQuery();
return query.processResultSet(results);
} catch (SQLException e) {
throw e;
} finally {
if (statement != null) {
statement.close();
}
if (results != null) {
results.close();
}
}
}
}
}

51
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetLatestVersionForProjectSQLQuery.java
View File

@@ -1,43 +1,40 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
/**
/*
* Created by Winston on 20/11/14.
*/
public class GetLatestVersionForProjectSQLQuery implements SQLQuery<Integer> {
private static final String GET_VERSION_IDS_FOR_PROJECT_NAME =
"SELECT `version_id` FROM `projects` WHERE `name` = ?";
private static final String GET_VERSION_IDS_FOR_PROJECT_NAME =
"SELECT `version_id` FROM `projects` WHERE `name` = ?";
private final String projectName;
private final String projectName;
public GetLatestVersionForProjectSQLQuery(String projectName) {
this.projectName = projectName;
public GetLatestVersionForProjectSQLQuery(String projectName) {
this.projectName = projectName;
}
@Override
public Integer processResultSet(ResultSet resultSet) throws SQLException {
int versionID = 0;
while (resultSet.next()) {
versionID = resultSet.getInt("version_id");
}
return versionID;
}
@Override
public Integer processResultSet(ResultSet resultSet) throws SQLException {
int versionID = 0;
while (resultSet.next()) {
versionID = resultSet.getInt("version_id");
}
return versionID;
}
@Override
public String getSQL() {
return GET_VERSION_IDS_FOR_PROJECT_NAME;
}
@Override
public String getSQL() {
return GET_VERSION_IDS_FOR_PROJECT_NAME;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setString(1, projectName);
}
@Override
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
statement.setString(1, projectName);
}
}

32
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumProjects.java
View File

@@ -1,30 +1,26 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
/**
/*
* Created by winston on 24/08/2016.
*/
public class GetNumProjects implements SQLQuery<Integer> {
private static final String GET_NUM_PROJECTS =
"SELECT COUNT(*)\n" +
" FROM `projects`";
private static final String GET_NUM_PROJECTS = "SELECT COUNT(*)\n" + " FROM `projects`";
@Override
public String getSQL() {
return GET_NUM_PROJECTS;
@Override
public String getSQL() {
return GET_NUM_PROJECTS;
}
@Override
public Integer processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
return resultSet.getInt("COUNT(*)");
}
@Override
public Integer processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
return resultSet.getInt("COUNT(*)");
}
throw new IllegalStateException("Count always returns results");
}
throw new IllegalStateException("Count always returns results");
}
}

34
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumUnswappedProjects.java
View File

@@ -1,31 +1,27 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
/**
/*
* Created by winston on 24/08/2016.
*/
public class GetNumUnswappedProjects implements SQLQuery<Integer> {
private static final String GET_NUM_UNSWAPPED_PROJECTS =
"SELECT COUNT(*)\n" +
" FROM `projects`\n" +
" WHERE `last_accessed` IS NOT NULL";
private static final String GET_NUM_UNSWAPPED_PROJECTS =
"SELECT COUNT(*)\n" + " FROM `projects`\n" + " WHERE `last_accessed` IS NOT NULL";
@Override
public String getSQL() {
return GET_NUM_UNSWAPPED_PROJECTS;
@Override
public String getSQL() {
return GET_NUM_UNSWAPPED_PROJECTS;
}
@Override
public Integer processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
return resultSet.getInt("COUNT(*)");
}
@Override
public Integer processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
return resultSet.getInt("COUNT(*)");
}
throw new IllegalStateException("Count always returns results");
}
throw new IllegalStateException("Count always returns results");
}
}

36
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetOldestProjectName.java
View File

@@ -1,31 +1,29 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
/**
/*
* Created by winston on 23/08/2016.
*/
public class GetOldestProjectName implements SQLQuery<String> {
private static final String GET_OLDEST_PROJECT_NAME =
"SELECT `name`, MIN(`last_accessed`)\n" +
" FROM `projects` \n" +
" WHERE `last_accessed` IS NOT NULL;";
private static final String GET_OLDEST_PROJECT_NAME =
"SELECT `name`, MIN(`last_accessed`)\n"
+ " FROM `projects` \n"
+ " WHERE `last_accessed` IS NOT NULL;";
@Override
public String getSQL() {
return GET_OLDEST_PROJECT_NAME;
@Override
public String getSQL() {
return GET_OLDEST_PROJECT_NAME;
}
@Override
public String processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
return resultSet.getString("name");
}
@Override
public String processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
return resultSet.getString("name");
}
return null;
}
return null;
}
}

61
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetPathForURLInProjectSQLQuery.java
View File

@@ -1,50 +1,43 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
/**
/*
* Created by Winston on 20/11/14.
*/
public class GetPathForURLInProjectSQLQuery implements SQLQuery<String> {
private static final String GET_URL_INDEXES_FOR_PROJECT_NAME =
"SELECT `path` "
+ "FROM `url_index_store` "
+ "WHERE `project_name` = ? "
+ "AND `url` = ?";
private static final String GET_URL_INDEXES_FOR_PROJECT_NAME =
"SELECT `path` " + "FROM `url_index_store` " + "WHERE `project_name` = ? " + "AND `url` = ?";
private final String projectName;
private final String url;
private final String projectName;
private final String url;
public GetPathForURLInProjectSQLQuery(String projectName, String url) {
this.projectName = projectName;
this.url = url;
public GetPathForURLInProjectSQLQuery(String projectName, String url) {
this.projectName = projectName;
this.url = url;
}
@Override
public String processResultSet(ResultSet resultSet) throws SQLException {
String path = null;
while (resultSet.next()) {
path = resultSet.getString("path");
}
return path;
}
@Override
public String processResultSet(ResultSet resultSet) throws SQLException {
String path = null;
while (resultSet.next()) {
path = resultSet.getString("path");
}
return path;
}
@Override
public String getSQL() {
return GET_URL_INDEXES_FOR_PROJECT_NAME;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setString(1, projectName);
statement.setString(2, url);
}
@Override
public String getSQL() {
return GET_URL_INDEXES_FOR_PROJECT_NAME;
}
@Override
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
statement.setString(1, projectName);
statement.setString(2, url);
}
}

33
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetProjectNamesSQLQuery.java
View File

@@ -1,34 +1,29 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
/**
/*
* Created by Winston on 21/02/15.
*/
public class GetProjectNamesSQLQuery implements SQLQuery<List<String>> {
private static final String GET_URL_INDEXES_FOR_PROJECT_NAME =
"SELECT `name` FROM `projects`";
private static final String GET_URL_INDEXES_FOR_PROJECT_NAME = "SELECT `name` FROM `projects`";
@Override
public List<String> processResultSet(
ResultSet resultSet
) throws SQLException {
List<String> projectNames = new ArrayList<>();
while (resultSet.next()) {
projectNames.add(resultSet.getString("name"));
}
return projectNames;
}
@Override
public String getSQL() {
return GET_URL_INDEXES_FOR_PROJECT_NAME;
@Override
public List<String> processResultSet(ResultSet resultSet) throws SQLException {
List<String> projectNames = new ArrayList<>();
while (resultSet.next()) {
projectNames.add(resultSet.getString("name"));
}
return projectNames;
}
@Override
public String getSQL() {
return GET_URL_INDEXES_FOR_PROJECT_NAME;
}
}

64
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetProjectState.java
View File

@@ -1,51 +1,43 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
/**
/*
* Created by winston on 24/08/2016.
*/
public class GetProjectState implements SQLQuery<ProjectState> {
private static final String GET_PROJECT_STATE =
"SELECT `last_accessed`\n" +
" FROM `projects`\n" +
" WHERE `name` = ?";
private static final String GET_PROJECT_STATE =
"SELECT `last_accessed`\n" + " FROM `projects`\n" + " WHERE `name` = ?";
private final String projectName;
private final String projectName;
public GetProjectState(String projectName) {
this.projectName = projectName;
}
@Override
public String getSQL() {
return GET_PROJECT_STATE;
}
@Override
public ProjectState processResultSet(
ResultSet resultSet
) throws SQLException {
while (resultSet.next()) {
if (resultSet.getTimestamp("last_accessed") == null) {
return ProjectState.SWAPPED;
}
return ProjectState.PRESENT;
}
return ProjectState.NOT_PRESENT;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setString(1, projectName);
public GetProjectState(String projectName) {
this.projectName = projectName;
}
@Override
public String getSQL() {
return GET_PROJECT_STATE;
}
@Override
public ProjectState processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getTimestamp("last_accessed") == null) {
return ProjectState.SWAPPED;
}
return ProjectState.PRESENT;
}
return ProjectState.NOT_PRESENT;
}
@Override
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
statement.setString(1, projectName);
}
}

9
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetSwapCompression.java
View File

@@ -1,14 +1,13 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
public class GetSwapCompression implements SQLQuery<String> {
private static final String GET_SWAP_COMPRESSION =
"SELECT `swap_compression` FROM `projects` WHERE `name` = ?";
"SELECT `swap_compression` FROM `projects` WHERE `name` = ?";
private final String projectName;
@@ -31,9 +30,7 @@ public class GetSwapCompression implements SQLQuery<String> {
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
statement.setString(1, projectName);
}
}

35
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/LastAccessedColumnExists.java
View File

@@ -1,31 +1,28 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
/**
/*
* Created by winston on 04/09/2016.
*/
public class LastAccessedColumnExists implements SQLQuery<Boolean> {
private static final String LAST_ACCESSED_COLUMN_EXISTS =
"PRAGMA table_info(`projects`)";
private static final String LAST_ACCESSED_COLUMN_EXISTS = "PRAGMA table_info(`projects`)";
@Override
public String getSQL() {
return LAST_ACCESSED_COLUMN_EXISTS;
@Override
public String getSQL() {
return LAST_ACCESSED_COLUMN_EXISTS;
}
@Override
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getString(2).equals("last_accessed")) {
return true;
}
}
@Override
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getString(2).equals("last_accessed")) {
return true;
}
}
return false;
}
return false;
}
}

30
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/RestoreTimeColumnExists.java
View File

@@ -1,26 +1,24 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
public class RestoreTimeColumnExists implements SQLQuery<Boolean> {
private static final String RESTORE_TIME_COLUMN_EXISTS =
"PRAGMA table_info(`projects`)";
private static final String RESTORE_TIME_COLUMN_EXISTS = "PRAGMA table_info(`projects`)";
@Override
public String getSQL() {
return RESTORE_TIME_COLUMN_EXISTS;
}
@Override
public String getSQL() {
return RESTORE_TIME_COLUMN_EXISTS;
}
@Override
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getString(2).equals("restore_time")) {
return true;
}
}
return false;
@Override
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getString(2).equals("restore_time")) {
return true;
}
}
return false;
}
}

33
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapCompressionColumnExists.java
View File

@@ -1,27 +1,24 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
public class SwapCompressionColumnExists implements SQLQuery<Boolean> {
private static final String SWAP_COMPRESSION_COLUMN_EXISTS =
"PRAGMA table_info(`projects`)";
private static final String SWAP_COMPRESSION_COLUMN_EXISTS = "PRAGMA table_info(`projects`)";
@Override
public String getSQL() {
return SWAP_COMPRESSION_COLUMN_EXISTS;
@Override
public String getSQL() {
return SWAP_COMPRESSION_COLUMN_EXISTS;
}
@Override
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getString(2).equals("swap_compression")) {
return true;
}
}
@Override
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getString(2).equals("swap_compression")) {
return true;
}
}
return false;
}
return false;
}
}

33
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapTimeColumnExists.java
View File

@@ -1,27 +1,24 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
public class SwapTimeColumnExists implements SQLQuery<Boolean> {
private static final String SWAP_TIME_COLUMN_EXISTS =
"PRAGMA table_info(`projects`)";
private static final String SWAP_TIME_COLUMN_EXISTS = "PRAGMA table_info(`projects`)";
@Override
public String getSQL() {
return SWAP_TIME_COLUMN_EXISTS;
@Override
public String getSQL() {
return SWAP_TIME_COLUMN_EXISTS;
}
@Override
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getString(2).equals("swap_time")) {
return true;
}
}
@Override
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getString(2).equals("swap_time")) {
return true;
}
}
return false;
}
return false;
}
}

16
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddLastAccessed.java
View File

@@ -2,18 +2,16 @@ package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
/*
* Created by winston on 03/09/2016.
*/
public class ProjectsAddLastAccessed implements SQLUpdate {
private static final String PROJECTS_ADD_LAST_ACCESSED =
"ALTER TABLE `projects`\n" +
"ADD COLUMN `last_accessed` DATETIME NULL DEFAULT 0";
@Override
public String getSQL() {
return PROJECTS_ADD_LAST_ACCESSED;
}
private static final String PROJECTS_ADD_LAST_ACCESSED =
"ALTER TABLE `projects`\n" + "ADD COLUMN `last_accessed` DATETIME NULL DEFAULT 0";
@Override
public String getSQL() {
return PROJECTS_ADD_LAST_ACCESSED;
}
}

3
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddRestoreTime.java
View File

@@ -4,8 +4,7 @@ import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
public class ProjectsAddRestoreTime implements SQLUpdate {
private static final String PROJECTS_ADD_RESTORE_TIME =
"ALTER TABLE `projects`\n" +
"ADD COLUMN `restore_time` DATETIME NULL;\n";
"ALTER TABLE `projects`\n" + "ADD COLUMN `restore_time` DATETIME NULL;\n";
@Override
public String getSQL() {

3
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapCompression.java
View File

@@ -4,8 +4,7 @@ import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
public class ProjectsAddSwapCompression implements SQLUpdate {
private static final String PROJECTS_ADD_SWAP_COMPRESSION =
"ALTER TABLE `projects`\n" +
"ADD COLUMN `swap_compression` VARCHAR NULL;\n";
"ALTER TABLE `projects`\n" + "ADD COLUMN `swap_compression` VARCHAR NULL;\n";
@Override
public String getSQL() {

4
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapTime.java
View File

@@ -4,12 +4,10 @@ import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
public class ProjectsAddSwapTime implements SQLUpdate {
private static final String PROJECTS_ADD_SWAP_TIME =
"ALTER TABLE `projects`\n" +
"ADD COLUMN `swap_time` DATETIME NULL;\n";
"ALTER TABLE `projects`\n" + "ADD COLUMN `swap_time` DATETIME NULL;\n";
@Override
public String getSQL() {
return PROJECTS_ADD_SWAP_TIME;
}
}

17
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/SetSoftHeapLimitPragma.java
View File

@@ -3,15 +3,14 @@ package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
public class SetSoftHeapLimitPragma implements SQLUpdate {
private int heapLimitBytes = 0;
private int heapLimitBytes = 0;
public SetSoftHeapLimitPragma(int heapLimitBytes) {
this.heapLimitBytes = heapLimitBytes;
}
@Override
public String getSQL() {
return "PRAGMA soft_heap_limit="+this.heapLimitBytes+";";
}
public SetSoftHeapLimitPragma(int heapLimitBytes) {
this.heapLimitBytes = heapLimitBytes;
}
@Override
public String getSQL() {
return "PRAGMA soft_heap_limit=" + this.heapLimitBytes + ";";
}
}

17
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateIndexURLIndexStore.java
View File

@@ -2,18 +2,17 @@ package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
/*
* Created by Winston on 21/02/15.
*/
public class CreateIndexURLIndexStore implements SQLUpdate {
public static final String CREATE_INDEX_URL_INDEX_STORE =
"CREATE UNIQUE INDEX IF NOT EXISTS `project_path_index` " +
"ON `url_index_store`(`project_name`, `path`);\n";
@Override
public String getSQL() {
return CREATE_INDEX_URL_INDEX_STORE;
}
public static final String CREATE_INDEX_URL_INDEX_STORE =
"CREATE UNIQUE INDEX IF NOT EXISTS `project_path_index` "
+ "ON `url_index_store`(`project_name`, `path`);\n";
@Override
public String getSQL() {
return CREATE_INDEX_URL_INDEX_STORE;
}
}

17
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsIndexLastAccessed.java
View File

@@ -2,18 +2,17 @@ package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
/*
* Created by winston on 23/08/2016.
*/
public class CreateProjectsIndexLastAccessed implements SQLUpdate {
private static final String CREATE_PROJECTS_INDEX_LAST_ACCESSED =
"CREATE INDEX IF NOT EXISTS `projects_index_last_accessed`\n" +
" ON `projects`(`last_accessed`)";
@Override
public String getSQL() {
return CREATE_PROJECTS_INDEX_LAST_ACCESSED;
}
private static final String CREATE_PROJECTS_INDEX_LAST_ACCESSED =
"CREATE INDEX IF NOT EXISTS `projects_index_last_accessed`\n"
+ " ON `projects`(`last_accessed`)";
@Override
public String getSQL() {
return CREATE_PROJECTS_INDEX_LAST_ACCESSED;
}
}

31
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsTableSQLUpdate.java
View File

@@ -2,25 +2,24 @@ package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
/*
* Created by Winston on 20/11/14.
*/
public class CreateProjectsTableSQLUpdate implements SQLUpdate {
private static final String CREATE_PROJECTS_TABLE =
"CREATE TABLE IF NOT EXISTS `projects` (\n" +
" `name` VARCHAR NOT NULL DEFAULT '',\n" +
" `version_id` INT NOT NULL DEFAULT 0,\n" +
" `last_accessed` DATETIME NULL DEFAULT 0,\n" +
" `swap_time` DATETIME NULL,\n" +
" `restore_time` DATETIME NULL,\n" +
" `swap_compression` VARCHAR NULL,\n" +
" PRIMARY KEY (`name`)\n" +
")";
@Override
public String getSQL() {
return CREATE_PROJECTS_TABLE;
}
private static final String CREATE_PROJECTS_TABLE =
"CREATE TABLE IF NOT EXISTS `projects` (\n"
+ " `name` VARCHAR NOT NULL DEFAULT '',\n"
+ " `version_id` INT NOT NULL DEFAULT 0,\n"
+ " `last_accessed` DATETIME NULL DEFAULT 0,\n"
+ " `swap_time` DATETIME NULL,\n"
+ " `restore_time` DATETIME NULL,\n"
+ " `swap_compression` VARCHAR NULL,\n"
+ " PRIMARY KEY (`name`)\n"
+ ")";
@Override
public String getSQL() {
return CREATE_PROJECTS_TABLE;
}
}

35
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateURLIndexStoreSQLUpdate.java
View File

@@ -2,27 +2,26 @@ package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
/*
* Created by Winston on 20/11/14.
*/
public class CreateURLIndexStoreSQLUpdate implements SQLUpdate {
private static final String CREATE_URL_INDEX_STORE =
"CREATE TABLE IF NOT EXISTS `url_index_store` (\n"+
" `project_name` varchar(10) NOT NULL DEFAULT '',\n"+
" `url` text NOT NULL,\n"+
" `path` text NOT NULL,\n"+
" PRIMARY KEY (`project_name`,`url`),\n"+
" CONSTRAINT `url_index_store_ibfk_1` " +
"FOREIGN KEY (`project_name`) " +
"REFERENCES `projects` (`name`) " +
"ON DELETE CASCADE " +
"ON UPDATE CASCADE\n"+
");\n";
@Override
public String getSQL() {
return CREATE_URL_INDEX_STORE;
}
private static final String CREATE_URL_INDEX_STORE =
"CREATE TABLE IF NOT EXISTS `url_index_store` (\n"
+ " `project_name` varchar(10) NOT NULL DEFAULT '',\n"
+ " `url` text NOT NULL,\n"
+ " `path` text NOT NULL,\n"
+ " PRIMARY KEY (`project_name`,`url`),\n"
+ " CONSTRAINT `url_index_store_ibfk_1` "
+ "FOREIGN KEY (`project_name`) "
+ "REFERENCES `projects` (`name`) "
+ "ON DELETE CASCADE "
+ "ON UPDATE CASCADE\n"
+ ");\n";
@Override
public String getSQL() {
return CREATE_URL_INDEX_STORE;
}
}

1
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteAllFilesInProjectSQLUpdate.java
View File

@@ -2,7 +2,6 @@ package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
public class DeleteAllFilesInProjectSQLUpdate implements SQLUpdate {

66
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteFilesForProjectSQLUpdate.java
View File

@@ -1,53 +1,43 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
/*
* Created by Winston on 20/11/14.
*/
public class DeleteFilesForProjectSQLUpdate implements SQLUpdate {
private static final String DELETE_URL_INDEXES_FOR_PROJECT_NAME =
"DELETE FROM `url_index_store` " +
"WHERE `project_name` = ? AND path IN (";
private static final String DELETE_URL_INDEXES_FOR_PROJECT_NAME =
"DELETE FROM `url_index_store` " + "WHERE `project_name` = ? AND path IN (";
private final String projectName;
private final String[] paths;
private final String projectName;
private final String[] paths;
public DeleteFilesForProjectSQLUpdate(
String projectName,
String... paths
) {
this.projectName = projectName;
this.paths = paths;
public DeleteFilesForProjectSQLUpdate(String projectName, String... paths) {
this.projectName = projectName;
this.paths = paths;
}
@Override
public String getSQL() {
StringBuilder sb = new StringBuilder(DELETE_URL_INDEXES_FOR_PROJECT_NAME);
for (int i = 0; i < paths.length; i++) {
sb.append("?");
if (i < paths.length - 1) {
sb.append(", ");
}
}
sb.append(");\n");
return sb.toString();
}
@Override
public String getSQL() {
StringBuilder sb = new StringBuilder(
DELETE_URL_INDEXES_FOR_PROJECT_NAME
);
for (int i = 0; i < paths.length; i++) {
sb.append("?");
if (i < paths.length - 1) {
sb.append(", ");
}
}
sb.append(");\n");
return sb.toString();
@Override
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
statement.setString(1, projectName);
for (int i = 0; i < paths.length; i++) {
statement.setString(i + 2, paths[i]);
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setString(1, projectName);
for (int i = 0; i < paths.length; i++) {
statement.setString(i + 2, paths[i]);
}
}
}
}

1
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteProjectSQLUpdate.java
View File

@@ -2,7 +2,6 @@ package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
public class DeleteProjectSQLUpdate implements SQLUpdate {

58
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/AddURLIndexSQLUpdate.java
View File

@@ -1,45 +1,41 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
/*
* Created by Winston on 20/11/14.
*/
public class AddURLIndexSQLUpdate implements SQLUpdate {
private static final String ADD_URL_INDEX =
"INSERT OR REPLACE INTO `url_index_store`(" +
"`project_name`, " +
"`url`, " +
"`path`" +
") VALUES " +
"(?, ?, ?)\n";
private static final String ADD_URL_INDEX =
"INSERT OR REPLACE INTO `url_index_store`("
+ "`project_name`, "
+ "`url`, "
+ "`path`"
+ ") VALUES "
+ "(?, ?, ?)\n";
private final String projectName;
private final String url;
private final String path;
private final String projectName;
private final String url;
private final String path;
public AddURLIndexSQLUpdate(String projectName, String url, String path) {
this.projectName = projectName;
this.url = url;
this.path = path;
}
public AddURLIndexSQLUpdate(String projectName, String url, String path) {
this.projectName = projectName;
this.url = url;
this.path = path;
}
@Override
public String getSQL() {
return ADD_URL_INDEX;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setString(1, projectName);
statement.setString(2, url);
statement.setString(3, path);
}
@Override
public String getSQL() {
return ADD_URL_INDEX;
}
@Override
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
statement.setString(1, projectName);
statement.setString(2, url);
statement.setString(3, path);
}
}

47
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/SetProjectLastAccessedTime.java
View File

@@ -1,43 +1,34 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
/*
* Created by winston on 23/08/2016.
*/
public class SetProjectLastAccessedTime implements SQLUpdate {
private static final String SET_PROJECT_LAST_ACCESSED_TIME =
"UPDATE `projects`\n" +
"SET `last_accessed` = ?\n" +
"WHERE `name` = ?";
private static final String SET_PROJECT_LAST_ACCESSED_TIME =
"UPDATE `projects`\n" + "SET `last_accessed` = ?\n" + "WHERE `name` = ?";
private final String projectName;
private final Timestamp lastAccessed;
private final String projectName;
private final Timestamp lastAccessed;
public SetProjectLastAccessedTime(
String projectName,
Timestamp lastAccessed
) {
this.projectName = projectName;
this.lastAccessed = lastAccessed;
}
public SetProjectLastAccessedTime(String projectName, Timestamp lastAccessed) {
this.projectName = projectName;
this.lastAccessed = lastAccessed;
}
@Override
public String getSQL() {
return SET_PROJECT_LAST_ACCESSED_TIME;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setTimestamp(1, lastAccessed);
statement.setString(2, projectName);
}
@Override
public String getSQL() {
return SET_PROJECT_LAST_ACCESSED_TIME;
}
@Override
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
statement.setTimestamp(1, lastAccessed);
statement.setString(2, projectName);
}
}

46
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/SetProjectSQLUpdate.java
View File

@@ -1,39 +1,35 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
/*
* Created by Winston on 20/11/14.
*/
public class SetProjectSQLUpdate implements SQLUpdate {
private static final String SET_PROJECT =
"INSERT OR REPLACE "
+ "INTO `projects`(`name`, `version_id`, `last_accessed`) "
+ "VALUES (?, ?, DATETIME('now'));\n";
private static final String SET_PROJECT =
"INSERT OR REPLACE "
+ "INTO `projects`(`name`, `version_id`, `last_accessed`) "
+ "VALUES (?, ?, DATETIME('now'));\n";
private final String projectName;
private final int versionID;
private final String projectName;
private final int versionID;
public SetProjectSQLUpdate(String projectName, int versionID) {
this.projectName = projectName;
this.versionID = versionID;
}
public SetProjectSQLUpdate(String projectName, int versionID) {
this.projectName = projectName;
this.versionID = versionID;
}
@Override
public String getSQL() {
return SET_PROJECT;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setString(1, projectName);
statement.setInt(2, versionID);
}
@Override
public String getSQL() {
return SET_PROJECT;
}
@Override
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
statement.setString(1, projectName);
statement.setInt(2, versionID);
}
}

19
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateRestore.java
View File

@@ -1,20 +1,19 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
public class UpdateRestore implements SQLUpdate {
private static final String UPDATE_RESTORE =
"UPDATE `projects`\n" +
"SET `last_accessed` = ?,\n" +
" `swap_time` = NULL,\n" +
" `restore_time` = ?,\n" +
" `swap_compression` = NULL\n" +
"WHERE `name` = ?;\n";
"UPDATE `projects`\n"
+ "SET `last_accessed` = ?,\n"
+ " `swap_time` = NULL,\n"
+ " `restore_time` = ?,\n"
+ " `swap_compression` = NULL\n"
+ "WHERE `name` = ?;\n";
private final String projectName;
private final Timestamp now;
@@ -30,9 +29,7 @@ public class UpdateRestore implements SQLUpdate {
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
statement.setTimestamp(1, now);
statement.setTimestamp(2, now);
statement.setString(3, projectName);

19
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateSwap.java
View File

@@ -1,20 +1,19 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
public class UpdateSwap implements SQLUpdate {
private static final String UPDATE_SWAP =
"UPDATE `projects`\n" +
"SET `last_accessed` = NULL,\n" +
" `swap_time` = ?,\n" +
" `restore_time` = NULL,\n" +
" `swap_compression` = ?\n" +
"WHERE `name` = ?;\n";
"UPDATE `projects`\n"
+ "SET `last_accessed` = NULL,\n"
+ " `swap_time` = ?,\n"
+ " `restore_time` = NULL,\n"
+ " `swap_compression` = ?\n"
+ "WHERE `name` = ?;\n";
private final String projectName;
private final String compression;
@@ -32,9 +31,7 @@ public class UpdateSwap implements SQLUpdate {
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
statement.setTimestamp(1, now);
statement.setString(2, compression);
statement.setString(3, projectName);

19
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/gc/GcJob.java
View File

@@ -1,13 +1,8 @@
package uk.ac.ic.wlgitbridge.bridge.gc;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
/**
/*
* Is started by the bridge. Every time a project is updated, we queue it for
* GC which executes every hour or so.
*
@@ -20,15 +15,15 @@ import java.util.concurrent.CompletableFuture;
*/
public interface GcJob {
void start();
void start();
void stop();
void stop();
void onPreGc(Runnable preGc);
void onPreGc(Runnable preGc);
void onPostGc(Runnable postGc);
void onPostGc(Runnable postGc);
void queueForGc(String projectName);
void queueForGc(String projectName);
CompletableFuture<Void> waitForRun();
CompletableFuture<Void> waitForRun();
}

218
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/gc/GcJobImpl.java
View File

@@ -1,13 +1,5 @@
package uk.ac.ic.wlgitbridge.bridge.gc;
import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.data.CannotAcquireLockException;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.TimerUtils;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.CompletableFuture;
@@ -16,129 +8,123 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.data.CannotAcquireLockException;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.TimerUtils;
/**
/*
* Implementation of {@link GcJob} using its own Timer and a synchronized
* queue.
*/
public class GcJobImpl implements GcJob {
private final RepoStore repoStore;
private final ProjectLock locks;
private final RepoStore repoStore;
private final ProjectLock locks;
private final long intervalMs;
private final Timer timer;
private final long intervalMs;
private final Timer timer;
private final Set<String> gcQueue;
private final Set<String> gcQueue;
/**
* Hooks in case they are needed, e.g. for testing.
*/
private AtomicReference<Runnable> preGc;
private AtomicReference<Runnable> postGc;
/*
* Hooks in case they are needed, e.g. for testing.
*/
private AtomicReference<Runnable> preGc;
private AtomicReference<Runnable> postGc;
/* We need to iterate over and empty it after every run */
private final Lock jobWaitersLock;
private final List<CompletableFuture<Void>> jobWaiters;
/* We need to iterate over and empty it after every run */
private final Lock jobWaitersLock;
private final List<CompletableFuture<Void>> jobWaiters;
public GcJobImpl(RepoStore repoStore, ProjectLock locks, long intervalMs) {
this.repoStore = repoStore;
this.locks = locks;
this.intervalMs = intervalMs;
timer = new Timer();
gcQueue = Collections.newSetFromMap(new ConcurrentHashMap<>());
preGc = new AtomicReference<>(() -> {});
postGc = new AtomicReference<>(() -> {});
jobWaitersLock = new ReentrantLock();
jobWaiters = new ArrayList<>();
public GcJobImpl(RepoStore repoStore, ProjectLock locks, long intervalMs) {
this.repoStore = repoStore;
this.locks = locks;
this.intervalMs = intervalMs;
timer = new Timer();
gcQueue = Collections.newSetFromMap(new ConcurrentHashMap<>());
preGc = new AtomicReference<>(() -> {});
postGc = new AtomicReference<>(() -> {});
jobWaitersLock = new ReentrantLock();
jobWaiters = new ArrayList<>();
}
public GcJobImpl(RepoStore repoStore, ProjectLock locks) {
this(repoStore, locks, TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS));
}
@Override
public void start() {
Log.info("Starting GC job to run every [{}] ms", intervalMs);
timer.scheduleAtFixedRate(TimerUtils.makeTimerTask(this::doGC), intervalMs, intervalMs);
}
@Override
public void stop() {
Log.info("Stopping GC job");
timer.cancel();
}
@Override
public void onPreGc(Runnable preGc) {
this.preGc.set(preGc);
}
@Override
public void onPostGc(Runnable postGc) {
this.postGc.set(postGc);
}
/*
* Needs to be callable from any thread.
* @param projectName
*/
@Override
public void queueForGc(String projectName) {
gcQueue.add(projectName);
}
@Override
public CompletableFuture<Void> waitForRun() {
CompletableFuture<Void> ret = new CompletableFuture<>();
jobWaitersLock.lock();
try {
jobWaiters.add(ret);
} finally {
jobWaitersLock.unlock();
}
return ret;
}
public GcJobImpl(RepoStore repoStore, ProjectLock locks) {
this(
repoStore,
locks,
TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS)
);
}
@Override
public void start() {
Log.info("Starting GC job to run every [{}] ms", intervalMs);
timer.scheduleAtFixedRate(
TimerUtils.makeTimerTask(this::doGC),
intervalMs,
intervalMs
);
}
@Override
public void stop() {
Log.info("Stopping GC job");
timer.cancel();
}
@Override
public void onPreGc(Runnable preGc) {
this.preGc.set(preGc);
}
@Override
public void onPostGc(Runnable postGc) {
this.postGc.set(postGc);
}
/**
* Needs to be callable from any thread.
* @param projectName
*/
@Override
public void queueForGc(String projectName) {
gcQueue.add(projectName);
}
@Override
public CompletableFuture<Void> waitForRun() {
CompletableFuture<Void> ret = new CompletableFuture<>();
jobWaitersLock.lock();
private void doGC() {
Log.info("GC job running");
int numGcs = 0;
preGc.get().run();
for (Iterator<String> it = gcQueue.iterator(); it.hasNext(); it.remove(), ++numGcs) {
String proj = it.next();
Log.debug("[{}] Running GC job on project", proj);
try (LockGuard __ = locks.lockGuard(proj)) {
try {
jobWaiters.add(ret);
} finally {
jobWaitersLock.unlock();
ProjectRepo repo = repoStore.getExistingRepo(proj);
repo.runGC();
repo.deleteIncomingPacks();
} catch (IOException e) {
Log.warn("[{}] Failed to GC project", proj);
}
return ret;
} catch (CannotAcquireLockException e) {
Log.warn("[{}] Cannot acquire project lock, skipping GC", proj);
}
}
private void doGC() {
Log.info("GC job running");
int numGcs = 0;
preGc.get().run();
for (
Iterator<String> it = gcQueue.iterator();
it.hasNext();
it.remove(), ++numGcs
) {
String proj = it.next();
Log.debug("[{}] Running GC job on project", proj);
try (LockGuard __ = locks.lockGuard(proj)) {
try {
ProjectRepo repo = repoStore.getExistingRepo(proj);
repo.runGC();
repo.deleteIncomingPacks();
} catch (IOException e) {
Log.warn("[{}] Failed to GC project", proj);
}
} catch (CannotAcquireLockException e) {
Log.warn("[{}] Cannot acquire project lock, skipping GC", proj);
}
}
Log.info("GC job finished, num gcs: {}", numGcs);
jobWaitersLock.lock();
try {
jobWaiters.forEach(w -> w.complete(null));
} finally {
jobWaitersLock.unlock();
}
postGc.get().run();
Log.info("GC job finished, num gcs: {}", numGcs);
jobWaitersLock.lock();
try {
jobWaiters.forEach(w -> w.complete(null));
} finally {
jobWaitersLock.unlock();
}
postGc.get().run();
}
}

5
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/LockGuard.java
View File

@@ -1,10 +1,9 @@
package uk.ac.ic.wlgitbridge.bridge.lock;
/**
/*
* Created by winston on 24/08/2016.
*/
public interface LockGuard extends AutoCloseable {
void close();
void close();
}

19
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/ProjectLock.java
View File

@@ -2,7 +2,7 @@ package uk.ac.ic.wlgitbridge.bridge.lock;
import uk.ac.ic.wlgitbridge.data.CannotAcquireLockException;
/**
/*
* Project Lock class.
*
* The locks should be re-entrant. For example, we are usually holding the lock
@@ -10,16 +10,15 @@ import uk.ac.ic.wlgitbridge.data.CannotAcquireLockException;
*/
public interface ProjectLock {
void lockAll();
void lockAll();
void lockForProject(String projectName) throws CannotAcquireLockException;
void lockForProject(String projectName) throws CannotAcquireLockException;
void unlockForProject(String projectName);
/* RAII hahaha */
default LockGuard lockGuard(String projectName) throws CannotAcquireLockException {
lockForProject(projectName);
return () -> unlockForProject(projectName);
}
void unlockForProject(String projectName);
/* RAII hahaha */
default LockGuard lockGuard(String projectName) throws CannotAcquireLockException {
lockForProject(projectName);
return () -> unlockForProject(projectName);
}
}

301
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java
View File

@@ -1,13 +1,8 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.Project;
import uk.ac.ic.wlgitbridge.util.Tar;
import static uk.ac.ic.wlgitbridge.util.Util.deleteInDirectoryApartFrom;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -17,201 +12,153 @@ import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import org.apache.commons.io.FileUtils;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.Project;
import uk.ac.ic.wlgitbridge.util.Tar;
import static uk.ac.ic.wlgitbridge.util.Util.deleteInDirectoryApartFrom;
/**
/*
* Created by winston on 20/08/2016.
*/
public class FSGitRepoStore implements RepoStore {
private static final long DEFAULT_MAX_FILE_SIZE = 50 * 1024 * 1024;
private static final long DEFAULT_MAX_FILE_SIZE = 50 * 1024 * 1024;
private final String repoStorePath;
private final String repoStorePath;
private final File rootDirectory;
private final File rootDirectory;
private final long maxFileSize;
private final long maxFileSize;
private final Function<File, Long> fsSizer;
private final Function<File, Long> fsSizer;
public FSGitRepoStore(
String repoStorePath,
Optional<Long> maxFileSize
) {
this(
repoStorePath,
maxFileSize.orElse(DEFAULT_MAX_FILE_SIZE),
d -> d.getTotalSpace() - d.getFreeSpace()
);
}
public FSGitRepoStore(String repoStorePath, Optional<Long> maxFileSize) {
this(
repoStorePath,
maxFileSize.orElse(DEFAULT_MAX_FILE_SIZE),
d -> d.getTotalSpace() - d.getFreeSpace());
}
public FSGitRepoStore(
String repoStorePath,
long maxFileSize,
Function<File, Long> fsSizer
) {
this.repoStorePath = repoStorePath;
rootDirectory = initRootGitDirectory(repoStorePath);
this.maxFileSize = maxFileSize;
this.fsSizer = fsSizer;
}
public FSGitRepoStore(String repoStorePath, long maxFileSize, Function<File, Long> fsSizer) {
this.repoStorePath = repoStorePath;
rootDirectory = initRootGitDirectory(repoStorePath);
this.maxFileSize = maxFileSize;
this.fsSizer = fsSizer;
}
@Override
public String getRepoStorePath() {
return repoStorePath;
}
@Override
public String getRepoStorePath() {
return repoStorePath;
}
@Override
public File getRootDirectory() {
return rootDirectory;
}
@Override
public File getRootDirectory() {
return rootDirectory;
}
@Override
public ProjectRepo initRepo(String project) throws IOException {
GitProjectRepo ret = GitProjectRepo.fromName(project);
ret.initRepo(this);
return new WalkOverrideGitRepo(
ret, Optional.of(maxFileSize), Optional.empty());
}
@Override
public ProjectRepo initRepo(String project) throws IOException {
GitProjectRepo ret = GitProjectRepo.fromName(project);
ret.initRepo(this);
return new WalkOverrideGitRepo(ret, Optional.of(maxFileSize), Optional.empty());
}
@Override
public ProjectRepo getExistingRepo(String project) throws IOException {
GitProjectRepo ret = GitProjectRepo.fromName(project);
ret.useExistingRepository(this);
return new WalkOverrideGitRepo(
ret, Optional.of(maxFileSize), Optional.empty());
}
@Override
public ProjectRepo getExistingRepo(String project) throws IOException {
GitProjectRepo ret = GitProjectRepo.fromName(project);
ret.useExistingRepository(this);
return new WalkOverrideGitRepo(ret, Optional.of(maxFileSize), Optional.empty());
}
@Override
public ProjectRepo useJGitRepo(Repository repo, ObjectId commitId) {
GitProjectRepo ret = GitProjectRepo.fromJGitRepo(repo);
return new WalkOverrideGitRepo(
ret, Optional.of(maxFileSize), Optional.of(commitId));
}
@Override
public ProjectRepo useJGitRepo(Repository repo, ObjectId commitId) {
GitProjectRepo ret = GitProjectRepo.fromJGitRepo(repo);
return new WalkOverrideGitRepo(ret, Optional.of(maxFileSize), Optional.of(commitId));
}
/* TODO: Perhaps we should just delete bad directories on the fly. */
@Override
public void purgeNonexistentProjects(
Collection<String> existingProjectNames
) {
List<String> excludedFromDeletion =
new ArrayList<>(existingProjectNames);
excludedFromDeletion.add(".wlgb");
deleteInDirectoryApartFrom(
rootDirectory,
excludedFromDeletion.toArray(new String[] {})
);
}
/* TODO: Perhaps we should just delete bad directories on the fly. */
@Override
public void purgeNonexistentProjects(Collection<String> existingProjectNames) {
List<String> excludedFromDeletion = new ArrayList<>(existingProjectNames);
excludedFromDeletion.add(".wlgb");
deleteInDirectoryApartFrom(rootDirectory, excludedFromDeletion.toArray(new String[] {}));
}
@Override
public long totalSize() {
return fsSizer.apply(rootDirectory);
}
@Override
public long totalSize() {
return fsSizer.apply(rootDirectory);
}
@Override
public InputStream bzip2Project(
String projectName,
long[] sizePtr
) throws IOException {
Project.checkValidProjectName(projectName);
Log.debug("[{}] bzip2 project", projectName);
return Tar.bz2.zip(getDotGitForProject(projectName), sizePtr);
}
@Override
public InputStream bzip2Project(String projectName, long[] sizePtr) throws IOException {
Project.checkValidProjectName(projectName);
Log.debug("[{}] bzip2 project", projectName);
return Tar.bz2.zip(getDotGitForProject(projectName), sizePtr);
}
@Override
public InputStream gzipProject(
String projectName,
long[] sizePtr
) throws IOException {
Project.checkValidProjectName(projectName);
Log.debug("[{}] gzip project", projectName);
return Tar.gzip.zip(getDotGitForProject(projectName), sizePtr);
}
@Override
public InputStream gzipProject(String projectName, long[] sizePtr) throws IOException {
Project.checkValidProjectName(projectName);
Log.debug("[{}] gzip project", projectName);
return Tar.gzip.zip(getDotGitForProject(projectName), sizePtr);
}
@Override
public void gcProject(String projectName) throws IOException {
Project.checkValidProjectName(projectName);
ProjectRepo repo = getExistingRepo(projectName);
repo.runGC();
}
@Override
public void gcProject(String projectName) throws IOException {
Project.checkValidProjectName(projectName);
ProjectRepo repo = getExistingRepo(projectName);
repo.runGC();
}
@Override
public void remove(String projectName) throws IOException {
Project.checkValidProjectName(projectName);
FileUtils.deleteDirectory(new File(rootDirectory, projectName));
}
@Override
public void remove(String projectName) throws IOException {
Project.checkValidProjectName(projectName);
FileUtils.deleteDirectory(new File(rootDirectory, projectName));
}
@Override
public void unbzip2Project(
String projectName,
InputStream dataStream
) throws IOException {
Preconditions.checkArgument(
Project.isValidProjectName(projectName),
"[%s] invalid project name: ",
projectName
);
Preconditions.checkState(
getDirForProject(projectName).mkdirs(),
"[%s] directories for " +
"evicted project already exist",
projectName
);
Log.debug("[{}] un-bzip2 project", projectName);
Tar.bz2.unzip(dataStream, getDirForProject(projectName));
}
@Override
public void unbzip2Project(String projectName, InputStream dataStream) throws IOException {
Preconditions.checkArgument(
Project.isValidProjectName(projectName), "[%s] invalid project name: ", projectName);
Preconditions.checkState(
getDirForProject(projectName).mkdirs(),
"[%s] directories for " + "evicted project already exist",
projectName);
Log.debug("[{}] un-bzip2 project", projectName);
Tar.bz2.unzip(dataStream, getDirForProject(projectName));
}
@Override
public void ungzipProject(
String projectName,
InputStream dataStream
) throws IOException {
Preconditions.checkArgument(
Project.isValidProjectName(projectName),
"[%s] invalid project name: ",
projectName
);
Preconditions.checkState(
getDirForProject(projectName).mkdirs(),
"[%s] directories for " +
"evicted project already exist",
projectName
);
Log.debug("[{}] un-gzip project", projectName);
Tar.gzip.unzip(dataStream, getDirForProject(projectName));
}
@Override
public void ungzipProject(String projectName, InputStream dataStream) throws IOException {
Preconditions.checkArgument(
Project.isValidProjectName(projectName), "[%s] invalid project name: ", projectName);
Preconditions.checkState(
getDirForProject(projectName).mkdirs(),
"[%s] directories for " + "evicted project already exist",
projectName);
Log.debug("[{}] un-gzip project", projectName);
Tar.gzip.unzip(dataStream, getDirForProject(projectName));
}
private File getDirForProject(String projectName) {
Project.checkValidProjectName(projectName);
return Paths.get(
rootDirectory.getAbsolutePath()
).resolve(
projectName
).toFile();
}
private File getDirForProject(String projectName) {
Project.checkValidProjectName(projectName);
return Paths.get(rootDirectory.getAbsolutePath()).resolve(projectName).toFile();
}
private File getDotGitForProject(String projectName) {
Project.checkValidProjectName(projectName);
return Paths.get(
rootDirectory.getAbsolutePath()
).resolve(
projectName
).resolve(
".git"
).toFile();
}
private File initRootGitDirectory(String rootGitDirectoryPath) {
File rootGitDirectory = new File(rootGitDirectoryPath);
rootGitDirectory.mkdirs();
Preconditions.checkArgument(
rootGitDirectory.isDirectory(),
"given root git directory " +
"is not a directory: %s",
rootGitDirectory.getAbsolutePath()
);
return rootGitDirectory;
}
private File getDotGitForProject(String projectName) {
Project.checkValidProjectName(projectName);
return Paths.get(rootDirectory.getAbsolutePath()).resolve(projectName).resolve(".git").toFile();
}
private File initRootGitDirectory(String rootGitDirectoryPath) {
File rootGitDirectory = new File(rootGitDirectoryPath);
rootGitDirectory.mkdirs();
Preconditions.checkArgument(
rootGitDirectory.isDirectory(),
"given root git directory " + "is not a directory: %s",
rootGitDirectory.getAbsolutePath());
return rootGitDirectory;
}
}

415
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepo.java
View File

@@ -1,6 +1,15 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import com.google.common.base.Preconditions;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.*;
import org.apache.commons.io.IOUtils;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.ResetCommand;
@@ -16,17 +25,7 @@ import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.Project;
import uk.ac.ic.wlgitbridge.util.Util;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.*;
/**
/*
* Class representing a Git repository.
*
* It stores the projectName and repo separately because the hooks need to be
@@ -40,241 +39,193 @@ import java.util.*;
*/
public class GitProjectRepo implements ProjectRepo {
private final String projectName;
private Optional<Repository> repository;
private final String projectName;
private Optional<Repository> repository;
public static GitProjectRepo fromJGitRepo(Repository repo) {
return new GitProjectRepo(
repo.getWorkTree().getName(), Optional.of(repo));
public static GitProjectRepo fromJGitRepo(Repository repo) {
return new GitProjectRepo(repo.getWorkTree().getName(), Optional.of(repo));
}
public static GitProjectRepo fromName(String projectName) {
return new GitProjectRepo(projectName, Optional.empty());
}
GitProjectRepo(String projectName, Optional<Repository> repository) {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
this.projectName = projectName;
this.repository = repository;
}
@Override
public String getProjectName() {
return projectName;
}
@Override
public void initRepo(RepoStore repoStore) throws IOException {
initRepositoryField(repoStore);
Preconditions.checkState(repository.isPresent());
Repository repo = this.repository.get();
// TODO: assert that this is a fresh repo. At the moment, we can't be
// sure whether the repo to be init'd doesn't exist or is just fresh
// and we crashed / aborted while committing
if (repo.getObjectDatabase().exists()) return;
repo.create();
}
@Override
public void useExistingRepository(RepoStore repoStore) throws IOException {
initRepositoryField(repoStore);
Preconditions.checkState(repository.isPresent());
Preconditions.checkState(repository.get().getObjectDatabase().exists());
}
@Override
public RawDirectory getDirectory() throws IOException, GitUserException {
Preconditions.checkState(repository.isPresent());
return new RepositoryObjectTreeWalker(repository.get()).getDirectoryContents(Optional.empty());
}
@Override
public Collection<String> commitAndGetMissing(GitDirectoryContents contents) throws IOException {
try {
return doCommitAndGetMissing(contents);
} catch (GitAPIException e) {
throw new IOException(e);
}
}
public static GitProjectRepo fromName(String projectName) {
return new GitProjectRepo(projectName, Optional.empty());
@Override
public void runGC() throws IOException {
Preconditions.checkState(repository.isPresent(), "Repo is not present");
File dir = getProjectDir();
Preconditions.checkState(dir.isDirectory());
Log.debug("[{}] Running git gc", projectName);
Process proc = new ProcessBuilder("git", "gc").directory(dir).start();
int exitCode;
try {
exitCode = proc.waitFor();
Log.debug("Exit: {}", exitCode);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
GitProjectRepo(String projectName, Optional<Repository> repository) {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
this.projectName = projectName;
this.repository = repository;
if (exitCode != 0) {
Log.warn("[{}] Git gc failed", dir.getAbsolutePath());
Log.warn(IOUtils.toString(proc.getInputStream(), StandardCharsets.UTF_8));
Log.warn(IOUtils.toString(proc.getErrorStream(), StandardCharsets.UTF_8));
throw new IOException("git gc error");
}
Log.debug("[{}] git gc successful", projectName);
}
@Override
public String getProjectName() {
return projectName;
}
@Override
public void deleteIncomingPacks() throws IOException {
Log.debug("[{}] Checking for garbage `incoming` files", projectName);
Files.walkFileTree(
getDotGitDir().toPath(),
new FileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
throws IOException {
return FileVisitResult.CONTINUE;
}
@Override
public void initRepo(
RepoStore repoStore
) throws IOException {
initRepositoryField(repoStore);
Preconditions.checkState(repository.isPresent());
Repository repo = this.repository.get();
// TODO: assert that this is a fresh repo. At the moment, we can't be
// sure whether the repo to be init'd doesn't exist or is just fresh
// and we crashed / aborted while committing
if (repo.getObjectDatabase().exists()) return;
repo.create();
}
@Override
public void useExistingRepository(
RepoStore repoStore
) throws IOException {
initRepositoryField(repoStore);
Preconditions.checkState(repository.isPresent());
Preconditions.checkState(
repository.get().getObjectDatabase().exists()
);
}
@Override
public RawDirectory getDirectory()
throws IOException, GitUserException {
Preconditions.checkState(repository.isPresent());
return new RepositoryObjectTreeWalker(
repository.get()
).getDirectoryContents(Optional.empty());
}
@Override
public Collection<String> commitAndGetMissing(
GitDirectoryContents contents
) throws IOException {
try {
return doCommitAndGetMissing(contents);
} catch (GitAPIException e) {
throw new IOException(e);
}
}
@Override
public void runGC() throws IOException {
Preconditions.checkState(
repository.isPresent(),
"Repo is not present"
);
File dir = getProjectDir();
Preconditions.checkState(dir.isDirectory());
Log.debug("[{}] Running git gc", projectName);
Process proc = new ProcessBuilder(
"git", "gc"
).directory(dir).start();
int exitCode;
try {
exitCode = proc.waitFor();
Log.debug("Exit: {}", exitCode);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
if (exitCode != 0) {
Log.warn("[{}] Git gc failed", dir.getAbsolutePath());
Log.warn(IOUtils.toString(
proc.getInputStream(),
StandardCharsets.UTF_8
));
Log.warn(IOUtils.toString(
proc.getErrorStream(),
StandardCharsets.UTF_8
));
throw new IOException("git gc error");
}
Log.debug("[{}] git gc successful", projectName);
}
@Override
public void deleteIncomingPacks() throws IOException {
Log.debug(
"[{}] Checking for garbage `incoming` files",
projectName
);
Files.walkFileTree(getDotGitDir().toPath(), new FileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(
Path dir,
BasicFileAttributes attrs
) throws IOException {
return FileVisitResult.CONTINUE;
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
File file_ = file.toFile();
String name = file_.getName();
if (name.startsWith("incoming_") && name.endsWith(".pack")) {
Log.debug("Deleting garbage `incoming` file: {}", file_);
Preconditions.checkState(file_.delete());
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(
Path file,
BasicFileAttributes attrs
) throws IOException {
File file_ = file.toFile();
String name = file_.getName();
if (name.startsWith("incoming_") && name.endsWith(".pack")) {
Log.debug("Deleting garbage `incoming` file: {}", file_);
Preconditions.checkState(file_.delete());
}
return FileVisitResult.CONTINUE;
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
Preconditions.checkNotNull(file);
Preconditions.checkNotNull(exc);
Log.warn("Failed to visit file: " + file, exc);
return FileVisitResult.TERMINATE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Preconditions.checkNotNull(dir);
if (exc != null) {
return FileVisitResult.TERMINATE;
}
@Override
public FileVisitResult visitFileFailed(
Path file,
IOException exc
) throws IOException {
Preconditions.checkNotNull(file);
Preconditions.checkNotNull(exc);
Log.warn("Failed to visit file: " + file, exc);
return FileVisitResult.TERMINATE;
}
@Override
public FileVisitResult postVisitDirectory(
Path dir,
IOException exc
) throws IOException {
Preconditions.checkNotNull(dir);
if (exc != null) {
return FileVisitResult.TERMINATE;
}
return FileVisitResult.CONTINUE;
}
return FileVisitResult.CONTINUE;
}
});
}
}
@Override
public File getProjectDir() {
return getJGitRepository().getDirectory().getParentFile();
}
@Override
public File getProjectDir() {
return getJGitRepository().getDirectory().getParentFile();
}
public void resetHard() throws IOException {
Git git = new Git(getJGitRepository());
try {
git.reset().setMode(ResetCommand.ResetType.HARD).call();
} catch (GitAPIException e) {
throw new IOException(e);
}
public void resetHard() throws IOException {
Git git = new Git(getJGitRepository());
try {
git.reset().setMode(ResetCommand.ResetType.HARD).call();
} catch (GitAPIException e) {
throw new IOException(e);
}
}
@Override
public Repository getJGitRepository() {
return repository.get();
@Override
public Repository getJGitRepository() {
return repository.get();
}
public File getDotGitDir() {
return getJGitRepository().getWorkTree();
}
private void initRepositoryField(RepoStore repoStore) throws IOException {
Preconditions.checkNotNull(repoStore);
Preconditions.checkArgument(Project.isValidProjectName(projectName));
Preconditions.checkState(!repository.isPresent());
repository = Optional.of(createJGitRepository(repoStore, projectName));
}
private Repository createJGitRepository(RepoStore repoStore, String projName) throws IOException {
File repoDir = new File(repoStore.getRootDirectory(), projName);
return new FileRepositoryBuilder().setWorkTree(repoDir).build();
}
private Collection<String> doCommitAndGetMissing(GitDirectoryContents contents)
throws IOException, GitAPIException {
Preconditions.checkState(repository.isPresent());
Repository repo = getJGitRepository();
resetHard();
String name = getProjectName();
Log.debug("[{}] Writing commit", name);
contents.write();
Git git = new Git(getJGitRepository());
Log.debug("[{}] Getting missing files", name);
Set<String> missingFiles = git.status().call().getMissing();
for (String missing : missingFiles) {
Log.debug("[{}] Git rm {}", name, missing);
git.rm().setCached(true).addFilepattern(missing).call();
}
public File getDotGitDir() {
return getJGitRepository().getWorkTree();
}
private void initRepositoryField(RepoStore repoStore) throws IOException {
Preconditions.checkNotNull(repoStore);
Preconditions.checkArgument(Project.isValidProjectName(projectName));
Preconditions.checkState(!repository.isPresent());
repository = Optional.of(createJGitRepository(repoStore, projectName));
}
private Repository createJGitRepository(
RepoStore repoStore,
String projName
) throws IOException {
File repoDir = new File(repoStore.getRootDirectory(), projName);
return new FileRepositoryBuilder().setWorkTree(repoDir).build();
}
private Collection<String> doCommitAndGetMissing(
GitDirectoryContents contents
) throws IOException, GitAPIException {
Preconditions.checkState(repository.isPresent());
Repository repo = getJGitRepository();
resetHard();
String name = getProjectName();
Log.debug("[{}] Writing commit", name);
contents.write();
Git git = new Git(getJGitRepository());
Log.debug("[{}] Getting missing files", name);
Set<String> missingFiles = git.status().call().getMissing();
for (String missing : missingFiles) {
Log.debug("[{}] Git rm {}", name, missing);
git.rm().setCached(true).addFilepattern(missing).call();
}
Log.debug("[{}] Calling Git add", name);
git.add(
).setWorkingTreeIterator(
new NoGitignoreIterator(repo)
).addFilepattern(".").call();
Log.debug("[{}] Calling Git commit", name);
git.commit(
).setAuthor(
new PersonIdent(
contents.getUserName(),
contents.getUserEmail(),
contents.getWhen(),
TimeZone.getDefault()
)
).setMessage(
contents.getCommitMessage()
).call();
Log.debug(
"[{}] Deleting files in directory: {}",
name,
contents.getDirectory().getAbsolutePath()
);
Util.deleteInDirectoryApartFrom(contents.getDirectory(), ".git");
return missingFiles;
}
Log.debug("[{}] Calling Git add", name);
git.add().setWorkingTreeIterator(new NoGitignoreIterator(repo)).addFilepattern(".").call();
Log.debug("[{}] Calling Git commit", name);
git.commit()
.setAuthor(
new PersonIdent(
contents.getUserName(),
contents.getUserEmail(),
contents.getWhen(),
TimeZone.getDefault()))
.setMessage(contents.getCommitMessage())
.call();
Log.debug(
"[{}] Deleting files in directory: {}", name, contents.getDirectory().getAbsolutePath());
Util.deleteInDirectoryApartFrom(contents.getDirectory(), ".git");
return missingFiles;
}
}

110
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/NoGitignoreIterator.java
View File

@@ -1,5 +1,7 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import java.io.File;
import java.lang.reflect.Field;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.treewalk.AbstractTreeIterator;
import org.eclipse.jgit.treewalk.FileTreeIterator;
@@ -7,81 +9,65 @@ import org.eclipse.jgit.treewalk.WorkingTreeIterator;
import org.eclipse.jgit.treewalk.WorkingTreeOptions;
import org.eclipse.jgit.util.FS;
import java.io.File;
import java.lang.reflect.Field;
/**
/*
* Created by winston on 08/10/2016.
*/
public class NoGitignoreIterator extends FileTreeIterator {
private static final Field ignoreNodeField;
private static final Field ignoreNodeField;
static {
try {
ignoreNodeField = WorkingTreeIterator.class.getDeclaredField(
"ignoreNode"
);
} catch (NoSuchFieldException e) {
throw new RuntimeException(e);
}
ignoreNodeField.setAccessible(true);
static {
try {
ignoreNodeField = WorkingTreeIterator.class.getDeclaredField("ignoreNode");
} catch (NoSuchFieldException e) {
throw new RuntimeException(e);
}
ignoreNodeField.setAccessible(true);
}
public NoGitignoreIterator(Repository repo) {
super(repo);
}
public NoGitignoreIterator(Repository repo) {
super(repo);
}
public NoGitignoreIterator(
Repository repo,
FileModeStrategy fileModeStrategy
) {
super(repo, fileModeStrategy);
}
public NoGitignoreIterator(Repository repo, FileModeStrategy fileModeStrategy) {
super(repo, fileModeStrategy);
}
public NoGitignoreIterator(File root, FS fs, WorkingTreeOptions options) {
super(root, fs, options);
}
public NoGitignoreIterator(File root, FS fs, WorkingTreeOptions options) {
super(root, fs, options);
}
public NoGitignoreIterator(
File root,
FS fs,
WorkingTreeOptions options,
FileModeStrategy fileModeStrategy
) {
super(root, fs, options, fileModeStrategy);
}
public NoGitignoreIterator(
File root, FS fs, WorkingTreeOptions options, FileModeStrategy fileModeStrategy) {
super(root, fs, options, fileModeStrategy);
}
protected NoGitignoreIterator(FileTreeIterator p, File root, FS fs) {
super(p, root, fs);
}
protected NoGitignoreIterator(FileTreeIterator p, File root, FS fs) {
super(p, root, fs);
}
protected NoGitignoreIterator(
WorkingTreeIterator p,
File root,
FS fs,
FileModeStrategy fileModeStrategy
) {
super(p, root, fs, fileModeStrategy);
}
protected NoGitignoreIterator(
WorkingTreeIterator p, File root, FS fs, FileModeStrategy fileModeStrategy) {
super(p, root, fs, fileModeStrategy);
}
// Note: the `list` is a list of top-level entities in this directory,
// not a full list of files in the tree.
@Override
protected void init(Entry[] list) {
super.init(list);
try {
ignoreNodeField.set(this, null);
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
// Note: the `list` is a list of top-level entities in this directory,
// not a full list of files in the tree.
@Override
protected void init(Entry[] list) {
super.init(list);
try {
ignoreNodeField.set(this, null);
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
}
// When entering a sub-directory, create a new instance of this class,
// so we can also ignore gitignore specifications in sub-directories
@Override
protected AbstractTreeIterator enterSubtree() {
String fullPath = getDirectory().getAbsolutePath() + "/" + current().getName();
return new NoGitignoreIterator(this, new File(fullPath), fs, fileModeStrategy);
}
// When entering a sub-directory, create a new instance of this class,
// so we can also ignore gitignore specifications in sub-directories
@Override
protected AbstractTreeIterator enterSubtree() {
String fullPath = getDirectory().getAbsolutePath() + "/" + current().getName();
return new NoGitignoreIterator(this, new File(fullPath), fs, fileModeStrategy);
}
}

35
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/ProjectRepo.java
View File

@@ -1,41 +1,34 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import org.eclipse.jgit.lib.Repository;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
/**
/*
* Created by winston on 20/08/2016.
*/
public interface ProjectRepo {
String getProjectName();
String getProjectName();
void initRepo(
RepoStore repoStore
) throws IOException;
void initRepo(RepoStore repoStore) throws IOException;
void useExistingRepository(
RepoStore repoStore
) throws IOException;
void useExistingRepository(RepoStore repoStore) throws IOException;
RawDirectory getDirectory(
) throws IOException, GitUserException;
RawDirectory getDirectory() throws IOException, GitUserException;
Collection<String> commitAndGetMissing(
GitDirectoryContents gitDirectoryContents
) throws IOException, GitUserException;
Collection<String> commitAndGetMissing(GitDirectoryContents gitDirectoryContents)
throws IOException, GitUserException;
void runGC() throws IOException;
void runGC() throws IOException;
void deleteIncomingPacks() throws IOException;
void deleteIncomingPacks() throws IOException;
File getProjectDir();
File getProjectDir();
Repository getJGitRepository();
Repository getJGitRepository();
}

132
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java
View File

@@ -1,102 +1,82 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
/**
/*
* Created by winston on 20/08/2016.
*/
public interface RepoStore {
/* Still need to get rid of these two methods.
Main dependency: GitRepoStore needs a Repository which needs a directory.
Instead, use a visitor or something. */
String getRepoStorePath();
/* Still need to get rid of these two methods.
Main dependency: GitRepoStore needs a Repository which needs a directory.
Instead, use a visitor or something. */
String getRepoStorePath();
File getRootDirectory();
File getRootDirectory();
ProjectRepo initRepo(String project) throws IOException;
ProjectRepo initRepo(String project) throws IOException;
ProjectRepo getExistingRepo(String project) throws IOException;
ProjectRepo getExistingRepo(String project) throws IOException;
ProjectRepo useJGitRepo(Repository repo, ObjectId commitId);
ProjectRepo useJGitRepo(Repository repo, ObjectId commitId);
void purgeNonexistentProjects(
Collection<String> existingProjectNames
);
void purgeNonexistentProjects(Collection<String> existingProjectNames);
long totalSize();
long totalSize();
/**
* Tars and bzip2s the .git directory of the given project. Throws an
* IOException if the project doesn't exist. The returned stream is a copy
* of the original .git directory, which must be deleted using remove().
*/
InputStream bzip2Project(
String projectName,
long[] sizePtr
) throws IOException;
/*
* Tars and bzip2s the .git directory of the given project. Throws an
* IOException if the project doesn't exist. The returned stream is a copy
* of the original .git directory, which must be deleted using remove().
*/
InputStream bzip2Project(String projectName, long[] sizePtr) throws IOException;
default InputStream bzip2Project(
String projectName
) throws IOException {
return bzip2Project(projectName, null);
}
default InputStream bzip2Project(String projectName) throws IOException {
return bzip2Project(projectName, null);
}
/**
* Tars and gzips the .git directory of the given project. Throws an
* IOException if the project doesn't exist. The returned stream is a copy
* of the original .git directory, which must be deleted using remove().
*/
InputStream gzipProject(
String projectName,
long[] sizePtr
) throws IOException;
/*
* Tars and gzips the .git directory of the given project. Throws an
* IOException if the project doesn't exist. The returned stream is a copy
* of the original .git directory, which must be deleted using remove().
*/
InputStream gzipProject(String projectName, long[] sizePtr) throws IOException;
default InputStream gzipProject(
String projectName
) throws IOException {
return gzipProject(projectName, null);
}
default InputStream gzipProject(String projectName) throws IOException {
return gzipProject(projectName, null);
}
void gcProject(String projectName) throws IOException;
void gcProject(String projectName) throws IOException;
/**
* Called after {@link #bzip2Project(String, long[])}'s has been safely
* uploaded to the swap store. Removes all traces of the project from disk,
* i.e. not just its .git, but the whole project's git directory.
* @param projectName
* @throws IOException
*/
void remove(String projectName) throws IOException;
/*
* Called after {@link #bzip2Project(String, long[])}'s has been safely
* uploaded to the swap store. Removes all traces of the project from disk,
* i.e. not just its .git, but the whole project's git directory.
* @param projectName
* @throws IOException
*/
void remove(String projectName) throws IOException;
/**
* Unbzip2s the given data stream into a .git directory for projectName.
* Creates the project's git directory.
* If projectName already exists, throws an IOException.
* @param projectName the name of the project, e.g. abc123
* @param dataStream the data stream containing the bzipped contents.
*/
void unbzip2Project(
String projectName,
InputStream dataStream
) throws IOException;
/**
* Ungzips the given data stream into a .git directory for projectName.
* Creates the project's git directory.
* If projectName already exists, throws an IOException.
* @param projectName the name of the project, e.g. abc123
* @param dataStream the data stream containing the gzip contents.
*/
void ungzipProject(
String projectName,
InputStream dataStream
) throws IOException;
/*
* Unbzip2s the given data stream into a .git directory for projectName.
* Creates the project's git directory.
* If projectName already exists, throws an IOException.
* @param projectName the name of the project, e.g. abc123
* @param dataStream the data stream containing the bzipped contents.
*/
void unbzip2Project(String projectName, InputStream dataStream) throws IOException;
/*
* Ungzips the given data stream into a .git directory for projectName.
* Creates the project's git directory.
* If projectName already exists, throws an IOException.
* @param projectName the name of the project, e.g. abc123
* @param dataStream the data stream containing the gzip contents.
*/
void ungzipProject(String projectName, InputStream dataStream) throws IOException;
}

30
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStoreConfig.java
View File

@@ -1,29 +1,27 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import javax.annotation.Nullable;
import java.util.Optional;
import javax.annotation.Nullable;
/**
/*
* Created by winston on 02/07/2017.
*/
public class RepoStoreConfig {
@Nullable
private final Long maxFileSize;
@Nullable private final Long maxFileSize;
@Nullable
private final Long maxFileNum;
@Nullable private final Long maxFileNum;
public RepoStoreConfig(Long maxFileSize, Long maxFileNum) {
this.maxFileSize = maxFileSize;
this.maxFileNum = maxFileNum;
}
public RepoStoreConfig(Long maxFileSize, Long maxFileNum) {
this.maxFileSize = maxFileSize;
this.maxFileNum = maxFileNum;
}
public Optional<Long> getMaxFileSize() {
return Optional.ofNullable(maxFileSize);
}
public Optional<Long> getMaxFileSize() {
return Optional.ofNullable(maxFileSize);
}
public Optional<Long> getMaxFileNum() {
return Optional.ofNullable(maxFileNum);
}
public Optional<Long> getMaxFileNum() {
return Optional.ofNullable(maxFileNum);
}
}

128
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/WalkOverrideGitRepo.java
View File

@@ -1,5 +1,9 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.Optional;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
@@ -7,12 +11,7 @@ import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.Optional;
/**
/*
* This class takes a GitProjectRepo and delegates all calls to it.
*
* The purpose is to insert a file size check in {@link #getDirectory()}.
@@ -22,74 +21,69 @@ import java.util.Optional;
*/
public class WalkOverrideGitRepo implements ProjectRepo {
private final GitProjectRepo gitRepo;
private final GitProjectRepo gitRepo;
private final Optional<Long> maxFileSize;
private final Optional<Long> maxFileSize;
private final Optional<ObjectId> commitId;
private final Optional<ObjectId> commitId;
public WalkOverrideGitRepo(
GitProjectRepo gitRepo,
Optional<Long> maxFileSize,
Optional<ObjectId> commitId
) {
this.gitRepo = gitRepo;
this.maxFileSize = maxFileSize;
this.commitId = commitId;
public WalkOverrideGitRepo(
GitProjectRepo gitRepo, Optional<Long> maxFileSize, Optional<ObjectId> commitId) {
this.gitRepo = gitRepo;
this.maxFileSize = maxFileSize;
this.commitId = commitId;
}
@Override
public String getProjectName() {
return gitRepo.getProjectName();
}
@Override
public void initRepo(RepoStore repoStore) throws IOException {
gitRepo.initRepo(repoStore);
}
@Override
public void useExistingRepository(RepoStore repoStore) throws IOException {
gitRepo.useExistingRepository(repoStore);
}
@Override
public RawDirectory getDirectory() throws IOException, GitUserException {
Repository repo = gitRepo.getJGitRepository();
RepositoryObjectTreeWalker walker;
if (commitId.isPresent()) {
walker = new RepositoryObjectTreeWalker(repo, commitId.get());
} else {
walker = new RepositoryObjectTreeWalker(repo);
}
return walker.getDirectoryContents(maxFileSize);
}
@Override
public String getProjectName() {
return gitRepo.getProjectName();
}
@Override
public Collection<String> commitAndGetMissing(GitDirectoryContents gitDirectoryContents)
throws GitUserException, IOException {
return gitRepo.commitAndGetMissing(gitDirectoryContents);
}
@Override
public void initRepo(RepoStore repoStore) throws IOException {
gitRepo.initRepo(repoStore);
}
@Override
public void runGC() throws IOException {
gitRepo.runGC();
}
@Override
public void useExistingRepository(RepoStore repoStore) throws IOException {
gitRepo.useExistingRepository(repoStore);
}
@Override
public void deleteIncomingPacks() throws IOException {
gitRepo.deleteIncomingPacks();
}
@Override
public RawDirectory getDirectory() throws IOException, GitUserException {
Repository repo = gitRepo.getJGitRepository();
RepositoryObjectTreeWalker walker;
if (commitId.isPresent()) {
walker = new RepositoryObjectTreeWalker(repo, commitId.get());
} else {
walker = new RepositoryObjectTreeWalker(repo);
}
return walker.getDirectoryContents(maxFileSize);
}
@Override
public Collection<String> commitAndGetMissing(
GitDirectoryContents gitDirectoryContents
) throws GitUserException, IOException {
return gitRepo.commitAndGetMissing(gitDirectoryContents);
}
@Override
public void runGC() throws IOException {
gitRepo.runGC();
}
@Override
public void deleteIncomingPacks() throws IOException {
gitRepo.deleteIncomingPacks();
}
@Override
public File getProjectDir() {
return gitRepo.getProjectDir();
}
@Override
public Repository getJGitRepository() {
return gitRepo.getJGitRepository();
}
@Override
public File getProjectDir() {
return gitRepo.getProjectDir();
}
@Override
public Repository getJGitRepository() {
return gitRepo.getJGitRepository();
}
}

24
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/ResourceCache.java
View File

@@ -1,24 +1,22 @@
package uk.ac.ic.wlgitbridge.bridge.resource;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
import java.io.IOException;
import java.util.Map;
import java.util.Optional;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
/**
/*
* Created by winston on 20/08/2016.
*/
public interface ResourceCache {
RawFile get(
String projectName,
String url,
String newPath,
Map<String, RawFile> fileTable,
Map<String, byte[]> fetchedUrls,
Optional<Long> maxFileSize
) throws IOException, SizeLimitExceededException;
RawFile get(
String projectName,
String url,
String newPath,
Map<String, RawFile> fileTable,
Map<String, byte[]> fetchedUrls,
Optional<Long> maxFileSize)
throws IOException, SizeLimitExceededException;
}

207
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/UrlResourceCache.java
View File

@@ -1,6 +1,12 @@
package uk.ac.ic.wlgitbridge.bridge.resource;
import static org.asynchttpclient.Dsl.*;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile;
@@ -10,133 +16,124 @@ import uk.ac.ic.wlgitbridge.io.http.ning.NingHttpClientFacade;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
/**
/*
* Created by winston on 20/08/2016.
*/
public class UrlResourceCache implements ResourceCache {
private final DBStore dbStore;
private final DBStore dbStore;
private final NingHttpClientFacade http;
private final NingHttpClientFacade http;
UrlResourceCache(DBStore dbStore, NingHttpClientFacade http) {
this.dbStore = dbStore;
this.http = http;
}
UrlResourceCache(DBStore dbStore, NingHttpClientFacade http) {
this.dbStore = dbStore;
this.http = http;
}
public UrlResourceCache(DBStore dbStore) {
this(dbStore, new NingHttpClient(asyncHttpClient()));
}
public UrlResourceCache(DBStore dbStore) {
this(dbStore, new NingHttpClient(asyncHttpClient()));
}
@Override
public RawFile get(
String projectName,
String url,
String newPath,
Map<String, RawFile> fileTable,
Map<String, byte[]> fetchedUrls,
Optional<Long> maxFileSize
) throws IOException, SizeLimitExceededException {
String path = dbStore.getPathForURLInProject(projectName, getCacheKeyFromUrl(url));
byte[] contents;
if (path == null) {
path = newPath;
contents = fetch(projectName, url, path, maxFileSize);
fetchedUrls.put(url, contents);
@Override
public RawFile get(
String projectName,
String url,
String newPath,
Map<String, RawFile> fileTable,
Map<String, byte[]> fetchedUrls,
Optional<Long> maxFileSize)
throws IOException, SizeLimitExceededException {
String path = dbStore.getPathForURLInProject(projectName, getCacheKeyFromUrl(url));
byte[] contents;
if (path == null) {
path = newPath;
contents = fetch(projectName, url, path, maxFileSize);
fetchedUrls.put(url, contents);
} else {
Log.debug("Found (" + projectName + "): " + url);
Log.debug("At (" + projectName + "): " + path);
contents = fetchedUrls.get(url);
if (contents == null) {
RawFile rawFile = fileTable.get(path);
if (rawFile == null) {
Log.warn(
"File "
+ path
+ " was not in the current commit, "
+ "or the git tree, yet path was not null. "
+ "File url is: "
+ url);
contents = fetch(projectName, url, path, maxFileSize);
} else {
Log.debug("Found (" + projectName + "): " + url);
Log.debug("At (" + projectName + "): " + path);
contents = fetchedUrls.get(url);
if (contents == null) {
RawFile rawFile = fileTable.get(path);
if (rawFile == null) {
Log.warn(
"File " + path
+ " was not in the current commit, "
+ "or the git tree, yet path was not null. "
+ "File url is: "
+ url
);
contents = fetch(projectName, url, path, maxFileSize);
} else {
contents = rawFile.getContents();
}
}
contents = rawFile.getContents();
}
return new RepositoryFile(newPath, contents);
}
}
return new RepositoryFile(newPath, contents);
}
private byte[] fetch(
String projectName,
final String url,
String path,
Optional<Long> maxFileSize
) throws FailedConnectionException, SizeLimitExceededException {
byte[] contents;
Log.debug("GET -> " + url);
try {
contents = http.get(url, hs -> {
private byte[] fetch(
String projectName, final String url, String path, Optional<Long> maxFileSize)
throws FailedConnectionException, SizeLimitExceededException {
byte[] contents;
Log.debug("GET -> " + url);
try {
contents =
http.get(
url,
hs -> {
List<String> contentLengths = hs.getAll("Content-Length");
if (!maxFileSize.isPresent()) {
return true;
return true;
}
if (contentLengths.isEmpty()) {
return true;
return true;
}
long contentLength = Long.parseLong(contentLengths.get(0));
long maxFileSize_ = maxFileSize.get();
if (contentLength <= maxFileSize_) {
return true;
return true;
}
throw new SizeLimitExceededException(
Optional.of(path), contentLength, maxFileSize_
);
});
} catch (ExecutionException e) {
Throwable cause = e.getCause();
if (cause instanceof SizeLimitExceededException) {
throw (SizeLimitExceededException) cause;
}
Log.warn(
"ExecutionException when fetching project: " +
projectName +
", url: " +
url +
", path: " +
path,
e
);
throw new FailedConnectionException();
}
if (maxFileSize.isPresent() && contents.length > maxFileSize.get()) {
throw new SizeLimitExceededException(
Optional.of(path), contents.length, maxFileSize.get());
}
dbStore.addURLIndexForProject(projectName, getCacheKeyFromUrl(url), path);
return contents;
Optional.of(path), contentLength, maxFileSize_);
});
} catch (ExecutionException e) {
Throwable cause = e.getCause();
if (cause instanceof SizeLimitExceededException) {
throw (SizeLimitExceededException) cause;
}
Log.warn(
"ExecutionException when fetching project: "
+ projectName
+ ", url: "
+ url
+ ", path: "
+ path,
e);
throw new FailedConnectionException();
}
if (maxFileSize.isPresent() && contents.length > maxFileSize.get()) {
throw new SizeLimitExceededException(Optional.of(path), contents.length, maxFileSize.get());
}
dbStore.addURLIndexForProject(projectName, getCacheKeyFromUrl(url), path);
return contents;
}
/**
* Construct a suitable cache key from the given file URL.
*
* The file URL returned by the web service may contain a token parameter
* used for authentication. This token changes for every request, so we
* need to strip it from the query string before using the URL as a cache
* key.
*/
private String getCacheKeyFromUrl(String url) {
// We're not doing proper URL parsing here, but it should be enough to
// remove the token without touching the important parts of the URL.
//
// The URL looks like:
//
// https://history.overleaf.com/api/projects/:project_id/blobs/:hash?token=:token&_path=:path
return url.replaceAll("token=[^&]*", "token=REMOVED");
}
/*
* Construct a suitable cache key from the given file URL.
*
* The file URL returned by the web service may contain a token parameter
* used for authentication. This token changes for every request, so we
* need to strip it from the query string before using the URL as a cache
* key.
*/
private String getCacheKeyFromUrl(String url) {
// We're not doing proper URL parsing here, but it should be enough to
// remove the token without touching the important parts of the URL.
//
// The URL looks like:
//
// https://history.overleaf.com/api/projects/:project_id/blobs/:hash?token=:token&_path=:path
return url.replaceAll("token=[^&]*", "token=REMOVED");
}
}

58
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/NetSnapshotApi.java
View File

@@ -1,6 +1,8 @@
package uk.ac.ic.wlgitbridge.bridge.snapshot;
import com.google.api.client.auth.oauth2.Credential;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
@@ -11,45 +13,35 @@ import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersResult;
import uk.ac.ic.wlgitbridge.snapshot.push.PushRequest;
import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
/**
/*
* Created by winston on 20/08/2016.
*/
public class NetSnapshotApi implements SnapshotApi {
@Override
public CompletableFuture<GetDocResult> getDoc(
Optional<Credential> oauth2, String projectName) {
return new GetDocRequest(opt(oauth2), projectName).request();
}
@Override
public CompletableFuture<GetDocResult> getDoc(Optional<Credential> oauth2, String projectName) {
return new GetDocRequest(opt(oauth2), projectName).request();
}
@Override
public CompletableFuture<GetForVersionResult> getForVersion(
Optional<Credential> oauth2, String projectName, int versionId) {
return new GetForVersionRequest(
opt(oauth2), projectName, versionId).request();
}
@Override
public CompletableFuture<GetForVersionResult> getForVersion(
Optional<Credential> oauth2, String projectName, int versionId) {
return new GetForVersionRequest(opt(oauth2), projectName, versionId).request();
}
@Override
public CompletableFuture<GetSavedVersResult> getSavedVers(
Optional<Credential> oauth2, String projectName) {
return new GetSavedVersRequest(opt(oauth2), projectName).request();
}
@Override
public CompletableFuture<GetSavedVersResult> getSavedVers(
Optional<Credential> oauth2, String projectName) {
return new GetSavedVersRequest(opt(oauth2), projectName).request();
}
@Override
public CompletableFuture<PushResult> push(
Optional<Credential> oauth2,
CandidateSnapshot candidateSnapshot,
String postbackKey
) {
return new PushRequest(
opt(oauth2), candidateSnapshot, postbackKey).request();
}
private static Credential opt(Optional<Credential> oauth2) {
return oauth2.orElse(null);
}
@Override
public CompletableFuture<PushResult> push(
Optional<Credential> oauth2, CandidateSnapshot candidateSnapshot, String postbackKey) {
return new PushRequest(opt(oauth2), candidateSnapshot, postbackKey).request();
}
private static Credential opt(Optional<Credential> oauth2) {
return oauth2.orElse(null);
}
}

61
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotApi.java
View File

@@ -1,54 +1,49 @@
package uk.ac.ic.wlgitbridge.bridge.snapshot;
import com.google.api.client.auth.oauth2.Credential;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
import uk.ac.ic.wlgitbridge.snapshot.base.MissingRepositoryException;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.snapshot.base.MissingRepositoryException;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionResult;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersResult;
import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
/**
/*
* Created by winston on 20/08/2016.
*/
public interface SnapshotApi {
CompletableFuture<GetDocResult> getDoc(
Optional<Credential> oauth2, String projectName);
CompletableFuture<GetDocResult> getDoc(Optional<Credential> oauth2, String projectName);
CompletableFuture<GetForVersionResult> getForVersion(
Optional<Credential> oauth2, String projectName, int versionId);
CompletableFuture<GetForVersionResult> getForVersion(
Optional<Credential> oauth2, String projectName, int versionId);
CompletableFuture<GetSavedVersResult> getSavedVers(
Optional<Credential> oauth2, String projectName);
CompletableFuture<GetSavedVersResult> getSavedVers(
Optional<Credential> oauth2, String projectName);
CompletableFuture<PushResult> push(
Optional<Credential> oauth2,
CandidateSnapshot candidateSnapshot,
String postbackKey);
CompletableFuture<PushResult> push(
Optional<Credential> oauth2, CandidateSnapshot candidateSnapshot, String postbackKey);
static <T> T getResult(CompletableFuture<T> result)
throws MissingRepositoryException, FailedConnectionException, ForbiddenException {
try {
return result.join();
} catch (CompletionException e) {
try {
throw e.getCause();
} catch (MissingRepositoryException
| FailedConnectionException
| ForbiddenException
| RuntimeException r) {
throw r;
} catch (Throwable __) {
throw e;
}
}
static <T> T getResult(CompletableFuture<T> result)
throws MissingRepositoryException, FailedConnectionException, ForbiddenException {
try {
return result.join();
} catch (CompletionException e) {
try {
throw e.getCause();
} catch (MissingRepositoryException
| FailedConnectionException
| ForbiddenException
| RuntimeException r) {
throw r;
} catch (Throwable __) {
throw e;
}
}
}
}

223
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotApiFacade.java
View File

@@ -1,11 +1,14 @@
package uk.ac.ic.wlgitbridge.bridge.snapshot;
import com.google.api.client.auth.oauth2.Credential;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.snapshot.base.MissingRepositoryException;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.snapshot.base.MissingRepositoryException;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionResult;
@@ -15,148 +18,104 @@ import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.SnapshotInfo;
import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.InvalidProjectException;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
/**
/*
* Created by winston on 02/07/2017.
*/
public class SnapshotApiFacade {
private final SnapshotApi api;
private final SnapshotApi api;
public SnapshotApiFacade(SnapshotApi api) {
this.api = api;
public SnapshotApiFacade(SnapshotApi api) {
this.api = api;
}
public boolean projectExists(Optional<Credential> oauth2, String projectName)
throws FailedConnectionException, GitUserException {
try {
SnapshotApi.getResult(api.getDoc(oauth2, projectName)).getVersionID();
return true;
} catch (InvalidProjectException e) {
return false;
}
}
public boolean projectExists(
Optional<Credential> oauth2,
String projectName
) throws FailedConnectionException, GitUserException {
try {
SnapshotApi
.getResult(api.getDoc(oauth2, projectName))
.getVersionID();
return true;
} catch (InvalidProjectException e) {
return false;
public Optional<GetDocResult> getDoc(Optional<Credential> oauth2, String projectName)
throws FailedConnectionException, GitUserException {
try {
GetDocResult doc = SnapshotApi.getResult(api.getDoc(oauth2, projectName));
doc.getVersionID();
return Optional.of(doc);
} catch (InvalidProjectException e) {
return Optional.empty();
}
}
public Deque<Snapshot> getSnapshots(
Optional<Credential> oauth2, String projectName, int afterVersionId)
throws GitUserException, FailedConnectionException {
List<SnapshotInfo> snapshotInfos =
getSnapshotInfosAfterVersion(oauth2, projectName, afterVersionId);
List<SnapshotData> snapshotDatas = getMatchingSnapshotData(oauth2, projectName, snapshotInfos);
return combine(snapshotInfos, snapshotDatas);
}
public PushResult push(
Optional<Credential> oauth2, CandidateSnapshot candidateSnapshot, String postbackKey)
throws MissingRepositoryException, FailedConnectionException, ForbiddenException {
return SnapshotApi.getResult(api.push(oauth2, candidateSnapshot, postbackKey));
}
private List<SnapshotInfo> getSnapshotInfosAfterVersion(
Optional<Credential> oauth2, String projectName, int version)
throws FailedConnectionException, GitUserException {
SortedSet<SnapshotInfo> versions = new TreeSet<>();
CompletableFuture<GetDocResult> getDoc = api.getDoc(oauth2, projectName);
CompletableFuture<GetSavedVersResult> savedVers = api.getSavedVers(oauth2, projectName);
GetDocResult latestDoc = SnapshotApi.getResult(getDoc);
int latest = latestDoc.getVersionID();
// Handle edge-case for projects with no changes, that were imported
// to v2. In which case both `latest` and `version` will be zero.
// See: https://github.com/overleaf/writelatex-git-bridge/pull/50
if (latest > version || (latest == 0 && version == 0)) {
for (SnapshotInfo snapshotInfo : SnapshotApi.getResult(savedVers).getSavedVers()) {
if (snapshotInfo.getVersionId() > version) {
versions.add(snapshotInfo);
}
}
versions.add(
new SnapshotInfo(
latest, latestDoc.getCreatedAt(), latestDoc.getName(), latestDoc.getEmail()));
}
return new ArrayList<>(versions);
}
public Optional<GetDocResult> getDoc(
Optional<Credential> oauth2,
String projectName
) throws FailedConnectionException, GitUserException {
try {
GetDocResult doc = SnapshotApi
.getResult(api.getDoc(oauth2, projectName));
doc.getVersionID();
return Optional.of(doc);
} catch (InvalidProjectException e) {
return Optional.empty();
}
private List<SnapshotData> getMatchingSnapshotData(
Optional<Credential> oauth2, String projectName, List<SnapshotInfo> snapshotInfos)
throws FailedConnectionException, ForbiddenException {
List<CompletableFuture<GetForVersionResult>> firedRequests =
fireDataRequests(oauth2, projectName, snapshotInfos);
List<SnapshotData> snapshotDataList = new ArrayList<>();
for (CompletableFuture<GetForVersionResult> fired : firedRequests) {
snapshotDataList.add(fired.join().getSnapshotData());
}
return snapshotDataList;
}
public Deque<Snapshot> getSnapshots(
Optional<Credential> oauth2,
String projectName,
int afterVersionId
) throws GitUserException, FailedConnectionException {
List<SnapshotInfo> snapshotInfos = getSnapshotInfosAfterVersion(
oauth2,
projectName,
afterVersionId
);
List<SnapshotData> snapshotDatas = getMatchingSnapshotData(
oauth2,
projectName,
snapshotInfos
);
return combine(snapshotInfos, snapshotDatas);
private List<CompletableFuture<GetForVersionResult>> fireDataRequests(
Optional<Credential> oauth2, String projectName, List<SnapshotInfo> snapshotInfos) {
return snapshotInfos.stream()
.map(snap -> api.getForVersion(oauth2, projectName, snap.getVersionId()))
.collect(Collectors.toList());
}
private static Deque<Snapshot> combine(
List<SnapshotInfo> snapshotInfos, List<SnapshotData> snapshotDatas) {
Deque<Snapshot> snapshots = new LinkedList<>();
Iterator<SnapshotInfo> infos = snapshotInfos.iterator();
Iterator<SnapshotData> datas = snapshotDatas.iterator();
while (infos.hasNext()) {
snapshots.add(new Snapshot(infos.next(), datas.next()));
}
public PushResult push(
Optional<Credential> oauth2,
CandidateSnapshot candidateSnapshot,
String postbackKey
) throws MissingRepositoryException, FailedConnectionException, ForbiddenException {
return SnapshotApi.getResult(api.push(
oauth2, candidateSnapshot, postbackKey));
}
private List<SnapshotInfo> getSnapshotInfosAfterVersion(
Optional<Credential> oauth2,
String projectName,
int version
) throws FailedConnectionException, GitUserException {
SortedSet<SnapshotInfo> versions = new TreeSet<>();
CompletableFuture<GetDocResult> getDoc
= api.getDoc(oauth2, projectName);
CompletableFuture<GetSavedVersResult> savedVers
= api.getSavedVers(oauth2, projectName);
GetDocResult latestDoc = SnapshotApi.getResult(getDoc);
int latest = latestDoc.getVersionID();
// Handle edge-case for projects with no changes, that were imported
// to v2. In which case both `latest` and `version` will be zero.
// See: https://github.com/overleaf/writelatex-git-bridge/pull/50
if (latest > version || (latest == 0 && version == 0)) {
for (
SnapshotInfo snapshotInfo :
SnapshotApi.getResult(savedVers).getSavedVers()
) {
if (snapshotInfo.getVersionId() > version) {
versions.add(snapshotInfo);
}
}
versions.add(new SnapshotInfo(
latest,
latestDoc.getCreatedAt(),
latestDoc.getName(),
latestDoc.getEmail()
));
}
return new ArrayList<>(versions);
}
private List<SnapshotData> getMatchingSnapshotData(
Optional<Credential> oauth2,
String projectName,
List<SnapshotInfo> snapshotInfos
) throws FailedConnectionException, ForbiddenException {
List<CompletableFuture<GetForVersionResult>> firedRequests
= fireDataRequests(oauth2, projectName, snapshotInfos);
List<SnapshotData> snapshotDataList = new ArrayList<>();
for (CompletableFuture<GetForVersionResult> fired : firedRequests) {
snapshotDataList.add(fired.join().getSnapshotData());
}
return snapshotDataList;
}
private List<CompletableFuture<GetForVersionResult>> fireDataRequests(
Optional<Credential> oauth2,
String projectName,
List<SnapshotInfo> snapshotInfos
) {
return snapshotInfos
.stream()
.map(snap -> api.getForVersion(
oauth2, projectName, snap.getVersionId()))
.collect(Collectors.toList());
}
private static Deque<Snapshot> combine(
List<SnapshotInfo> snapshotInfos,
List<SnapshotData> snapshotDatas
) {
Deque<Snapshot> snapshots = new LinkedList<>();
Iterator<SnapshotInfo> infos = snapshotInfos.iterator();
Iterator<SnapshotData> datas = snapshotDatas.iterator();
while (infos.hasNext()) {
snapshots.add(new Snapshot(infos.next(), datas.next()));
}
return snapshots;
}
return snapshots;
}
}

19
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/NoopSwapJob.java
View File

@@ -1,20 +1,19 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
/**
/*
* Created by winston on 24/08/2016.
*/
public class NoopSwapJob implements SwapJob {
@Override
public void start() {}
@Override
public void start() {}
@Override
public void stop() {}
@Override
public void stop() {}
@Override
public void evict(String projName) {}
@Override
public void restore(String projName) {}
@Override
public void evict(String projName) {}
@Override
public void restore(String projName) {}
}

207
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java
View File

@@ -1,129 +1,124 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
import java.io.IOException;
import java.util.Optional;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.IOException;
import java.util.Optional;
/**
/*
* Created by winston on 20/08/2016.
*/
public interface SwapJob {
enum CompressionMethod { Bzip2, Gzip }
enum CompressionMethod {
Bzip2,
Gzip
}
static CompressionMethod stringToCompressionMethod(String compressionString) {
if (compressionString == null) {
return null;
}
CompressionMethod result;
switch (compressionString) {
case "gzip":
result = CompressionMethod.Gzip;
break;
case "bzip2":
result = CompressionMethod.Bzip2;
break;
default:
result = null;
break;
}
return result;
static CompressionMethod stringToCompressionMethod(String compressionString) {
if (compressionString == null) {
return null;
}
static String compressionMethodAsString(CompressionMethod compressionMethod) {
if (compressionMethod == null) {
return null;
}
String result;
switch (compressionMethod) {
case Gzip:
result = "gzip";
break;
case Bzip2:
result = "bzip2";
break;
default:
result = null;
break;
}
return result;
CompressionMethod result;
switch (compressionString) {
case "gzip":
result = CompressionMethod.Gzip;
break;
case "bzip2":
result = CompressionMethod.Bzip2;
break;
default:
result = null;
break;
}
return result;
}
static SwapJob fromConfig(
Optional<SwapJobConfig> cfg,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore
) {
if (!cfg.isPresent()) {
return new NoopSwapJob();
}
if (!swapStore.isSafe() && !cfg.get().getAllowUnsafeStores()) {
Log.warn("Swap store '{}' is not safe; disabling swap job", swapStore.getClass().getSimpleName());
return new NoopSwapJob();
}
return new SwapJobImpl(
cfg.get(),
lock,
repoStore,
dbStore,
swapStore
);
static String compressionMethodAsString(CompressionMethod compressionMethod) {
if (compressionMethod == null) {
return null;
}
String result;
switch (compressionMethod) {
case Gzip:
result = "gzip";
break;
case Bzip2:
result = "bzip2";
break;
default:
result = null;
break;
}
return result;
}
/**
* Starts the swap job, which should schedule an attempted swap at the given
* configured interval (config["swapJob"]["intervalMillis"]
*/
void start();
static SwapJob fromConfig(
Optional<SwapJobConfig> cfg,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore) {
if (!cfg.isPresent()) {
return new NoopSwapJob();
}
if (!swapStore.isSafe() && !cfg.get().getAllowUnsafeStores()) {
Log.warn(
"Swap store '{}' is not safe; disabling swap job", swapStore.getClass().getSimpleName());
return new NoopSwapJob();
}
return new SwapJobImpl(cfg.get(), lock, repoStore, dbStore, swapStore);
}
/**
* Stops the stop job.
*/
void stop();
/*
* Starts the swap job, which should schedule an attempted swap at the given
* configured interval (config["swapJob"]["intervalMillis"]
*/
void start();
/**
* Called by the swap job when a project should be evicted.
*
* Pre:
* 1. projName must be in repoStore
* 2. projName should not be in swapStore
* 3. projName should be PRESENT in dbStore (last_accessed is not null)
*
* Acquires the project lock and performs an eviction of projName.
*
* Post:
* 1. projName should not in repoStore
* 2. projName must be in swapStore
* 3. projName must be SWAPPED in dbStore (last_accessed is null)
* @param projName
* @throws IOException
*/
void evict(String projName) throws IOException;
/*
* Stops the stop job.
*/
void stop();
/**
* Called on a project when it must be restored.
*
* Pre:
* 1. projName should not be in repoStore
* 2. projName must be in swapStore
* 3. projName must be SWAPPED in dbStore (last_accessed is null)
*
* Acquires the project lock and restores projName.
*
* Post:
* 1. projName must be in repoStore
* 2. projName should not in swapStore
* 3. projName should be PRESENT in dbStore (last_accessed is not null)
* @param projName
* @throws IOException
*/
void restore(String projName) throws IOException;
/*
* Called by the swap job when a project should be evicted.
*
* Pre:
* 1. projName must be in repoStore
* 2. projName should not be in swapStore
* 3. projName should be PRESENT in dbStore (last_accessed is not null)
*
* Acquires the project lock and performs an eviction of projName.
*
* Post:
* 1. projName should not in repoStore
* 2. projName must be in swapStore
* 3. projName must be SWAPPED in dbStore (last_accessed is null)
* @param projName
* @throws IOException
*/
void evict(String projName) throws IOException;
/*
* Called on a project when it must be restored.
*
* Pre:
* 1. projName should not be in repoStore
* 2. projName must be in swapStore
* 3. projName must be SWAPPED in dbStore (last_accessed is null)
*
* Acquires the project lock and restores projName.
*
* Post:
* 1. projName must be in repoStore
* 2. projName should not in swapStore
* 3. projName should be PRESENT in dbStore (last_accessed is not null)
* @param projName
* @throws IOException
*/
void restore(String projName) throws IOException;
}

91
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java
View File

@@ -1,62 +1,63 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob.CompressionMethod;
import uk.ac.ic.wlgitbridge.util.Log;
/**
/*
* Created by winston on 23/08/2016.
*/
public class SwapJobConfig {
private final int minProjects;
private final int lowGiB;
private final int highGiB;
private final long intervalMillis;
private final String compressionMethod;
private final boolean allowUnsafeStores;
private final int minProjects;
private final int lowGiB;
private final int highGiB;
private final long intervalMillis;
private final String compressionMethod;
private final boolean allowUnsafeStores;
public SwapJobConfig(
int minProjects,
int lowGiB,
int highGiB,
long intervalMillis,
String compressionMethod,
boolean allowUnsafeStores
) {
this.minProjects = minProjects;
this.lowGiB = lowGiB;
this.highGiB = highGiB;
this.intervalMillis = intervalMillis;
this.compressionMethod = compressionMethod;
this.allowUnsafeStores = allowUnsafeStores;
}
public SwapJobConfig(
int minProjects,
int lowGiB,
int highGiB,
long intervalMillis,
String compressionMethod,
boolean allowUnsafeStores) {
this.minProjects = minProjects;
this.lowGiB = lowGiB;
this.highGiB = highGiB;
this.intervalMillis = intervalMillis;
this.compressionMethod = compressionMethod;
this.allowUnsafeStores = allowUnsafeStores;
}
public int getMinProjects() {
return minProjects;
}
public int getMinProjects() {
return minProjects;
}
public int getLowGiB() {
return lowGiB;
}
public int getLowGiB() {
return lowGiB;
}
public int getHighGiB() {
return highGiB;
}
public int getHighGiB() {
return highGiB;
}
public long getIntervalMillis() {
return intervalMillis;
}
public long getIntervalMillis() {
return intervalMillis;
}
public boolean getAllowUnsafeStores() {
return allowUnsafeStores;
}
public boolean getAllowUnsafeStores() {
return allowUnsafeStores;
}
public SwapJob.CompressionMethod getCompressionMethod() {
CompressionMethod result = SwapJob.stringToCompressionMethod(compressionMethod);
if (result == null) {
Log.info("SwapJobConfig: un-supported compressionMethod '{}', default to 'bzip2'", compressionMethod);
result = CompressionMethod.Bzip2;
}
return result;
public SwapJob.CompressionMethod getCompressionMethod() {
CompressionMethod result = SwapJob.stringToCompressionMethod(compressionMethod);
if (result == null) {
Log.info(
"SwapJobConfig: un-supported compressionMethod '{}', default to 'bzip2'",
compressionMethod);
result = CompressionMethod.Bzip2;
}
return result;
}
}

443
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java
View File

@@ -1,6 +1,14 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import java.io.IOException;
import java.io.InputStream;
import java.sql.Timestamp;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Timer;
import java.util.concurrent.atomic.AtomicInteger;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
@@ -10,259 +18,228 @@ import uk.ac.ic.wlgitbridge.data.CannotAcquireLockException;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.TimerUtils;
import java.io.IOException;
import java.io.InputStream;
import java.sql.Timestamp;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Timer;
import java.util.concurrent.atomic.AtomicInteger;
/**
/*
* Created by winston on 20/08/2016.
*/
public class SwapJobImpl implements SwapJob {
private static final long GiB = (1l << 30);
private static final long GiB = (1l << 30);
int minProjects;
long lowWatermarkBytes;
long highWatermarkBytes;
Duration interval;
int minProjects;
long lowWatermarkBytes;
long highWatermarkBytes;
Duration interval;
private final ProjectLock lock;
private final RepoStore repoStore;
private final DBStore dbStore;
private final SwapStore swapStore;
private final CompressionMethod compressionMethod;
private final ProjectLock lock;
private final RepoStore repoStore;
private final DBStore dbStore;
private final SwapStore swapStore;
private final CompressionMethod compressionMethod;
private final Timer timer;
private final Timer timer;
final AtomicInteger swaps;
final AtomicInteger swaps;
public SwapJobImpl(
SwapJobConfig cfg,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore
) {
this(
cfg.getMinProjects(),
GiB * cfg.getLowGiB(),
GiB * cfg.getHighGiB(),
Duration.ofMillis(cfg.getIntervalMillis()),
cfg.getCompressionMethod(),
lock,
repoStore,
dbStore,
swapStore
);
public SwapJobImpl(
SwapJobConfig cfg,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore) {
this(
cfg.getMinProjects(),
GiB * cfg.getLowGiB(),
GiB * cfg.getHighGiB(),
Duration.ofMillis(cfg.getIntervalMillis()),
cfg.getCompressionMethod(),
lock,
repoStore,
dbStore,
swapStore);
}
SwapJobImpl(
int minProjects,
long lowWatermarkBytes,
long highWatermarkBytes,
Duration interval,
CompressionMethod method,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore) {
this.minProjects = minProjects;
this.lowWatermarkBytes = lowWatermarkBytes;
this.highWatermarkBytes = highWatermarkBytes;
this.interval = interval;
this.compressionMethod = method;
this.lock = lock;
this.repoStore = repoStore;
this.dbStore = dbStore;
this.swapStore = swapStore;
timer = new Timer();
swaps = new AtomicInteger(0);
}
@Override
public void start() {
timer.schedule(TimerUtils.makeTimerTask(this::doSwap), 0);
}
@Override
public void stop() {
timer.cancel();
}
private void doSwap() {
try {
doSwap_();
} catch (Throwable t) {
Log.warn("Exception thrown during swap job", t);
}
timer.schedule(TimerUtils.makeTimerTask(this::doSwap), interval.toMillis());
}
SwapJobImpl(
int minProjects,
long lowWatermarkBytes,
long highWatermarkBytes,
Duration interval,
CompressionMethod method,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore
) {
this.minProjects = minProjects;
this.lowWatermarkBytes = lowWatermarkBytes;
this.highWatermarkBytes = highWatermarkBytes;
this.interval = interval;
this.compressionMethod = method;
this.lock = lock;
this.repoStore = repoStore;
this.dbStore = dbStore;
this.swapStore = swapStore;
timer = new Timer();
swaps = new AtomicInteger(0);
private void doSwap_() {
ArrayList<String> exceptionProjectNames = new ArrayList<String>();
Log.debug("Running swap number {}", swaps.get() + 1);
long totalSize = repoStore.totalSize();
Log.debug("Size is {}/{} (high)", totalSize, highWatermarkBytes);
if (totalSize < highWatermarkBytes) {
Log.debug("No need to swap.");
swaps.incrementAndGet();
return;
}
@Override
public void start() {
timer.schedule(
TimerUtils.makeTimerTask(this::doSwap),
0
);
}
@Override
public void stop() {
timer.cancel();
}
private void doSwap() {
try {
doSwap_();
} catch (Throwable t) {
Log.warn("Exception thrown during swap job", t);
int numProjects = dbStore.getNumProjects();
// while we have too many projects on disk
while ((totalSize = repoStore.totalSize()) > lowWatermarkBytes
&& (numProjects = dbStore.getNumUnswappedProjects()) > minProjects) {
// check if we've had too many exceptions so far
if (exceptionProjectNames.size() >= 20) {
StringBuilder sb = new StringBuilder();
for (String s : exceptionProjectNames) {
sb.append(s);
sb.append(' ');
}
timer.schedule(
TimerUtils.makeTimerTask(this::doSwap),
interval.toMillis()
);
Log.error(
"Too many exceptions while running swap, giving up on this run: {}", sb.toString());
break;
}
// get the oldest project and try to swap it
String projectName = dbStore.getOldestUnswappedProject();
try {
evict(projectName);
} catch (Exception e) {
Log.warn("[{}] Exception while swapping, mark project and move on", projectName, e);
// NOTE: this is something of a hack. If a project fails to swap we get stuck in a
// loop where `dbStore.getOldestUnswappedProject()` gives the same failing project over and
// over again,
// which fills up the disk with errors. By touching the access time we can mark the project
// as a
// non-candidate for swapping. Ideally we should be checking the logs for these log events
// and fixing
// whatever is wrong with the project
dbStore.setLastAccessedTime(projectName, Timestamp.valueOf(LocalDateTime.now()));
exceptionProjectNames.add(projectName);
}
}
private void doSwap_() {
ArrayList<String> exceptionProjectNames = new ArrayList<String>();
Log.debug("Running swap number {}", swaps.get() + 1);
long totalSize = repoStore.totalSize();
Log.debug("Size is {}/{} (high)", totalSize, highWatermarkBytes);
if (totalSize < highWatermarkBytes) {
Log.debug("No need to swap.");
swaps.incrementAndGet();
return;
}
int numProjects = dbStore.getNumProjects();
// while we have too many projects on disk
while (
(totalSize = repoStore.totalSize()) > lowWatermarkBytes &&
(numProjects = dbStore.getNumUnswappedProjects()) > minProjects
) {
// check if we've had too many exceptions so far
if (exceptionProjectNames.size() >= 20) {
StringBuilder sb = new StringBuilder();
for (String s: exceptionProjectNames) {
sb.append(s);
sb.append(' ');
}
Log.error(
"Too many exceptions while running swap, giving up on this run: {}",
sb.toString()
);
break;
}
// get the oldest project and try to swap it
String projectName = dbStore.getOldestUnswappedProject();
try {
evict(projectName);
} catch (Exception e) {
Log.warn("[{}] Exception while swapping, mark project and move on", projectName, e);
// NOTE: this is something of a hack. If a project fails to swap we get stuck in a
// loop where `dbStore.getOldestUnswappedProject()` gives the same failing project over and over again,
// which fills up the disk with errors. By touching the access time we can mark the project as a
// non-candidate for swapping. Ideally we should be checking the logs for these log events and fixing
// whatever is wrong with the project
dbStore.setLastAccessedTime(
projectName,
Timestamp.valueOf(LocalDateTime.now())
);
exceptionProjectNames.add(projectName);
}
}
if (totalSize > lowWatermarkBytes) {
Log.warn(
"Finished swapping, but total size is still too high."
);
}
Log.debug(
"Size: {}/{} (low), " +
"{} (high), " +
"projects on disk: {}/{}, " +
"min projects on disk: {}",
totalSize,
lowWatermarkBytes,
highWatermarkBytes,
numProjects,
dbStore.getNumProjects(),
minProjects
);
swaps.incrementAndGet();
if (totalSize > lowWatermarkBytes) {
Log.warn("Finished swapping, but total size is still too high.");
}
Log.debug(
"Size: {}/{} (low), "
+ "{} (high), "
+ "projects on disk: {}/{}, "
+ "min projects on disk: {}",
totalSize,
lowWatermarkBytes,
highWatermarkBytes,
numProjects,
dbStore.getNumProjects(),
minProjects);
swaps.incrementAndGet();
}
/**
* @see SwapJob#evict(String) for high-level description.
*
* 1. Acquires the project lock.
* 2. Gets a bz2 stream and size of a project from the repo store, or throws
* 3. Uploads the bz2 stream and size to the projName in the swapStore.
* 4. Sets the last accessed time in the dbStore to null, which makes our
* state SWAPPED
* 5. Removes the project from the repo store.
* @param projName
* @throws IOException
*/
@Override
public void evict(String projName) throws IOException {
Preconditions.checkNotNull(projName, "projName was null");
Log.info("Evicting project: {}", projName);
try (LockGuard __ = lock.lockGuard(projName)) {
try {
repoStore.gcProject(projName);
} catch (Exception e) {
Log.error("[{}] Exception while running gc on project: {}", projName, e);
}
long[] sizePtr = new long[1];
try (InputStream blob = getBlobStream(projName, sizePtr)) {
swapStore.upload(projName, blob, sizePtr[0]);
String compression = SwapJob.compressionMethodAsString(compressionMethod);
if (compression == null) {
throw new RuntimeException("invalid compression method, should not happen");
}
dbStore.swap(projName, compression);
repoStore.remove(projName);
}
} catch (CannotAcquireLockException e) {
Log.warn("[{}] Cannot acquire project lock, skipping swap", projName);
return;
}
Log.info("Evicted project: {}", projName);
}
private InputStream getBlobStream(String projName, long[] sizePtr) throws IOException {
if (compressionMethod == CompressionMethod.Gzip) {
return repoStore.gzipProject(projName, sizePtr);
} else if (compressionMethod == CompressionMethod.Bzip2) {
return repoStore.bzip2Project(projName, sizePtr);
} else {
/*
* @see SwapJob#evict(String) for high-level description.
*
* 1. Acquires the project lock.
* 2. Gets a bz2 stream and size of a project from the repo store, or throws
* 3. Uploads the bz2 stream and size to the projName in the swapStore.
* 4. Sets the last accessed time in the dbStore to null, which makes our
* state SWAPPED
* 5. Removes the project from the repo store.
* @param projName
* @throws IOException
*/
@Override
public void evict(String projName) throws IOException {
Preconditions.checkNotNull(projName, "projName was null");
Log.info("Evicting project: {}", projName);
try (LockGuard __ = lock.lockGuard(projName)) {
try {
repoStore.gcProject(projName);
} catch (Exception e) {
Log.error("[{}] Exception while running gc on project: {}", projName, e);
}
long[] sizePtr = new long[1];
try (InputStream blob = getBlobStream(projName, sizePtr)) {
swapStore.upload(projName, blob, sizePtr[0]);
String compression = SwapJob.compressionMethodAsString(compressionMethod);
if (compression == null) {
throw new RuntimeException("invalid compression method, should not happen");
}
dbStore.swap(projName, compression);
repoStore.remove(projName);
}
} catch (CannotAcquireLockException e) {
Log.warn("[{}] Cannot acquire project lock, skipping swap", projName);
return;
}
Log.info("Evicted project: {}", projName);
}
/**
* @see SwapJob#restore(String) for high-level description.
*
* 1. Acquires the project lock.
* 2. Gets a bz2 stream for the project from the swapStore.
* 3. Fully downloads and places the bz2 stream back in the repo store.
* 4. Sets the last accessed time in the dbStore to now, which makes our
* state PRESENT and the last project to be evicted.
* @param projName
* @throws IOException
*/
@Override
public void restore(String projName) throws IOException {
try (LockGuard __ = lock.lockGuard(projName)) {
try (InputStream zipped = swapStore.openDownloadStream(projName)) {
String compression = dbStore.getSwapCompression(projName);
if (compression == null) {
throw new RuntimeException("Missing compression method during restore, should not happen");
}
if ("gzip".equals(compression)) {
repoStore.ungzipProject(
projName,
zipped
);
} else if ("bzip2".equals(compression)) {
repoStore.unbzip2Project(
projName,
zipped
);
}
swapStore.remove(projName);
dbStore.restore(projName);
}
} catch (CannotAcquireLockException e) {
throw new RuntimeException(e);
private InputStream getBlobStream(String projName, long[] sizePtr) throws IOException {
if (compressionMethod == CompressionMethod.Gzip) {
return repoStore.gzipProject(projName, sizePtr);
} else if (compressionMethod == CompressionMethod.Bzip2) {
return repoStore.bzip2Project(projName, sizePtr);
} else {
throw new RuntimeException("invalid compression method, should not happen");
}
}
/*
* @see SwapJob#restore(String) for high-level description.
*
* 1. Acquires the project lock.
* 2. Gets a bz2 stream for the project from the swapStore.
* 3. Fully downloads and places the bz2 stream back in the repo store.
* 4. Sets the last accessed time in the dbStore to now, which makes our
* state PRESENT and the last project to be evicted.
* @param projName
* @throws IOException
*/
@Override
public void restore(String projName) throws IOException {
try (LockGuard __ = lock.lockGuard(projName)) {
try (InputStream zipped = swapStore.openDownloadStream(projName)) {
String compression = dbStore.getSwapCompression(projName);
if (compression == null) {
throw new RuntimeException(
"Missing compression method during restore, should not happen");
}
if ("gzip".equals(compression)) {
repoStore.ungzipProject(projName, zipped);
} else if ("bzip2".equals(compression)) {
repoStore.unbzip2Project(projName, zipped);
}
swapStore.remove(projName);
dbStore.restore(projName);
}
} catch (CannotAcquireLockException e) {
throw new RuntimeException(e);
}
}
}

67
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStore.java
View File

@@ -1,58 +1,49 @@
package uk.ac.ic.wlgitbridge.bridge.swap.store;
import org.apache.commons.io.IOUtils;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.io.IOUtils;
/**
/*
* Created by winston on 23/08/2016.
*/
public class InMemorySwapStore implements SwapStore {
private final Map<String, byte[]> store;
private final Map<String, byte[]> store;
public InMemorySwapStore() {
store = new HashMap<>();
}
public InMemorySwapStore() {
store = new HashMap<>();
}
public InMemorySwapStore(SwapStoreConfig __) {
this();
}
public InMemorySwapStore(SwapStoreConfig __) {
this();
}
@Override
public void upload(
String projectName,
InputStream uploadStream,
long contentLength
) throws IOException {
store.put(
projectName,
IOUtils.toByteArray(uploadStream, contentLength)
);
}
@Override
public void upload(String projectName, InputStream uploadStream, long contentLength)
throws IOException {
store.put(projectName, IOUtils.toByteArray(uploadStream, contentLength));
}
@Override
public InputStream openDownloadStream(String projectName) {
byte[] buf = store.get(projectName);
if (buf == null) {
throw new IllegalArgumentException(
"no such project in swap store: " + projectName
);
}
return new ByteArrayInputStream(buf);
@Override
public InputStream openDownloadStream(String projectName) {
byte[] buf = store.get(projectName);
if (buf == null) {
throw new IllegalArgumentException("no such project in swap store: " + projectName);
}
return new ByteArrayInputStream(buf);
}
@Override
public void remove(String projectName) {
store.remove(projectName);
}
@Override
public void remove(String projectName) {
store.remove(projectName);
}
@Override
public boolean isSafe() {
return false;
}
@Override
public boolean isSafe() {
return false;
}
}

32
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/NoopSwapStore.java
View File

@@ -3,30 +3,26 @@ package uk.ac.ic.wlgitbridge.bridge.swap.store;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
/**
/*
* Created by winston on 24/08/2016.
*/
public class NoopSwapStore implements SwapStore {
public NoopSwapStore(SwapStoreConfig __) {}
public NoopSwapStore(SwapStoreConfig __) {}
@Override
public void upload(
String projectName,
InputStream uploadStream,
long contentLength
) {}
@Override
public void upload(String projectName, InputStream uploadStream, long contentLength) {}
@Override
public InputStream openDownloadStream(String projectName) {
return new ByteArrayInputStream(new byte[0]);
}
@Override
public InputStream openDownloadStream(String projectName) {
return new ByteArrayInputStream(new byte[0]);
}
@Override
public void remove(String projectName) {}
@Override
public void remove(String projectName) {}
@Override
public boolean isSafe() {
return false;
}
@Override
public boolean isSafe() {
return false;
}
}

112
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStore.java
View File

@@ -5,86 +5,60 @@ import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.*;
import java.io.InputStream;
/**
/*
* Created by winston on 21/08/2016.
*/
public class S3SwapStore implements SwapStore {
private final AmazonS3 s3;
private final AmazonS3 s3;
private final String bucketName;
private final String bucketName;
public S3SwapStore(SwapStoreConfig cfg) {
this(
cfg.getAwsAccessKey(),
cfg.getAwsSecret(),
cfg.getS3BucketName(),
cfg.getAwsRegion()
);
public S3SwapStore(SwapStoreConfig cfg) {
this(cfg.getAwsAccessKey(), cfg.getAwsSecret(), cfg.getS3BucketName(), cfg.getAwsRegion());
}
S3SwapStore(String accessKey, String secret, String bucketName, String region) {
String regionToUse = null;
if (region == null) {
regionToUse = "us-east-1";
} else {
regionToUse = region;
}
s3 =
AmazonS3ClientBuilder.standard()
.withRegion(regionToUse)
.withCredentials(
new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secret)))
.build();
this.bucketName = bucketName;
}
S3SwapStore(
String accessKey,
String secret,
String bucketName,
String region
) {
String regionToUse = null;
if (region == null) {
regionToUse = "us-east-1";
} else {
regionToUse = region;
}
s3 = AmazonS3ClientBuilder
.standard()
.withRegion(regionToUse)
.withCredentials(
new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secret))
).build();
this.bucketName = bucketName;
}
@Override
public void upload(String projectName, InputStream uploadStream, long contentLength) {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(contentLength);
PutObjectRequest put = new PutObjectRequest(bucketName, projectName, uploadStream, metadata);
PutObjectResult res = s3.putObject(put);
}
@Override
public void upload(
String projectName,
InputStream uploadStream,
long contentLength
) {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(contentLength);
PutObjectRequest put = new PutObjectRequest(
bucketName,
projectName,
uploadStream,
metadata
);
PutObjectResult res = s3.putObject(put);
}
@Override
public InputStream openDownloadStream(String projectName) {
GetObjectRequest get = new GetObjectRequest(bucketName, projectName);
S3Object res = s3.getObject(get);
return res.getObjectContent();
}
@Override
public InputStream openDownloadStream(String projectName) {
GetObjectRequest get = new GetObjectRequest(
bucketName,
projectName
);
S3Object res = s3.getObject(get);
return res.getObjectContent();
}
@Override
public void remove(String projectName) {
DeleteObjectRequest del = new DeleteObjectRequest(bucketName, projectName);
s3.deleteObject(del);
}
@Override
public void remove(String projectName) {
DeleteObjectRequest del = new DeleteObjectRequest(
bucketName,
projectName
);
s3.deleteObject(del);
}
@Override
public boolean isSafe() {
return true;
}
@Override
public boolean isSafe() {
return true;
}
}

54
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStore.java
View File

@@ -7,45 +7,37 @@ import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
/**
/*
* Created by winston on 20/08/2016.
*/
public interface SwapStore {
Map<String, Function<SwapStoreConfig, SwapStore>> swapStores =
new HashMap<String, Function<SwapStoreConfig, SwapStore>>() {
Map<String, Function<SwapStoreConfig, SwapStore>> swapStores =
new HashMap<String, Function<SwapStoreConfig, SwapStore>>() {
{
put("noop", NoopSwapStore::new);
put("memory", InMemorySwapStore::new);
put("s3", S3SwapStore::new);
}
{
put("noop", NoopSwapStore::new);
put("memory", InMemorySwapStore::new);
put("s3", S3SwapStore::new);
}
};
};
static SwapStore fromConfig(Optional<SwapStoreConfig> cfg) {
SwapStoreConfig cfg_ = cfg.orElse(SwapStoreConfig.NOOP);
String type = cfg_.getType();
return swapStores.get(type).apply(cfg_);
}
static SwapStore fromConfig(
Optional<SwapStoreConfig> cfg
) {
SwapStoreConfig cfg_ = cfg.orElse(SwapStoreConfig.NOOP);
String type = cfg_.getType();
return swapStores.get(type).apply(cfg_);
}
void upload(String projectName, InputStream uploadStream, long contentLength) throws IOException;
void upload(
String projectName,
InputStream uploadStream,
long contentLength
) throws IOException;
InputStream openDownloadStream(String projectName);
InputStream openDownloadStream(String projectName);
void remove(String projectName);
/**
* Returns true if the swap store safely persists swapped projects.
*
* Fake swap stores should return false.
*/
boolean isSafe();
void remove(String projectName);
/*
* Returns true if the swap store safely persists swapped projects.
*
* Fake swap stores should return false.
*/
boolean isSafe();
}

113
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStoreConfig.java
View File

@@ -1,85 +1,64 @@
package uk.ac.ic.wlgitbridge.bridge.swap.store;
/**
/*
* Created by winston on 24/08/2016.
*/
public class SwapStoreConfig {
public static final SwapStoreConfig NOOP = new SwapStoreConfig(
"noop",
null,
null,
null,
null
);
public static final SwapStoreConfig NOOP = new SwapStoreConfig("noop", null, null, null, null);
private String type;
private String awsAccessKey;
private String awsSecret;
private String s3BucketName;
private String awsRegion;
private String type;
private String awsAccessKey;
private String awsSecret;
private String s3BucketName;
private String awsRegion;
public SwapStoreConfig() {}
public SwapStoreConfig() {}
public SwapStoreConfig(
String awsAccessKey,
String awsSecret,
String s3BucketName,
String awsRegion
) {
this(
"s3",
awsAccessKey,
awsSecret,
s3BucketName,
awsRegion
);
}
public SwapStoreConfig(
String awsAccessKey, String awsSecret, String s3BucketName, String awsRegion) {
this("s3", awsAccessKey, awsSecret, s3BucketName, awsRegion);
}
SwapStoreConfig(
String type,
String awsAccessKey,
String awsSecret,
String s3BucketName,
String awsRegion
) {
this.type = type;
this.awsAccessKey = awsAccessKey;
this.awsSecret = awsSecret;
this.s3BucketName = s3BucketName;
this.awsRegion = awsRegion;
}
SwapStoreConfig(
String type, String awsAccessKey, String awsSecret, String s3BucketName, String awsRegion) {
this.type = type;
this.awsAccessKey = awsAccessKey;
this.awsSecret = awsSecret;
this.s3BucketName = s3BucketName;
this.awsRegion = awsRegion;
}
public String getType() {
return type;
}
public String getType() {
return type;
}
public String getAwsAccessKey() {
return awsAccessKey;
}
public String getAwsAccessKey() {
return awsAccessKey;
}
public String getAwsSecret() {
return awsSecret;
}
public String getAwsSecret() {
return awsSecret;
}
public String getS3BucketName() {
return s3BucketName;
}
public String getS3BucketName() {
return s3BucketName;
}
public String getAwsRegion() { return awsRegion; }
public String getAwsRegion() {
return awsRegion;
}
public SwapStoreConfig sanitisedCopy() {
return new SwapStoreConfig(
type,
awsAccessKey == null ? null : "<awsAccessKey>",
awsSecret == null ? null : "<awsSecret>",
s3BucketName,
awsRegion
);
}
public SwapStoreConfig sanitisedCopy() {
return new SwapStoreConfig(
type,
awsAccessKey == null ? null : "<awsAccessKey>",
awsSecret == null ? null : "<awsSecret>",
s3BucketName,
awsRegion);
}
public static SwapStoreConfig sanitisedCopy(SwapStoreConfig swapStore) {
return swapStore == null ? null : swapStore.sanitisedCopy();
}
public static SwapStoreConfig sanitisedCopy(SwapStoreConfig swapStore) {
return swapStore == null ? null : swapStore.sanitisedCopy();
}
}

16
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/util/CastUtil.java
View File

@@ -2,17 +2,15 @@ package uk.ac.ic.wlgitbridge.bridge.util;
import com.google.common.base.Preconditions;
/**
/*
* Created by winston on 01/07/2017.
*/
public class CastUtil {
public static int assumeInt(long l) {
Preconditions.checkArgument(
l <= (long) Integer.MAX_VALUE
&& l >= (long) Integer.MIN_VALUE,
l + " cannot fit inside an int");
return (int) l;
}
public static int assumeInt(long l) {
Preconditions.checkArgument(
l <= (long) Integer.MAX_VALUE && l >= (long) Integer.MIN_VALUE,
l + " cannot fit inside an int");
return (int) l;
}
}

214
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/CandidateSnapshot.java
View File

@@ -3,148 +3,128 @@ package uk.ac.ic.wlgitbridge.data;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.util.Util;
import java.io.File;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.util.Util;
/**
/*
* Created by Winston on 16/11/14.
*/
public class CandidateSnapshot implements AutoCloseable {
private final String projectName;
private final int currentVersion;
private final List<ServletFile> files;
private final List<String> deleted;
private File attsDirectory;
private final String projectName;
private final int currentVersion;
private final List<ServletFile> files;
private final List<String> deleted;
private File attsDirectory;
public CandidateSnapshot(
String projectName,
int currentVersion,
RawDirectory directoryContents,
RawDirectory oldDirectoryContents
) {
this.projectName = projectName;
this.currentVersion = currentVersion;
files = diff(directoryContents, oldDirectoryContents);
deleted = deleted(directoryContents, oldDirectoryContents);
public CandidateSnapshot(
String projectName,
int currentVersion,
RawDirectory directoryContents,
RawDirectory oldDirectoryContents) {
this.projectName = projectName;
this.currentVersion = currentVersion;
files = diff(directoryContents, oldDirectoryContents);
deleted = deleted(directoryContents, oldDirectoryContents);
}
private List<ServletFile> diff(
RawDirectory directoryContents, RawDirectory oldDirectoryContents) {
List<ServletFile> files = new LinkedList<ServletFile>();
Map<String, RawFile> fileTable = directoryContents.getFileTable();
Map<String, RawFile> oldFileTable = oldDirectoryContents.getFileTable();
for (Entry<String, RawFile> entry : fileTable.entrySet()) {
RawFile file = entry.getValue();
files.add(new ServletFile(file, oldFileTable.get(file.getPath())));
}
return files;
}
private List<ServletFile> diff(
RawDirectory directoryContents,
RawDirectory oldDirectoryContents
) {
List<ServletFile> files = new LinkedList<ServletFile>();
Map<String, RawFile> fileTable = directoryContents.getFileTable();
Map<String, RawFile> oldFileTable = oldDirectoryContents.getFileTable();
for (Entry<String, RawFile> entry : fileTable.entrySet()) {
RawFile file = entry.getValue();
files.add(new ServletFile(file, oldFileTable.get(file.getPath())));
}
return files;
private List<String> deleted(RawDirectory directoryContents, RawDirectory oldDirectoryContents) {
List<String> deleted = new LinkedList<String>();
Map<String, RawFile> fileTable = directoryContents.getFileTable();
for (Entry<String, RawFile> entry : oldDirectoryContents.getFileTable().entrySet()) {
String path = entry.getKey();
RawFile newFile = fileTable.get(path);
if (newFile == null) {
deleted.add(path);
}
}
return deleted;
}
private List<String> deleted(
RawDirectory directoryContents,
RawDirectory oldDirectoryContents
) {
List<String> deleted = new LinkedList<String>();
Map<String, RawFile> fileTable = directoryContents.getFileTable();
for (
Entry<String, RawFile> entry :
oldDirectoryContents.getFileTable().entrySet()
) {
String path = entry.getKey();
RawFile newFile = fileTable.get(path);
if (newFile == null) {
deleted.add(path);
}
}
return deleted;
public void writeServletFiles(File rootGitDirectory) throws IOException {
attsDirectory = new File(rootGitDirectory, ".wlgb/atts/" + projectName);
for (ServletFile file : files) {
if (file.isChanged()) {
file.writeToDiskWithName(attsDirectory, file.getUniqueIdentifier());
}
}
}
public void writeServletFiles(File rootGitDirectory) throws IOException {
attsDirectory = new File(
rootGitDirectory,
".wlgb/atts/" + projectName
);
for (ServletFile file : files) {
if (file.isChanged()) {
file.writeToDiskWithName(attsDirectory, file.getUniqueIdentifier());
}
}
public void deleteServletFiles() throws IOException {
if (attsDirectory != null) {
Util.deleteDirectory(attsDirectory);
}
}
public void deleteServletFiles() throws IOException {
if (attsDirectory != null) {
Util.deleteDirectory(attsDirectory);
}
public JsonElement getJsonRepresentation(String postbackKey) {
String projectURL = Util.getPostbackURL() + "api/" + projectName;
JsonObject jsonObject = new JsonObject();
jsonObject.addProperty("latestVerId", currentVersion);
jsonObject.add("files", getFilesAsJson(projectURL, postbackKey));
jsonObject.addProperty("postbackUrl", projectURL + "/" + postbackKey + "/postback");
return jsonObject;
}
private JsonArray getFilesAsJson(String projectURL, String postbackKey) {
JsonArray filesArray = new JsonArray();
for (ServletFile file : files) {
filesArray.add(getFileAsJson(file, projectURL, postbackKey));
}
return filesArray;
}
public JsonElement getJsonRepresentation(String postbackKey) {
String projectURL = Util.getPostbackURL() + "api/" + projectName;
JsonObject jsonObject = new JsonObject();
jsonObject.addProperty("latestVerId", currentVersion);
jsonObject.add("files", getFilesAsJson(projectURL, postbackKey));
jsonObject.addProperty(
"postbackUrl", projectURL + "/" + postbackKey + "/postback"
);
return jsonObject;
private JsonObject getFileAsJson(ServletFile file, String projectURL, String postbackKey) {
JsonObject jsonFile = new JsonObject();
jsonFile.addProperty("name", file.getPath());
if (file.isChanged()) {
String identifier = file.getUniqueIdentifier();
String url = projectURL + "/" + identifier + "?key=" + postbackKey;
jsonFile.addProperty("url", url);
}
return jsonFile;
}
private JsonArray getFilesAsJson(String projectURL, String postbackKey) {
JsonArray filesArray = new JsonArray();
for (ServletFile file : files) {
filesArray.add(getFileAsJson(file, projectURL, postbackKey));
}
return filesArray;
}
public String getProjectName() {
return projectName;
}
private JsonObject getFileAsJson(
ServletFile file,
String projectURL,
String postbackKey
) {
JsonObject jsonFile = new JsonObject();
jsonFile.addProperty("name", file.getPath());
if (file.isChanged()) {
String identifier = file.getUniqueIdentifier();
String url = projectURL + "/" + identifier + "?key=" + postbackKey;
jsonFile.addProperty("url", url);
}
return jsonFile;
}
public List<String> getDeleted() {
return deleted;
}
public String getProjectName() {
return projectName;
}
public List<String> getDeleted() {
return deleted;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("VersionId: ");
sb.append(currentVersion);
sb.append(", files: ");
sb.append(files);
sb.append(", deleted: ");
sb.append(deleted);
return sb.toString();
}
@Override
public void close() throws IOException {
deleteServletFiles();
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("VersionId: ");
sb.append(currentVersion);
sb.append(", files: ");
sb.append(files);
sb.append(", deleted: ");
sb.append(deleted);
return sb.toString();
}
@Override
public void close() throws IOException {
deleteServletFiles();
}
}

8
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/CannotAcquireLockException.java
View File

@@ -1,9 +1,9 @@
package uk.ac.ic.wlgitbridge.data;
public class CannotAcquireLockException extends Exception {
String projectName;
String projectName;
public CannotAcquireLockException() {
super("Another operation is in progress. Please try again later.");
}
public CannotAcquireLockException() {
super("Another operation is in progress. Please try again later.");
}
}

5
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/LockAllWaiter.java
View File

@@ -1,10 +1,9 @@
package uk.ac.ic.wlgitbridge.data;
/**
/*
* Created by Winston on 21/02/15.
*/
public interface LockAllWaiter {
void threadsRemaining(int threads);
void threadsRemaining(int threads);
}

146
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ProjectLockImpl.java
View File

@@ -1,98 +1,96 @@
package uk.ac.ic.wlgitbridge.data;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.util.Log;
/**
/*
* Created by Winston on 20/11/14.
*/
public class ProjectLockImpl implements ProjectLock {
private final Map<String, Lock> projectLocks;
private final ReentrantReadWriteLock rwlock;
private final Lock rlock;
private final ReentrantReadWriteLock.WriteLock wlock;
private LockAllWaiter waiter;
private boolean waiting;
private final Map<String, Lock> projectLocks;
private final ReentrantReadWriteLock rwlock;
private final Lock rlock;
private final ReentrantReadWriteLock.WriteLock wlock;
private LockAllWaiter waiter;
private boolean waiting;
public ProjectLockImpl() {
projectLocks = new HashMap<String, Lock>();
rwlock = new ReentrantReadWriteLock();
rlock = rwlock.readLock();
wlock = rwlock.writeLock();
waiting = false;
public ProjectLockImpl() {
projectLocks = new HashMap<String, Lock>();
rwlock = new ReentrantReadWriteLock();
rlock = rwlock.readLock();
wlock = rwlock.writeLock();
waiting = false;
}
public ProjectLockImpl(LockAllWaiter waiter) {
this();
setWaiter(waiter);
}
@Override
public void lockForProject(String projectName) throws CannotAcquireLockException {
Log.debug("[{}] taking project lock", projectName);
Lock projectLock = getLockForProjectName(projectName);
try {
if (!projectLock.tryLock(5, TimeUnit.SECONDS)) {
Log.debug("[{}] failed to acquire project lock", projectName);
throw new CannotAcquireLockException();
}
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
Log.debug("[{}] taking reentrant lock", projectName);
rlock.lock();
Log.debug("[{}] taken locks", projectName);
}
public ProjectLockImpl(LockAllWaiter waiter) {
this();
setWaiter(waiter);
@Override
public void unlockForProject(String projectName) {
Log.debug("[{}] releasing project lock", projectName);
getLockForProjectName(projectName).unlock();
Log.debug("[{}] releasing reentrant lock", projectName);
rlock.unlock();
Log.debug("[{}] released locks", projectName);
if (waiting) {
Log.debug("[{}] waiting for remaining threads", projectName);
trySignal();
}
}
@Override
public void lockForProject(String projectName) throws CannotAcquireLockException {
Log.debug("[{}] taking project lock", projectName);
Lock projectLock = getLockForProjectName(projectName);
try {
if (!projectLock.tryLock(5, TimeUnit.SECONDS)) {
Log.debug("[{}] failed to acquire project lock", projectName);
throw new CannotAcquireLockException();
}
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
Log.debug("[{}] taking reentrant lock", projectName);
rlock.lock();
Log.debug("[{}] taken locks", projectName);
private void trySignal() {
int threads = rwlock.getReadLockCount();
Log.debug("-> waiting for {} threads", threads);
if (waiter != null && threads > 0) {
waiter.threadsRemaining(threads);
}
Log.debug("-> finished waiting for threads");
}
@Override
public void unlockForProject(String projectName) {
Log.debug("[{}] releasing project lock", projectName);
getLockForProjectName(projectName).unlock();
Log.debug("[{}] releasing reentrant lock", projectName);
rlock.unlock();
Log.debug("[{}] released locks", projectName);
if (waiting) {
Log.debug("[{}] waiting for remaining threads", projectName);
trySignal();
}
}
private void trySignal() {
int threads = rwlock.getReadLockCount();
Log.debug("-> waiting for {} threads", threads);
if (waiter != null && threads > 0) {
waiter.threadsRemaining(threads);
}
Log.debug("-> finished waiting for threads");
}
public void lockAll() {
Log.debug("-> locking all threads");
waiting = true;
trySignal();
Log.debug("-> locking reentrant write lock");
wlock.lock();
}
private synchronized Lock getLockForProjectName(String projectName) {
Lock lock = projectLocks.get(projectName);
if (lock == null) {
lock = new ReentrantLock();
projectLocks.put(projectName, lock);
}
return lock;
}
public void setWaiter(LockAllWaiter waiter) {
this.waiter = waiter;
public void lockAll() {
Log.debug("-> locking all threads");
waiting = true;
trySignal();
Log.debug("-> locking reentrant write lock");
wlock.lock();
}
private synchronized Lock getLockForProjectName(String projectName) {
Lock lock = projectLocks.get(projectName);
if (lock == null) {
lock = new ReentrantLock();
projectLocks.put(projectName, lock);
}
return lock;
}
public void setWaiter(LockAllWaiter waiter) {
this.waiter = waiter;
}
}

62
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ServletFile.java
View File

@@ -1,46 +1,48 @@
package uk.ac.ic.wlgitbridge.data;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import java.util.UUID;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
/**
/*
* Created by Winston on 21/02/15.
*/
public class ServletFile extends RawFile {
private final RawFile file;
private final boolean changed;
private String uuid;
private final RawFile file;
private final boolean changed;
private String uuid;
public ServletFile(RawFile file, RawFile oldFile) {
this.file = file;
this.uuid = UUID.randomUUID().toString();
changed = !equals(oldFile);
}
public ServletFile(RawFile file, RawFile oldFile) {
this.file = file;
this.uuid = UUID.randomUUID().toString();
changed = !equals(oldFile);
}
public String getUniqueIdentifier() { return uuid; }
public String getUniqueIdentifier() {
return uuid;
}
@Override
public String getPath() {
return file.getPath();
}
@Override
public String getPath() {
return file.getPath();
}
@Override
public byte[] getContents() {
return file.getContents();
}
@Override
public byte[] getContents() {
return file.getContents();
}
@Override
public long size() {
return getContents().length;
}
@Override
public long size() {
return getContents().length;
}
public boolean isChanged() {
return changed;
}
public boolean isChanged() {
return changed;
}
@Override
public String toString() {
return getPath();
}
@Override
public String toString() {
return getPath();
}
}

122
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/GitDirectoryContents.java
View File

@@ -1,84 +1,76 @@
package uk.ac.ic.wlgitbridge.data.filestore;
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
import uk.ac.ic.wlgitbridge.util.Util;
import java.io.File;
import java.io.IOException;
import java.util.Date;
import java.util.List;
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
import uk.ac.ic.wlgitbridge.util.Util;
/**
/*
* Created by Winston on 14/11/14.
*/
public class GitDirectoryContents {
private final List<RawFile> files;
private final File gitDirectory;
private final String userName;
private final String userEmail;
private final String commitMessage;
private final Date when;
private final List<RawFile> files;
private final File gitDirectory;
private final String userName;
private final String userEmail;
private final String commitMessage;
private final Date when;
public GitDirectoryContents(
List<RawFile> files,
File rootGitDirectory,
String projectName,
String userName,
String userEmail,
String commitMessage,
Date when
) {
this.files = files;
this.gitDirectory = new File(rootGitDirectory, projectName);
this.userName = userName;
this.userEmail = userEmail;
this.commitMessage = commitMessage;
this.when = when;
public GitDirectoryContents(
List<RawFile> files,
File rootGitDirectory,
String projectName,
String userName,
String userEmail,
String commitMessage,
Date when) {
this.files = files;
this.gitDirectory = new File(rootGitDirectory, projectName);
this.userName = userName;
this.userEmail = userEmail;
this.commitMessage = commitMessage;
this.when = when;
}
public GitDirectoryContents(
List<RawFile> files, File rootGitDirectory, String projectName, Snapshot snapshot) {
this(
files,
rootGitDirectory,
projectName,
snapshot.getUserName(),
snapshot.getUserEmail(),
snapshot.getComment(),
snapshot.getCreatedAt());
}
public void write() throws IOException {
Util.deleteInDirectoryApartFrom(gitDirectory, ".git");
for (RawFile fileNode : files) {
fileNode.writeToDisk(gitDirectory);
}
}
public GitDirectoryContents(
List<RawFile> files,
File rootGitDirectory,
String projectName,
Snapshot snapshot
) {
this(
files,
rootGitDirectory,
projectName,
snapshot.getUserName(),
snapshot.getUserEmail(),
snapshot.getComment(),
snapshot.getCreatedAt()
);
}
public File getDirectory() {
return gitDirectory;
}
public void write() throws IOException {
Util.deleteInDirectoryApartFrom(gitDirectory, ".git");
for (RawFile fileNode : files) {
fileNode.writeToDisk(gitDirectory);
}
}
public String getUserName() {
return userName;
}
public File getDirectory() {
return gitDirectory;
}
public String getUserEmail() {
return userEmail;
}
public String getUserName() {
return userName;
}
public String getUserEmail() {
return userEmail;
}
public String getCommitMessage() {
return commitMessage;
}
public Date getWhen() {
return when;
}
public String getCommitMessage() {
return commitMessage;
}
public Date getWhen() {
return when;
}
}

20
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawDirectory.java
View File

@@ -1,23 +1,19 @@
package uk.ac.ic.wlgitbridge.data.filestore;
import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
import java.util.Map;
import java.util.Optional;
/**
/*
* Created by Winston on 16/11/14.
*/
public class RawDirectory {
private final Map<String, RawFile> fileTable;
private final Map<String, RawFile> fileTable;
public RawDirectory(Map<String, RawFile> fileTable) {
this.fileTable = fileTable;
}
public Map<String, RawFile> getFileTable() {
return fileTable;
}
public RawDirectory(Map<String, RawFile> fileTable) {
this.fileTable = fileTable;
}
public Map<String, RawFile> getFileTable() {
return fileTable;
}
}

55
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawFile.java
View File

@@ -1,46 +1,43 @@
package uk.ac.ic.wlgitbridge.data.filestore;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Arrays;
import uk.ac.ic.wlgitbridge.util.Log;
/**
/*
* Created by Winston on 16/11/14.
*/
public abstract class RawFile {
public abstract String getPath();
public abstract String getPath();
public abstract byte[] getContents();
public abstract byte[] getContents();
public abstract long size();
public abstract long size();
public final void writeToDisk(File directory) throws IOException {
writeToDiskWithName(directory, getPath());
public final void writeToDisk(File directory) throws IOException {
writeToDiskWithName(directory, getPath());
}
public final void writeToDiskWithName(File directory, String name) throws IOException {
File file = new File(directory, name);
file.getParentFile().mkdirs();
file.createNewFile();
OutputStream out = new FileOutputStream(file);
out.write(getContents());
out.close();
Log.debug("Wrote file: {}", file.getAbsolutePath());
}
@Override
public boolean equals(Object obj) {
if (!(obj instanceof RawFile)) {
return false;
}
public final void writeToDiskWithName(File directory, String name) throws IOException {
File file = new File(directory, name);
file.getParentFile().mkdirs();
file.createNewFile();
OutputStream out = new FileOutputStream(file);
out.write(getContents());
out.close();
Log.debug("Wrote file: {}", file.getAbsolutePath());
}
@Override
public boolean equals(Object obj) {
if (!(obj instanceof RawFile)) {
return false;
}
RawFile that = (RawFile) obj;
return getPath().equals(that.getPath())
&& Arrays.equals(getContents(), that.getContents());
}
RawFile that = (RawFile) obj;
return getPath().equals(that.getPath()) && Arrays.equals(getContents(), that.getContents());
}
}

39
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RepositoryFile.java
View File

@@ -1,31 +1,30 @@
package uk.ac.ic.wlgitbridge.data.filestore;
/**
/*
* Created by Winston on 16/11/14.
*/
public class RepositoryFile extends RawFile {
private final String path;
private final byte[] contents;
private final String path;
private final byte[] contents;
public RepositoryFile(String path, byte[] contents) {
this.path = path;
this.contents = contents;
}
public RepositoryFile(String path, byte[] contents) {
this.path = path;
this.contents = contents;
}
@Override
public String getPath() {
return path;
}
@Override
public String getPath() {
return path;
}
@Override
public byte[] getContents() {
return contents;
}
@Override
public long size() {
return contents.length;
}
@Override
public byte[] getContents() {
return contents;
}
@Override
public long size() {
return contents.length;
}
}

100
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/Snapshot.java
View File

@@ -1,5 +1,7 @@
package uk.ac.ic.wlgitbridge.data.model;
import java.util.Date;
import java.util.List;
import org.joda.time.DateTime;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotData;
@@ -7,71 +9,67 @@ import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotFile;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.SnapshotInfo;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.WLUser;
import java.util.Date;
import java.util.List;
/**
/*
* Created by Winston on 03/11/14.
*/
public class Snapshot implements Comparable<Snapshot> {
private final int versionID;
private final String comment;
private final String userName;
private final String userEmail;
private final Date createdAt;
private final int versionID;
private final String comment;
private final String userName;
private final String userEmail;
private final Date createdAt;
private final List<SnapshotFile> srcs;
private final List<SnapshotAttachment> atts;
private final List<SnapshotFile> srcs;
private final List<SnapshotAttachment> atts;
public Snapshot(SnapshotInfo info, SnapshotData data) {
versionID = info.getVersionId();
comment = info.getComment();
WLUser user = info.getUser();
userName = user.getName();
userEmail = user.getEmail();
createdAt = new DateTime(info.getCreatedAt()).toDate();
public Snapshot(SnapshotInfo info, SnapshotData data) {
versionID = info.getVersionId();
comment = info.getComment();
WLUser user = info.getUser();
userName = user.getName();
userEmail = user.getEmail();
createdAt = new DateTime(info.getCreatedAt()).toDate();
srcs = data.getSrcs();
atts = data.getAtts();
}
srcs = data.getSrcs();
atts = data.getAtts();
}
public int getVersionID() {
return versionID;
}
public int getVersionID() {
return versionID;
}
public String getComment() {
return comment;
}
public String getComment() {
return comment;
}
public String getUserName() {
return userName;
}
public String getUserName() {
return userName;
}
public String getUserEmail() {
return userEmail;
}
public String getUserEmail() {
return userEmail;
}
public List<SnapshotFile> getSrcs() {
return srcs;
}
public List<SnapshotFile> getSrcs() {
return srcs;
}
public List<SnapshotAttachment> getAtts() {
return atts;
}
public List<SnapshotAttachment> getAtts() {
return atts;
}
public Date getCreatedAt() {
return createdAt;
}
public Date getCreatedAt() {
return createdAt;
}
@Override
public int compareTo(Snapshot snapshot) {
return Integer.compare(versionID, snapshot.versionID);
}
@Override
public String toString() {
return String.valueOf(versionID);
}
@Override
public int compareTo(Snapshot snapshot) {
return Integer.compare(versionID, snapshot.versionID);
}
@Override
public String toString() {
return String.valueOf(versionID);
}
}

41
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/FileLimitExceededException.java
View File

@@ -1,34 +1,31 @@
package uk.ac.ic.wlgitbridge.git.exception;
import uk.ac.ic.wlgitbridge.util.Util;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
public class FileLimitExceededException extends GitUserException {
private final long numFiles;
private final long numFiles;
private final long maxFiles;
private final long maxFiles;
public FileLimitExceededException(long numFiles, long maxFiles) {
this.numFiles = numFiles;
this.maxFiles = maxFiles;
}
public FileLimitExceededException(long numFiles, long maxFiles) {
this.numFiles = numFiles;
this.maxFiles = maxFiles;
}
@Override
public String getMessage() {
return "too many files";
}
@Override
public List<String> getDescriptionLines() {
return Arrays.asList(
"repository contains " +
numFiles + " files, which exceeds the limit of " +
maxFiles + " files"
);
}
@Override
public String getMessage() {
return "too many files";
}
@Override
public List<String> getDescriptionLines() {
return Arrays.asList(
"repository contains "
+ numFiles
+ " files, which exceeds the limit of "
+ maxFiles
+ " files");
}
}

7
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/GitUserException.java
View File

@@ -2,13 +2,12 @@ package uk.ac.ic.wlgitbridge.git.exception;
import java.util.List;
/**
/*
* Created by winston on 20/08/2016.
*/
public abstract class GitUserException extends Exception {
public abstract String getMessage();
public abstract List<String> getDescriptionLines();
public abstract String getMessage();
public abstract List<String> getDescriptionLines();
}

24
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/InvalidGitRepository.java
View File

@@ -5,18 +5,16 @@ import java.util.List;
public class InvalidGitRepository extends GitUserException {
@Override
public String getMessage() {
return "invalid git repo";
}
@Override
public List<String> getDescriptionLines() {
return Arrays.asList(
"Your Git repository contains a reference we cannot resolve.",
"If your project contains a Git submodule,",
"please remove it and try again."
);
}
@Override
public String getMessage() {
return "invalid git repo";
}
@Override
public List<String> getDescriptionLines() {
return Arrays.asList(
"Your Git repository contains a reference we cannot resolve.",
"If your project contains a Git submodule,",
"please remove it and try again.");
}
}

46
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/SizeLimitExceededException.java
View File

@@ -1,40 +1,34 @@
package uk.ac.ic.wlgitbridge.git.exception;
import uk.ac.ic.wlgitbridge.util.Util;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import uk.ac.ic.wlgitbridge.util.Util;
public class SizeLimitExceededException extends GitUserException {
private final Optional<String> path;
private final Optional<String> path;
private final long actualSize;
private final long actualSize;
private final long maxSize;
private final long maxSize;
public SizeLimitExceededException(
Optional<String> path, long actualSize, long maxSize) {
this.path = path;
this.actualSize = actualSize;
this.maxSize = maxSize;
}
public SizeLimitExceededException(Optional<String> path, long actualSize, long maxSize) {
this.path = path;
this.actualSize = actualSize;
this.maxSize = maxSize;
}
@Override
public String getMessage() {
return "file too big";
}
@Override
public List<String> getDescriptionLines() {
String filename =
path.isPresent() ? "File '" + path.get() + "' is" : "There's a file";
return Arrays.asList(
filename + " too large to push to "
+ Util.getServiceName() + " via git",
"the recommended maximum file size is 50 MiB"
);
}
@Override
public String getMessage() {
return "file too big";
}
@Override
public List<String> getDescriptionLines() {
String filename = path.isPresent() ? "File '" + path.get() + "' is" : "There's a file";
return Arrays.asList(
filename + " too large to push to " + Util.getServiceName() + " via git",
"the recommended maximum file size is 50 MiB");
}
}

6
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/SnapshotAPIException.java
View File

@@ -2,9 +2,7 @@ package uk.ac.ic.wlgitbridge.git.exception;
import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
/**
/*
* Created by winston on 20/08/2016.
*/
public abstract class SnapshotAPIException
extends GitUserException
implements JSONSource {}
public abstract class SnapshotAPIException extends GitUserException implements JSONSource {}

97
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java
View File

@@ -1,81 +1,68 @@
package uk.ac.ic.wlgitbridge.git.handler;
import com.google.api.client.auth.oauth2.Credential;
import java.util.Optional;
import javax.servlet.http.HttpServletRequest;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.ReceivePack;
import org.eclipse.jgit.transport.resolver.ReceivePackFactory;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi;
import uk.ac.ic.wlgitbridge.git.handler.hook.WriteLatexPutHook;
import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet;
import uk.ac.ic.wlgitbridge.server.Oauth2Filter;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.Util;
import javax.servlet.http.HttpServletRequest;
import java.util.Optional;
/**
/*
* Created by Winston on 02/11/14.
*/
/**
/*
* One of the "big three" interfaces created by {@link WLGitServlet} to handle
* user Git requests.
*
* This class just puts a {@link WriteLatexPutHook} into the {@link ReceivePack}
* that it returns.
*/
public class WLReceivePackFactory
implements ReceivePackFactory<HttpServletRequest> {
public class WLReceivePackFactory implements ReceivePackFactory<HttpServletRequest> {
private final RepoStore repoStore;
private final RepoStore repoStore;
private final Bridge bridge;
private final Bridge bridge;
public WLReceivePackFactory(RepoStore repoStore, Bridge bridge) {
this.repoStore = repoStore;
this.bridge = bridge;
public WLReceivePackFactory(RepoStore repoStore, Bridge bridge) {
this.repoStore = repoStore;
this.bridge = bridge;
}
/*
* Puts a {@link WriteLatexPutHook} into the returned {@link ReceivePack}.
*
* The {@link WriteLatexPutHook} needs our hostname, which we get from the
* original {@link HttpServletRequest}, used to provide a postback URL to
* the {@link SnapshotApi}. We also give it the oauth2 that we injected in
* the {@link Oauth2Filter}, and the {@link Bridge}.
*
* At this point, the repository will have been synced to the latest on
* Overleaf, but it's possible that an update happens on Overleaf while our
* put hook is running. In this case, we fail, and the user tries again,
* triggering another sync, and so on.
* @param httpServletRequest the original request
* @param repository the JGit {@link Repository} provided by
* {@link WLRepositoryResolver}
* @return a correctly hooked {@link ReceivePack}
*/
@Override
public ReceivePack create(HttpServletRequest httpServletRequest, Repository repository) {
Log.debug("[{}] Creating receive-pack", repository.getWorkTree().getName());
Optional<Credential> oauth2 =
Optional.ofNullable(
(Credential) httpServletRequest.getAttribute(Oauth2Filter.ATTRIBUTE_KEY));
ReceivePack receivePack = new ReceivePack(repository);
String hostname = Util.getPostbackURL();
if (hostname == null) {
hostname = httpServletRequest.getLocalName();
}
/**
* Puts a {@link WriteLatexPutHook} into the returned {@link ReceivePack}.
*
* The {@link WriteLatexPutHook} needs our hostname, which we get from the
* original {@link HttpServletRequest}, used to provide a postback URL to
* the {@link SnapshotApi}. We also give it the oauth2 that we injected in
* the {@link Oauth2Filter}, and the {@link Bridge}.
*
* At this point, the repository will have been synced to the latest on
* Overleaf, but it's possible that an update happens on Overleaf while our
* put hook is running. In this case, we fail, and the user tries again,
* triggering another sync, and so on.
* @param httpServletRequest the original request
* @param repository the JGit {@link Repository} provided by
* {@link WLRepositoryResolver}
* @return a correctly hooked {@link ReceivePack}
*/
@Override
public ReceivePack create(
HttpServletRequest httpServletRequest,
Repository repository
) {
Log.debug(
"[{}] Creating receive-pack",
repository.getWorkTree().getName()
);
Optional<Credential> oauth2 = Optional.ofNullable(
(Credential) httpServletRequest.getAttribute(
Oauth2Filter.ATTRIBUTE_KEY));
ReceivePack receivePack = new ReceivePack(repository);
String hostname = Util.getPostbackURL();
if (hostname == null) {
hostname = httpServletRequest.getLocalName();
}
receivePack.setPreReceiveHook(
new WriteLatexPutHook(repoStore, bridge, hostname, oauth2)
);
return receivePack;
}
receivePack.setPreReceiveHook(new WriteLatexPutHook(repoStore, bridge, hostname, oauth2));
return receivePack;
}
}

175
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLRepositoryResolver.java
View File

@@ -1,6 +1,9 @@
package uk.ac.ic.wlgitbridge.git.handler;
import com.google.api.client.auth.oauth2.Credential;
import java.io.IOException;
import java.util.Optional;
import javax.servlet.http.HttpServletRequest;
import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.ServiceMayNotContinueException;
@@ -9,23 +12,15 @@ import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.data.CannotAcquireLockException;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.handler.hook.WriteLatexPutHook;
import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet;
import uk.ac.ic.wlgitbridge.server.GitBridgeServer;
import uk.ac.ic.wlgitbridge.server.Oauth2Filter;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.Util;
import javax.servlet.http.HttpServletRequest;
import java.io.IOException;
import java.util.Arrays;
import java.util.Optional;
/**
/*
* Created by Winston on 02/11/14.
*/
/**
/*
* One of the "big three" interfaces created by {@link WLGitServlet} to handle
* user Git requests.
*
@@ -37,93 +32,81 @@ import java.util.Optional;
* bringing it onto disk and applying commits to it until it is up-to-date with
* Overleaf.
*/
public class WLRepositoryResolver
implements RepositoryResolver<HttpServletRequest> {
public class WLRepositoryResolver implements RepositoryResolver<HttpServletRequest> {
private final Bridge bridge;
private final Bridge bridge;
public WLRepositoryResolver(Bridge bridge) {
this.bridge = bridge;
public WLRepositoryResolver(Bridge bridge) {
this.bridge = bridge;
}
/*
* Calls into the Bridge to resolve a project name to a JGit
* {@link Repository}, or throw an exception.
*
* On success, the repository will have been brought onto disk and updated
* to the latest (synced).
*
* In the case of clones and fetches, upload packs are created from the
* returned JGit {@link Repository} by the {@link WLUploadPackFactory}.
*
* The project lock is acquired for this process so it can't be swapped out.
*
* However, it can still be swapped out between this and a Git push. The
* push would fail due to the project changed on Overleaf between the sync
* and the actual push to Overleaf (performed by the
* {@link WLReceivePackFactory} and {@link WriteLatexPutHook}. In this case,
* the user will have to try again (which prompts another update, etc. until
* this no longer happens).
* @param httpServletRequest The HttpServletRequest as required by the
* interface. We injected the oauth2 creds into it with
* {@link Oauth2Filter}, which was set up by the {@link GitBridgeServer}.
* @param name The name of the project
* @return the JGit {@link Repository}.
* @throws RepositoryNotFoundException If the project does not exist
* @throws ServiceNotAuthorizedException If the user did not auth when
* required to
* @throws ServiceMayNotContinueException If any other general user
* exception occurs that must be propogated back to the user, e.g.
* internal errors (IOException, etc), too large file, and so on.
*/
@Override
public Repository open(HttpServletRequest httpServletRequest, String name)
throws RepositoryNotFoundException,
ServiceNotAuthorizedException,
ServiceMayNotContinueException {
Log.debug("[{}] Request to open git repo", name);
Optional<Credential> oauth2 =
Optional.ofNullable(
(Credential) httpServletRequest.getAttribute(Oauth2Filter.ATTRIBUTE_KEY));
String projName = Util.removeAllSuffixes(name, "/", ".git");
try {
return bridge.getUpdatedRepo(oauth2, projName).getJGitRepository();
} catch (RepositoryNotFoundException e) {
Log.warn("Repository not found: " + name);
throw e;
/*
} catch (ServiceNotAuthorizedException e) {
cannot occur
} catch (ServiceNotEnabledException e) {
cannot occur
*/
} catch (ServiceMayNotContinueException e) {
/* Such as FailedConnectionException */
throw e;
} catch (CannotAcquireLockException e) {
throw new ServiceMayNotContinueException(e.getMessage());
} catch (RuntimeException e) {
Log.warn("Runtime exception when trying to open repo: " + projName, e);
throw new ServiceMayNotContinueException(e);
} catch (ForbiddenException e) {
throw new ServiceNotAuthorizedException();
} catch (GitUserException e) {
throw new ServiceMayNotContinueException(
e.getMessage() + "\n" + String.join("\n", e.getDescriptionLines()), e);
} catch (IOException e) {
Log.warn("IOException when trying to open repo: " + projName, e);
throw new ServiceMayNotContinueException("Internal server error.");
}
/**
* Calls into the Bridge to resolve a project name to a JGit
* {@link Repository}, or throw an exception.
*
* On success, the repository will have been brought onto disk and updated
* to the latest (synced).
*
* In the case of clones and fetches, upload packs are created from the
* returned JGit {@link Repository} by the {@link WLUploadPackFactory}.
*
* The project lock is acquired for this process so it can't be swapped out.
*
* However, it can still be swapped out between this and a Git push. The
* push would fail due to the project changed on Overleaf between the sync
* and the actual push to Overleaf (performed by the
* {@link WLReceivePackFactory} and {@link WriteLatexPutHook}. In this case,
* the user will have to try again (which prompts another update, etc. until
* this no longer happens).
* @param httpServletRequest The HttpServletRequest as required by the
* interface. We injected the oauth2 creds into it with
* {@link Oauth2Filter}, which was set up by the {@link GitBridgeServer}.
* @param name The name of the project
* @return the JGit {@link Repository}.
* @throws RepositoryNotFoundException If the project does not exist
* @throws ServiceNotAuthorizedException If the user did not auth when
* required to
* @throws ServiceMayNotContinueException If any other general user
* exception occurs that must be propogated back to the user, e.g.
* internal errors (IOException, etc), too large file, and so on.
*/
@Override
public Repository open(
HttpServletRequest httpServletRequest,
String name
) throws RepositoryNotFoundException,
ServiceNotAuthorizedException,
ServiceMayNotContinueException {
Log.debug("[{}] Request to open git repo", name);
Optional<Credential> oauth2 = Optional.ofNullable(
(Credential) httpServletRequest.getAttribute(
Oauth2Filter.ATTRIBUTE_KEY));
String projName = Util.removeAllSuffixes(name, "/", ".git");
try {
return bridge.getUpdatedRepo(oauth2, projName).getJGitRepository();
} catch (RepositoryNotFoundException e) {
Log.warn("Repository not found: " + name);
throw e;
/*
} catch (ServiceNotAuthorizedException e) {
cannot occur
} catch (ServiceNotEnabledException e) {
cannot occur
*/
} catch (ServiceMayNotContinueException e) {
/* Such as FailedConnectionException */
throw e;
} catch (CannotAcquireLockException e) {
throw new ServiceMayNotContinueException(e.getMessage());
} catch (RuntimeException e) {
Log.warn(
"Runtime exception when trying to open repo: " + projName,
e
);
throw new ServiceMayNotContinueException(e);
} catch (ForbiddenException e) {
throw new ServiceNotAuthorizedException();
} catch (GitUserException e) {
throw new ServiceMayNotContinueException(
e.getMessage() + "\n" +
String.join("\n", e.getDescriptionLines()),
e);
} catch (IOException e) {
Log.warn(
"IOException when trying to open repo: " + projName,
e
);
throw new ServiceMayNotContinueException("Internal server error.");
}
}
}
}

44
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLUploadPackFactory.java
View File

@@ -1,44 +1,34 @@
package uk.ac.ic.wlgitbridge.git.handler;
import javax.servlet.http.HttpServletRequest;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.UploadPack;
import org.eclipse.jgit.transport.resolver.UploadPackFactory;
import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet;
import uk.ac.ic.wlgitbridge.util.Log;
import javax.servlet.http.HttpServletRequest;
/**
/*
* Created by Winston on 02/11/14.
*/
/**
/*
* One of the "big three" interfaces created by {@link WLGitServlet} to handle
* user Git requests.
*
* The actual class doesn't do much, and most of the work is done when the
* project name is being resolved by the {@link WLRepositoryResolver}.
*/
public class WLUploadPackFactory
implements UploadPackFactory<HttpServletRequest> {
/**
* This does nothing special. Synchronising the project with Overleaf will
* have been performed by {@link WLRepositoryResolver}.
* @param __ Not used, required by the {@link UploadPackFactory} interface
* @param repository The JGit repository provided by the
* {@link WLRepositoryResolver}
* @return the {@link UploadPack}, used by JGit to serve the request
*/
@Override
public UploadPack create(
HttpServletRequest __,
Repository repository
) {
Log.debug(
"[{}] Creating upload-pack",
repository.getWorkTree().getName()
);
return new UploadPack(repository);
}
public class WLUploadPackFactory implements UploadPackFactory<HttpServletRequest> {
/*
* This does nothing special. Synchronising the project with Overleaf will
* have been performed by {@link WLRepositoryResolver}.
* @param __ Not used, required by the {@link UploadPackFactory} interface
* @param repository The JGit repository provided by the
* {@link WLRepositoryResolver}
* @return the {@link UploadPack}, used by JGit to serve the request
*/
@Override
public UploadPack create(HttpServletRequest __, Repository repository) {
Log.debug("[{}] Creating upload-pack", repository.getWorkTree().getName());
return new UploadPack(repository);
}
}

251
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/WriteLatexPutHook.java
View File

@@ -1,6 +1,10 @@
package uk.ac.ic.wlgitbridge.git.handler.hook;
import com.google.api.client.auth.oauth2.Credential;
import java.io.IOException;
import java.util.Collection;
import java.util.Iterator;
import java.util.Optional;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.PreReceiveHook;
import org.eclipse.jgit.transport.ReceiveCommand;
@@ -11,176 +15,129 @@ import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.data.CannotAcquireLockException;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.handler.WLReceivePackFactory;
import uk.ac.ic.wlgitbridge.git.handler.hook.exception.ForcedPushException;
import uk.ac.ic.wlgitbridge.git.handler.hook.exception.WrongBranchException;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.InternalErrorException;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.OutOfDateException;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.IOException;
import java.util.Collection;
import java.util.Iterator;
import java.util.Optional;
/**
/*
* Created by Winston on 03/11/14.
*/
/**
/*
* Created by {@link WLReceivePackFactory} to update the {@link Bridge} for a
* user's Git push request, or fail with an error. The hook is able to approve
* or reject a request.
*/
public class WriteLatexPutHook implements PreReceiveHook {
private final RepoStore repoStore;
private final RepoStore repoStore;
private final Bridge bridge;
private final String hostname;
private final Optional<Credential> oauth2;
private final Bridge bridge;
private final String hostname;
private final Optional<Credential> oauth2;
/**
* The constructor to use, which provides the hook with the {@link Bridge},
* the hostname (used to construct a URL to give to Overleaf to postback),
* and the oauth2 (used to authenticate with the Snapshot API).
* @param repoStore
* @param bridge the {@link Bridge}
* @param hostname the hostname used for postback from the Snapshot API
* @param oauth2 used to authenticate with the snapshot API, or null
*/
public WriteLatexPutHook(
RepoStore repoStore,
Bridge bridge,
String hostname,
Optional<Credential> oauth2
) {
this.repoStore = repoStore;
this.bridge = bridge;
this.hostname = hostname;
this.oauth2 = oauth2;
/*
* The constructor to use, which provides the hook with the {@link Bridge},
* the hostname (used to construct a URL to give to Overleaf to postback),
* and the oauth2 (used to authenticate with the Snapshot API).
* @param repoStore
* @param bridge the {@link Bridge}
* @param hostname the hostname used for postback from the Snapshot API
* @param oauth2 used to authenticate with the snapshot API, or null
*/
public WriteLatexPutHook(
RepoStore repoStore, Bridge bridge, String hostname, Optional<Credential> oauth2) {
this.repoStore = repoStore;
this.bridge = bridge;
this.hostname = hostname;
this.oauth2 = oauth2;
}
@Override
public void onPreReceive(ReceivePack receivePack, Collection<ReceiveCommand> receiveCommands) {
Log.debug(
"-> Handling {} commands in {}",
receiveCommands.size(),
receivePack.getRepository().getDirectory().getAbsolutePath());
for (ReceiveCommand receiveCommand : receiveCommands) {
try {
handleReceiveCommand(oauth2, receivePack.getRepository(), receiveCommand);
} catch (IOException e) {
Log.error("IOException on pre receive", e);
receivePack.sendError(e.getMessage());
receiveCommand.setResult(Result.REJECTED_OTHER_REASON, e.getMessage());
} catch (OutOfDateException e) {
Log.error("OutOfDateException on pre receive", e);
receiveCommand.setResult(Result.REJECTED_NONFASTFORWARD);
} catch (GitUserException e) {
Log.error("GitUserException on pre receive", e);
handleSnapshotPostException(receivePack, receiveCommand, e);
} catch (CannotAcquireLockException e) {
Log.info("CannotAcquireLockException on pre receive");
receivePack.sendError(e.getMessage());
receiveCommand.setResult(Result.REJECTED_OTHER_REASON, e.getMessage());
} catch (Throwable t) {
Log.error("Throwable on pre receive", t);
handleSnapshotPostException(receivePack, receiveCommand, new InternalErrorException());
}
}
Log.debug(
"-> Handled {} commands in {}",
receiveCommands.size(),
receivePack.getRepository().getDirectory().getAbsolutePath());
}
@Override
public void onPreReceive(
ReceivePack receivePack,
Collection<ReceiveCommand> receiveCommands
) {
Log.debug("-> Handling {} commands in {}", receiveCommands.size(), receivePack.getRepository().getDirectory().getAbsolutePath());
for (ReceiveCommand receiveCommand : receiveCommands) {
try {
handleReceiveCommand(
oauth2,
receivePack.getRepository(),
receiveCommand
);
} catch (IOException e) {
Log.error("IOException on pre receive", e);
receivePack.sendError(e.getMessage());
receiveCommand.setResult(
Result.REJECTED_OTHER_REASON,
e.getMessage()
);
} catch (OutOfDateException e) {
Log.error("OutOfDateException on pre receive", e);
receiveCommand.setResult(Result.REJECTED_NONFASTFORWARD);
} catch (GitUserException e) {
Log.error("GitUserException on pre receive", e);
handleSnapshotPostException(receivePack, receiveCommand, e);
} catch (CannotAcquireLockException e) {
Log.info("CannotAcquireLockException on pre receive");
receivePack.sendError(e.getMessage());
receiveCommand.setResult(
Result.REJECTED_OTHER_REASON,
e.getMessage()
);
} catch (Throwable t) {
Log.error("Throwable on pre receive", t);
handleSnapshotPostException(
receivePack,
receiveCommand,
new InternalErrorException()
);
}
}
Log.debug("-> Handled {} commands in {}", receiveCommands.size(), receivePack.getRepository().getDirectory().getAbsolutePath());
private void handleSnapshotPostException(
ReceivePack receivePack, ReceiveCommand receiveCommand, GitUserException e) {
String message = e.getMessage();
receivePack.sendError(message);
StringBuilder msg = new StringBuilder();
for (Iterator<String> it = e.getDescriptionLines().iterator(); it.hasNext(); ) {
String line = it.next();
msg.append("hint: ");
msg.append(line);
if (it.hasNext()) {
msg.append('\n');
}
}
receivePack.sendMessage("");
receivePack.sendMessage(msg.toString());
receiveCommand.setResult(Result.REJECTED_OTHER_REASON, message);
}
private void handleSnapshotPostException(
ReceivePack receivePack,
ReceiveCommand receiveCommand,
GitUserException e
) {
String message = e.getMessage();
receivePack.sendError(message);
StringBuilder msg = new StringBuilder();
for (
Iterator<String> it = e.getDescriptionLines().iterator();
it.hasNext();
) {
String line = it.next();
msg.append("hint: ");
msg.append(line);
if (it.hasNext()) {
msg.append('\n');
}
}
receivePack.sendMessage("");
receivePack.sendMessage(msg.toString());
receiveCommand.setResult(Result.REJECTED_OTHER_REASON, message);
private void handleReceiveCommand(
Optional<Credential> oauth2, Repository repository, ReceiveCommand receiveCommand)
throws IOException, GitUserException, CannotAcquireLockException {
checkBranch(receiveCommand);
checkForcedPush(receiveCommand);
bridge.push(
oauth2,
repository.getWorkTree().getName(),
getPushedDirectoryContents(repository, receiveCommand),
getOldDirectoryContents(repository),
hostname);
}
private void checkBranch(ReceiveCommand receiveCommand) throws WrongBranchException {
if (!receiveCommand.getRefName().equals("refs/heads/master")) {
throw new WrongBranchException();
}
}
private void handleReceiveCommand(
Optional<Credential> oauth2,
Repository repository,
ReceiveCommand receiveCommand
) throws IOException, GitUserException, CannotAcquireLockException {
checkBranch(receiveCommand);
checkForcedPush(receiveCommand);
bridge.push(
oauth2,
repository.getWorkTree().getName(),
getPushedDirectoryContents(repository,
receiveCommand),
getOldDirectoryContents(repository),
hostname
);
private void checkForcedPush(ReceiveCommand receiveCommand) throws ForcedPushException {
if (receiveCommand.getType() == ReceiveCommand.Type.UPDATE_NONFASTFORWARD) {
throw new ForcedPushException();
}
}
private void checkBranch(
ReceiveCommand receiveCommand
) throws WrongBranchException {
if (!receiveCommand.getRefName().equals("refs/heads/master")) {
throw new WrongBranchException();
}
}
private void checkForcedPush(
ReceiveCommand receiveCommand
) throws ForcedPushException {
if (
receiveCommand.getType()
== ReceiveCommand.Type.UPDATE_NONFASTFORWARD
) {
throw new ForcedPushException();
}
}
private RawDirectory getPushedDirectoryContents(
Repository repository,
ReceiveCommand receiveCommand
) throws IOException, GitUserException {
return repoStore
.useJGitRepo(repository, receiveCommand.getNewId())
.getDirectory();
}
private RawDirectory getOldDirectoryContents(
Repository repository
) throws IOException, GitUserException {
return repoStore
.useJGitRepo(repository, repository.resolve("HEAD"))
.getDirectory();
}
private RawDirectory getPushedDirectoryContents(
Repository repository, ReceiveCommand receiveCommand) throws IOException, GitUserException {
return repoStore.useJGitRepo(repository, receiveCommand.getNewId()).getDirectory();
}
private RawDirectory getOldDirectoryContents(Repository repository)
throws IOException, GitUserException {
return repoStore.useJGitRepo(repository, repository.resolve("HEAD")).getDirectory();
}
}

42
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/exception/ForcedPushException.java
View File

@@ -1,37 +1,33 @@
package uk.ac.ic.wlgitbridge.git.handler.hook.exception;
import com.google.gson.JsonElement;
import uk.ac.ic.wlgitbridge.util.Util;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
import java.util.Arrays;
import java.util.List;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
import uk.ac.ic.wlgitbridge.util.Util;
/**
/*
* Created by Winston on 16/11/14.
*/
public class ForcedPushException extends SnapshotPostException {
private static final String[] DESCRIPTION_LINES = {
"You can't git push --force to a "
+ Util.getServiceName()
+ " project.",
"Try to put your changes on top of the current head.",
"If everything else fails, delete and reclone your repository, "
+ "make your changes, then push again."
};
private static final String[] DESCRIPTION_LINES = {
"You can't git push --force to a " + Util.getServiceName() + " project.",
"Try to put your changes on top of the current head.",
"If everything else fails, delete and reclone your repository, "
+ "make your changes, then push again."
};
@Override
public String getMessage() {
return "forced push prohibited";
}
@Override
public String getMessage() {
return "forced push prohibited";
}
@Override
public List<String> getDescriptionLines() {
return Arrays.asList(DESCRIPTION_LINES);
}
@Override
public void fromJSON(JsonElement json) {}
@Override
public List<String> getDescriptionLines() {
return Arrays.asList(DESCRIPTION_LINES);
}
@Override
public void fromJSON(JsonElement json) {}
}

35
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/exception/WrongBranchException.java
View File

@@ -1,34 +1,29 @@
package uk.ac.ic.wlgitbridge.git.handler.hook.exception;
import com.google.gson.JsonElement;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
import java.util.Arrays;
import java.util.List;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
/**
/*
* Created by Winston on 19/12/14.
*/
public class WrongBranchException extends SnapshotPostException {
private static final String[] DESCRIPTION_LINES = {
"You can't push any new branches.",
"Please use the master branch."
};
private static final String[] DESCRIPTION_LINES = {
"You can't push any new branches.", "Please use the master branch."
};
@Override
public String getMessage() {
return "wrong branch";
}
@Override
public String getMessage() {
return "wrong branch";
}
@Override
public List<String> getDescriptionLines() {
return Arrays.asList(DESCRIPTION_LINES);
}
@Override
public void fromJSON(JsonElement json) {
}
@Override
public List<String> getDescriptionLines() {
return Arrays.asList(DESCRIPTION_LINES);
}
@Override
public void fromJSON(JsonElement json) {}
}

46
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/servlet/WLGitServlet.java
View File

@@ -1,5 +1,6 @@
package uk.ac.ic.wlgitbridge.git.servlet;
import javax.servlet.ServletException;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jgit.http.server.GitServlet;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
@@ -7,14 +8,11 @@ import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.git.handler.WLReceivePackFactory;
import uk.ac.ic.wlgitbridge.git.handler.WLRepositoryResolver;
import uk.ac.ic.wlgitbridge.git.handler.WLUploadPackFactory;
import uk.ac.ic.wlgitbridge.server.GitBridgeServer;
import javax.servlet.ServletException;
/**
/*
* Created by Winston on 02/11/14.
*/
/**
/*
* This is the Servlet created by the {@link GitBridgeServer} that does all of
* the work in handling user Git requests and directing them to the
* {@link Bridge}.
@@ -28,25 +26,21 @@ import javax.servlet.ServletException;
*/
public class WLGitServlet extends GitServlet {
/**
* Constructor that sets all of the resolvers and factories for the
* {@link GitServlet}.
*
* Also needs to call init with a config ({@link WLGitServletConfig}, as
* required by the {@link GitServlet}.
* @param ctxHandler
* @param bridge
* @throws ServletException
*/
public WLGitServlet(
ServletContextHandler ctxHandler,
RepoStore repoStore,
Bridge bridge
) throws ServletException {
setRepositoryResolver(new WLRepositoryResolver(bridge));
setReceivePackFactory(new WLReceivePackFactory(repoStore, bridge));
setUploadPackFactory(new WLUploadPackFactory());
init(new WLGitServletConfig(ctxHandler));
}
/*
* Constructor that sets all of the resolvers and factories for the
* {@link GitServlet}.
*
* Also needs to call init with a config ({@link WLGitServletConfig}, as
* required by the {@link GitServlet}.
* @param ctxHandler
* @param bridge
* @throws ServletException
*/
public WLGitServlet(ServletContextHandler ctxHandler, RepoStore repoStore, Bridge bridge)
throws ServletException {
setRepositoryResolver(new WLRepositoryResolver(bridge));
setReceivePackFactory(new WLReceivePackFactory(repoStore, bridge));
setUploadPackFactory(new WLUploadPackFactory());
init(new WLGitServletConfig(ctxHandler));
}
}

50
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/servlet/WLGitServletConfig.java
View File

@@ -1,42 +1,40 @@
package uk.ac.ic.wlgitbridge.git.servlet;
import org.eclipse.jetty.servlet.ServletContextHandler;
import java.util.Enumeration;
import javax.servlet.ServletConfig;
import javax.servlet.ServletContext;
import java.util.Enumeration;
import org.eclipse.jetty.servlet.ServletContextHandler;
/**
/*
* Created by Winston on 02/11/14.
*/
public class WLGitServletConfig implements ServletConfig {
private static final String SERVLET_NAME = "git-servlet";
private static final String SERVLET_NAME = "git-servlet";
private ServletContext servletContext;
private ServletContext servletContext;
public WLGitServletConfig(ServletContextHandler ctxHandler) {
servletContext = ctxHandler.getServletContext();
}
public WLGitServletConfig(ServletContextHandler ctxHandler) {
servletContext = ctxHandler.getServletContext();
}
@Override
public String getServletName() {
return SERVLET_NAME;
}
@Override
public String getServletName() {
return SERVLET_NAME;
}
@Override
public ServletContext getServletContext() {
return servletContext;
}
@Override
public ServletContext getServletContext() {
return servletContext;
}
@Override
public String getInitParameter(String s) {
return null;
}
@Override
public Enumeration<String> getInitParameterNames() {
return null;
}
@Override
public String getInitParameter(String s) {
return null;
}
@Override
public Enumeration<String> getInitParameterNames() {
return null;
}
}

135
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/util/RepositoryObjectTreeWalker.java
View File

@@ -1,5 +1,10 @@
package uk.ac.ic.wlgitbridge.git.util;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.Repository;
@@ -12,91 +17,67 @@ import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile;
import uk.ac.ic.wlgitbridge.git.exception.InvalidGitRepository;
import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
/**
/*
* Created by Winston on 16/11/14.
*/
public class RepositoryObjectTreeWalker {
private final TreeWalk treeWalk;
private final Repository repository;
private final TreeWalk treeWalk;
private final Repository repository;
public RepositoryObjectTreeWalker(
Repository repository,
ObjectId objectId
) throws IOException {
treeWalk = initTreeWalk(repository, objectId);
this.repository = repository;
public RepositoryObjectTreeWalker(Repository repository, ObjectId objectId) throws IOException {
treeWalk = initTreeWalk(repository, objectId);
this.repository = repository;
}
public RepositoryObjectTreeWalker(Repository repository) throws IOException {
this(repository, 0);
}
public RepositoryObjectTreeWalker(Repository repository, int fromHead) throws IOException {
this(repository, repository.resolve("HEAD~" + fromHead));
}
public RawDirectory getDirectoryContents(Optional<Long> maxFileSize)
throws IOException, SizeLimitExceededException, InvalidGitRepository {
return new RawDirectory(walkGitObjectTree(maxFileSize));
}
private TreeWalk initTreeWalk(Repository repository, ObjectId objectId) throws IOException {
if (objectId == null) {
return null;
}
RevWalk walk = new RevWalk(repository);
TreeWalk treeWalk = new TreeWalk(repository);
treeWalk.addTree(walk.parseCommit(objectId).getTree());
treeWalk.setRecursive(true);
return treeWalk;
}
public RepositoryObjectTreeWalker(
Repository repository
) throws IOException {
this(repository, 0);
private Map<String, RawFile> walkGitObjectTree(Optional<Long> maxFileSize)
throws IOException, SizeLimitExceededException, InvalidGitRepository {
Map<String, RawFile> fileContentsTable = new HashMap<>();
if (treeWalk == null) {
return fileContentsTable;
}
while (treeWalk.next()) {
String path = treeWalk.getPathString();
public RepositoryObjectTreeWalker(
Repository repository,
int fromHead
) throws IOException {
this(repository, repository.resolve("HEAD~" + fromHead));
ObjectId objectId = treeWalk.getObjectId(0);
if (!repository.hasObject(objectId)) {
throw new InvalidGitRepository();
}
ObjectLoader obj = repository.open(objectId);
long size = obj.getSize();
if (maxFileSize.isPresent() && size > maxFileSize.get()) {
throw new SizeLimitExceededException(Optional.ofNullable(path), size, maxFileSize.get());
}
try (ByteArrayOutputStream o = new ByteArrayOutputStream(CastUtil.assumeInt(size))) {
obj.copyTo(o);
fileContentsTable.put(path, new RepositoryFile(path, o.toByteArray()));
}
;
}
public RawDirectory getDirectoryContents(Optional<Long> maxFileSize)
throws IOException,
SizeLimitExceededException,
InvalidGitRepository {
return new RawDirectory(walkGitObjectTree(maxFileSize));
}
private TreeWalk initTreeWalk(
Repository repository,
ObjectId objectId
) throws IOException {
if (objectId == null) {
return null;
}
RevWalk walk = new RevWalk(repository);
TreeWalk treeWalk = new TreeWalk(repository);
treeWalk.addTree(walk.parseCommit(objectId).getTree());
treeWalk.setRecursive(true);
return treeWalk;
}
private Map<String, RawFile> walkGitObjectTree(Optional<Long> maxFileSize)
throws IOException,
SizeLimitExceededException,
InvalidGitRepository {
Map<String, RawFile> fileContentsTable = new HashMap<>();
if (treeWalk == null) {
return fileContentsTable;
}
while (treeWalk.next()) {
String path = treeWalk.getPathString();
ObjectId objectId = treeWalk.getObjectId(0);
if (!repository.hasObject(objectId)) {
throw new InvalidGitRepository();
}
ObjectLoader obj = repository.open(objectId);
long size = obj.getSize();
if (maxFileSize.isPresent() && size > maxFileSize.get()) {
throw new SizeLimitExceededException(
Optional.ofNullable(path), size, maxFileSize.get());
}
try (ByteArrayOutputStream o = new ByteArrayOutputStream(
CastUtil.assumeInt(size))) {
obj.copyTo(o);
fileContentsTable.put(
path, new RepositoryFile(path, o.toByteArray()));
};
}
return fileContentsTable;
}
return fileContentsTable;
}
}

96
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/io/http/ning/NingHttpClient.java
View File

@@ -1,81 +1,67 @@
package uk.ac.ic.wlgitbridge.io.http.ning;
import io.netty.handler.codec.http.HttpHeaders;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.concurrent.ExecutionException;
import org.asynchttpclient.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import uk.ac.ic.wlgitbridge.util.FunctionT;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.concurrent.ExecutionException;
public class NingHttpClient implements NingHttpClientFacade {
private static final Logger log
= LoggerFactory.getLogger(NingHttpClient.class);
private static final Logger log = LoggerFactory.getLogger(NingHttpClient.class);
private final AsyncHttpClient http;
private final AsyncHttpClient http;
public NingHttpClient(AsyncHttpClient http) {
this.http = http;
}
public NingHttpClient(AsyncHttpClient http) {
this.http = http;
}
@Override
public <E extends Exception> byte[] get(
String url,
FunctionT<HttpHeaders, Boolean, E> handler
) throws ExecutionException {
try {
return http
.prepareGet(url)
.execute(new AsyncCompletionHandler<byte[]>() {
@Override
public <E extends Exception> byte[] get(String url, FunctionT<HttpHeaders, Boolean, E> handler)
throws ExecutionException {
try {
return http.prepareGet(url)
.execute(
new AsyncCompletionHandler<byte[]>() {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
@Override
public State onHeadersReceived(
HttpHeaders headers
) throws E {
return handler.apply(headers)
? State.CONTINUE : State.ABORT;
public State onHeadersReceived(HttpHeaders headers) throws E {
return handler.apply(headers) ? State.CONTINUE : State.ABORT;
}
@Override
public State onBodyPartReceived(
HttpResponseBodyPart content
) throws IOException {
bytes.write(content.getBodyPartBytes());
return State.CONTINUE;
public State onBodyPartReceived(HttpResponseBodyPart content) throws IOException {
bytes.write(content.getBodyPartBytes());
return State.CONTINUE;
}
@Override
public byte[] onCompleted(
Response response
) throws Exception {
int statusCode = response.getStatusCode();
if (statusCode >= 400) {
throw new Exception("got status " + statusCode +
" fetching " + url);
}
byte[] ret = bytes.toByteArray();
bytes.close();
log.debug(
statusCode
+ " "
+ response.getStatusText()
+ " ("
+ ret.length
+ "B) -> "
+ url
);
return ret;
public byte[] onCompleted(Response response) throws Exception {
int statusCode = response.getStatusCode();
if (statusCode >= 400) {
throw new Exception("got status " + statusCode + " fetching " + url);
}
byte[] ret = bytes.toByteArray();
bytes.close();
log.debug(
statusCode
+ " "
+ response.getStatusText()
+ " ("
+ ret.length
+ "B) -> "
+ url);
return ret;
}
}).get();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
})
.get();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
}

24
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/io/http/ning/NingHttpClientFacade.java
View File

@@ -1,22 +1,18 @@
package uk.ac.ic.wlgitbridge.io.http.ning;
import io.netty.handler.codec.http.HttpHeaders;
import uk.ac.ic.wlgitbridge.util.FunctionT;
import java.util.concurrent.ExecutionException;
import uk.ac.ic.wlgitbridge.util.FunctionT;
public interface NingHttpClientFacade {
/**
* Performs a GET request
* @param url the target URL
* @param handler handler for the response headers. Returning false
* aborts the request.
* @return
*/
<E extends Exception> byte[] get(
String url,
FunctionT<HttpHeaders, Boolean, E> handler
) throws ExecutionException;
/*
* Performs a GET request
* @param url the target URL
* @param handler handler for the response headers. Returning false
* aborts the request.
* @return
*/
<E extends Exception> byte[] get(String url, FunctionT<HttpHeaders, Boolean, E> handler)
throws ExecutionException;
}

1
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/BasicAuthCredentials.java
View File

@@ -17,5 +17,4 @@ public class BasicAuthCredentials {
public String getPassword() {
return password;
}
}

46
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/DiagnosticsHandler.java
View File

@@ -1,40 +1,26 @@
package uk.ac.ic.wlgitbridge.server;
import org.eclipse.jetty.server.HttpConnection;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.handler.AbstractHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import javax.management.JMException;
import javax.management.ObjectName;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.Writer;
import java.lang.management.ManagementFactory;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.handler.AbstractHandler;
import uk.ac.ic.wlgitbridge.util.Log;
public class DiagnosticsHandler extends AbstractHandler {
public DiagnosticsHandler() {
}
public DiagnosticsHandler() {}
@Override
public void handle(
String target,
Request baseRequest,
HttpServletRequest request,
HttpServletResponse response
) throws IOException, ServletException {
String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
throws IOException, ServletException {
String method = baseRequest.getMethod();
if (
("GET".equals(method))
&& target != null
&& target.matches("^/diags/?$")
) {
if (("GET".equals(method)) && target != null && target.matches("^/diags/?$")) {
baseRequest.setHandled(true);
Log.debug(method + " <- /diags");
@@ -45,7 +31,7 @@ public class DiagnosticsHandler extends AbstractHandler {
try {
detail = execute("vmNativeMemory", "detail");
summary = execute("vmNativeMemory", "summary");
} catch(JMException e) {
} catch (JMException e) {
Log.error("Failed to get native memory detail: " + e.getMessage());
response.setStatus(500);
return;
@@ -62,10 +48,12 @@ public class DiagnosticsHandler extends AbstractHandler {
}
public static String execute(String command, String... args) throws JMException {
return (String) ManagementFactory.getPlatformMBeanServer().invoke(
new ObjectName("com.sun.management:type=DiagnosticCommand"),
command,
new Object[]{args},
new String[]{"[Ljava.lang.String;"});
return (String)
ManagementFactory.getPlatformMBeanServer()
.invoke(
new ObjectName("com.sun.management:type=DiagnosticCommand"),
command,
new Object[] {args},
new String[] {"[Ljava.lang.String;"});
}
}

Some files were not shown because too many files have changed in this diff Show More