agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-05-26 18:51:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #17 from overleaf/swapstore

Browse Source 
v0.5: Upgrade deps, huge refactor, swap store
This commit is contained in:
Winston Li
2016-09-12 15:58:54 +01:00
committed by GitHub
parent 1850689a63 e9ffcfe19e
commit 184cdd7783
150 changed files with 4337 additions and 1013 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
services/git-bridge/.gitignore vendored
View File

@@ -1,6 +1,11 @@
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# Let's not share anything because we're using Maven.
.idea
*.iml
# User-specific stuff:
.idea/workspace.xml
.idea/tasks.xml

32
services/git-bridge/.idea/compiler.xml generated
View File

@@ -1,32 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<resourceExtensions />
<wildcardResourcePatterns>
<entry name="!?*.java" />
<entry name="!?*.form" />
<entry name="!?*.class" />
<entry name="!?*.groovy" />
<entry name="!?*.scala" />
<entry name="!?*.flex" />
<entry name="!?*.kt" />
<entry name="!?*.clj" />
<entry name="!?*.aj" />
</wildcardResourcePatterns>
<annotationProcessing>
<profile default="true" name="Default" enabled="false">
<processorPath useClasspath="true" />
</profile>
<profile default="false" name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<processorPath useClasspath="true" />
<module name="writelatex-git-bridge" />
</profile>
</annotationProcessing>
<bytecodeTargetLevel>
<module name="writelatex-git-bridge" target="1.8" />
</bytecodeTargetLevel>
</component>
</project>

3
services/git-bridge/.idea/copyright/profiles_settings.xml generated
View File

@@ -1,3 +0,0 @@
<component name="CopyrightManager">
<settings default="" />
</component>

7
services/git-bridge/.idea/encodings.xml generated
View File

@@ -1,7 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<file url="file://$PROJECT_DIR$" charset="UTF-8" />
<file url="PROJECT" charset="UTF-8" />
</component>
</project>

42
services/git-bridge/.idea/misc.xml generated
View File

@@ -1,42 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="EntryPointsManager">
<entry_points version="2.0" />
</component>
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectLevelVcsManager" settingsEditedManually="false">
<OptionsSetting value="true" id="Add" />
<OptionsSetting value="true" id="Remove" />
<OptionsSetting value="true" id="Checkout" />
<OptionsSetting value="true" id="Update" />
<OptionsSetting value="true" id="Status" />
<OptionsSetting value="true" id="Edit" />
<ConfirmationsSetting value="0" id="Add" />
<ConfirmationsSetting value="0" id="Remove" />
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="true" assert-keyword="true" jdk-15="true" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
<component name="masterDetails">
<states>
<state key="ProjectJDKs.UI">
<settings>
<last-edited>1.8</last-edited>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
</states>
</component>
</project>

8
services/git-bridge/.idea/modules.xml generated
View File

@@ -1,8 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/writelatex-git-bridge.iml" filepath="$PROJECT_DIR$/writelatex-git-bridge.iml" />
</modules>
</component>
</project>

58
services/git-bridge/README.md
View File

@@ -3,9 +3,23 @@ writelatex-git-bridge
Required
--------
* `maven` (for building)
* `maven` (for building, running tests and packaging)
* `jdk-8` (for compiling and running)
Commands
--------
To be run from the base directory:
**Build jar**:
`mvn package`
**Run tests**:
`mvn test`
**Clean**:
`mvn clean`
Installation
------------
@@ -32,16 +46,46 @@ The configuration file is in `.json` format.
{
"port" (int): the port number,
"rootGitDirectory" (string): the directory in which to store git repos and the db/atts,
"rootGitDirectory" (string): the directory in which to store
git repos and the db/atts,
"apiBaseUrl" (string): base url for the snapshot api,
"username" (string, optional): username for http basic auth,
"password" (string, optional): password for http basic auth,
"postbackBaseUrl" (string): the postback url,
"serviceName" (string): current name of writeLaTeX in case it ever changes
"oauth2" (object): { /* null or missing if oauth2 shouldn't be used */
"oauth2ClientID" (string): oauth2 client ID
"oauth2ClientSecret" (string): oauth2 client secret
"oauth2Server" (string): oauth2 server, with protocol and without trailing slash
"serviceName" (string): current name of writeLaTeX
in case it ever changes,
"oauth2" (object): { null or missing if oauth2 shouldn't be used
"oauth2ClientID" (string): oauth2 client ID,
"oauth2ClientSecret" (string): oauth2 client secret,
"oauth2Server" (string): oauth2 server,
with protocol and
without trailing slash
},
"swapStore" (object, optional): { the place to swap projects to.
if null, type defaults to
"noop"
"type" (string): "s3", "memory", "noop" (not recommended),
"awsAccessKey" (string, optional): only for s3,
"awsSecret" (string, optional): only for s3,
"s3BucketName" (string, optional): only for s3
},
"swapJob" (object, optional): { configure the project
swapping job.
if null, defaults to no-op
"minProjects" (int64): lower bound on number of projects
present. The swap job will never go
below this, regardless of what the
watermark shows. Regardless, if
minProjects prevents an eviction,
the swap job will WARN,
"lowGiB" (int32): the low watermark for swapping,
i.e. swap until disk usage is below this,
"highGiB" (int32): the high watermark for swapping,
i.e. start swapping when
disk usage becomes this,
"intervalMillis" (int64): amount of time in between running
swap job and checking watermarks.
3600000 is 1 hour
}
}

26
services/git-bridge/conf/example_config.json Normal file
View File

@@ -0,0 +1,26 @@
{
"port": 8080,
"rootGitDirectory": "/tmp/wlgb",
"apiBaseUrl": "https://localhost/api/v0",
"username": "user",
"password": "pass",
"postbackBaseUrl": "https://localhost",
"serviceName": "Overleaf",
"oauth2": {
"oauth2ClientID": "asdf",
"oauth2ClientSecret": "asdf",
"oauth2Server": "https://localhost"
},
"swapStore": {
"type": "s3",
"awsAccessKey": "asdf",
"awsSecret": "asdf",
"s3BucketName": "com.overleaf.testbucket"
},
"swapJob": {
"minProjects": 50,
"lowGiB": 128,
"highGiB": 256,
"intervalMillis": 3600000
}
}

BIN
services/git-bridge/lib/newrelic.jar Normal file
View File

Binary file not shown.

300
services/git-bridge/newrelic.yml Normal file
View File

@@ -0,0 +1,300 @@
# This file configures the New Relic Agent. New Relic monitors
# Java applications with deep visibility and low overhead. For more details and additional
# configuration options visit https://docs.newrelic.com/docs/java/java-agent-configuration.
#
# This configuration file is custom generated for Winston
#
# This section is for settings common to all environments.
# Do not add anything above this next line.
common: &default_settings
# ============================== LICENSE KEY ===============================
# You must specify the license key associated with your New Relic
# account. For example, if your license key is 12345 use this:
# license_key: '12345'
# The key binds your Agent's data to your account in the New Relic service.
license_key: '<LICENSE KEY>'
# Agent Enabled
# Use this setting to disable the agent instead of removing it from the startup command.
# Default is true.
agent_enabled: true
# Set the name of your application as you'd like it show up in New Relic.
# If enable_auto_app_naming is false, the agent reports all data to this application.
# Otherwise, the agent reports only background tasks (transactions for non-web applications)
# to this application. To report data to more than one application
# (useful for rollup reporting), separate the application names with ";".
# For example, to report data to "My Application" and "My Application 2" use this:
# app_name: My Application;My Application 2
# This setting is required. Up to 3 different application names can be specified.
# The first application name must be unique.
app_name: Git Bridge
# To enable high security, set this property to true. When in high
# security mode, the agent will use SSL and obfuscated SQL. Additionally,
# request parameters and message parameters will not be sent to New Relic.
high_security: false
# Set to true to enable support for auto app naming.
# The name of each web app is detected automatically
# and the agent reports data separately for each one.
# This provides a finer-grained performance breakdown for
# web apps in New Relic.
# Default is false.
enable_auto_app_naming: false
# Set to true to enable component-based transaction naming.
# Set to false to use the URI of a web request as the name of the transaction.
# Default is true.
enable_auto_transaction_naming: true
# The agent uses its own log file to keep its logging
# separate from that of your application. Specify the log level here.
# This setting is dynamic, so changes do not require restarting your application.
# The levels in increasing order of verboseness are:
# off, severe, warning, info, fine, finer, finest
# Default is info.
log_level: info
# Log all data sent to and from New Relic in plain text.
# This setting is dynamic, so changes do not require restarting your application.
# Default is false.
audit_mode: false
# The number of backup log files to save.
# Default is 1.
log_file_count: 1
# The maximum number of kbytes to write to any one log file.
# The log_file_count must be set greater than 1.
# Default is 0 (no limit).
log_limit_in_kbytes: 0
# Override other log rolling configuration and roll the logs daily.
# Default is false.
log_daily: false
# The name of the log file.
# Default is newrelic_agent.log.
log_file_name: newrelic_agent.log
# The log file directory.
# Default is the logs directory in the newrelic.jar parent directory.
#log_file_path:
# The agent communicates with New Relic via https by
# default. If you want to communicate with newrelic via http,
# then turn off SSL by setting this value to false.
# This work is done asynchronously to the threads that process your
# application code, so response times will not be directly affected
# by this change.
# Default is true.
ssl: true
# Proxy settings for connecting to the New Relic server:
# If a proxy is used, the host setting is required. Other settings
# are optional. Default port is 8080. The username and password
# settings will be used to authenticate to Basic Auth challenges
# from a proxy server.
#proxy_host: hostname
#proxy_port: 8080
#proxy_user: username
#proxy_password: password
# Limits the number of lines to capture for each stack trace.
# Default is 30
max_stack_trace_lines: 30
# Provides the ability to configure the attributes sent to New Relic. These
# attributes can be found in transaction traces, traced errors, Insight's
# transaction events, and Insight's page views.
attributes:
# When true, attributes will be sent to New Relic. The default is true.
enabled: true
#A comma separated list of attribute keys whose values should
# be sent to New Relic.
#include:
# A comma separated list of attribute keys whose values should
# not be sent to New Relic.
#exclude:
# Transaction tracer captures deep information about slow
# transactions and sends this to the New Relic service once a
# minute. Included in the transaction is the exact call sequence of
# the transactions including any SQL statements issued.
transaction_tracer:
# Transaction tracer is enabled by default. Set this to false to turn it off.
# This feature is not available to Lite accounts and is automatically disabled.
# Default is true.
enabled: true
# Threshold in seconds for when to collect a transaction
# trace. When the response time of a controller action exceeds
# this threshold, a transaction trace will be recorded and sent to
# New Relic. Valid values are any float value, or (default) "apdex_f",
# which will use the threshold for the "Frustrated" Apdex level
# (greater than four times the apdex_t value).
# Default is apdex_f.
transaction_threshold: apdex_f
# When transaction tracer is on, SQL statements can optionally be
# recorded. The recorder has three modes, "off" which sends no
# SQL, "raw" which sends the SQL statement in its original form,
# and "obfuscated", which strips out numeric and string literals.
# Default is obfuscated.
record_sql: obfuscated
# Set this to true to log SQL statements instead of recording them.
# SQL is logged using the record_sql mode.
# Default is false.
log_sql: false
# Threshold in seconds for when to collect stack trace for a SQL
# call. In other words, when SQL statements exceed this threshold,
# then capture and send to New Relic the current stack trace. This is
# helpful for pinpointing where long SQL calls originate from.
# Default is 0.5 seconds.
stack_trace_threshold: 0.5
# Determines whether the agent will capture query plans for slow
# SQL queries. Only supported for MySQL and PostgreSQL.
# Default is true.
explain_enabled: true
# Threshold for query execution time below which query plans will not
# not be captured. Relevant only when `explain_enabled` is true.
# Default is 0.5 seconds.
explain_threshold: 0.5
# Use this setting to control the variety of transaction traces.
# The higher the setting, the greater the variety.
# Set this to 0 to always report the slowest transaction trace.
# Default is 20.
top_n: 20
# Error collector captures information about uncaught exceptions and
# sends them to New Relic for viewing.
error_collector:
# This property enables the collection of errors. If the property is not
# set or the property is set to false, then errors will not be collected.
# Default is true.
enabled: true
# Use this property to exclude specific exceptions from being reported as errors
# by providing a comma separated list of full class names.
# The default is to exclude akka.actor.ActorKilledException. If you want to override
# this, you must provide any new value as an empty list is ignored.
ignore_errors: akka.actor.ActorKilledException
# Use this property to exclude specific http status codes from being reported as errors
# by providing a comma separated list of status codes.
# The default is to exclude 404s. If you want to override
# this, you must provide any new value as an empty list is ignored.
ignore_status_codes: 404
# Transaction Events are used for Histograms and Percentiles. Unaggregated data is collected
# for each web transaction and sent to the server on harvest.
transaction_events:
# Set to false to disable transaction events.
# Default is true.
enabled: true
# Events are collected up to the configured amount. Afterwards, events are sampled to
# maintain an even distribution across the harvest cycle.
# Default is 2000. Setting to 0 will disable.
max_samples_stored: 2000
# Cross Application Tracing adds request and response headers to
# external calls using supported HTTP libraries to provide better
# performance data when calling applications monitored by other New Relic Agents.
cross_application_tracer:
# Set to false to disable cross application tracing.
# Default is true.
enabled: true
# Thread profiler measures wall clock time, CPU time, and method call counts
# in your application's threads as they run.
# This feature is not available to Lite accounts and is automatically disabled.
thread_profiler:
# Set to false to disable the thread profiler.
# Default is true.
enabled: true
# New Relic Real User Monitoring gives you insight into the performance real users are
# experiencing with your website. This is accomplished by measuring the time it takes for
# your users' browsers to download and render your web pages by injecting a small amount
# of JavaScript code into the header and footer of each page.
browser_monitoring:
# By default the agent automatically inserts API calls in compiled JSPs to
# inject the monitoring JavaScript into web pages. Not all rendering engines are supported.
# See https://docs.newrelic.com/docs/java/real-user-monitoring-in-java#manual_instrumentation
# for instructions to add these manually to your pages.
# Set this attribute to false to turn off this behavior.
auto_instrument: true
class_transformer:
# This instrumentation reports the name of the user principal returned from
# HttpServletRequest.getUserPrincipal() when servlets and filters are invoked.
com.newrelic.instrumentation.servlet-user:
enabled: false
com.newrelic.instrumentation.spring-aop-2:
enabled: false
# Classes loaded by classloaders in this list will not be instrumented.
# This is a useful optimization for runtimes which use classloaders to
# load dynamic classes which the agent would not instrument.
classloader_excludes:
groovy.lang.GroovyClassLoader$InnerLoader,
org.codehaus.groovy.runtime.callsite.CallSiteClassLoader,
com.collaxa.cube.engine.deployment.BPELClassLoader,
org.springframework.data.convert.ClassGeneratingEntityInstantiator$ObjectInstantiatorClassGenerator,
org.mvel2.optimizers.impl.asm.ASMAccessorOptimizer$ContextClassLoader,
gw.internal.gosu.compiler.SingleServingGosuClassLoader,
# User-configurable custom labels for this agent. Labels are name-value pairs.
# There is a maximum of 64 labels per agent. Names and values are limited to 255 characters.
# Names and values may not contain colons (:) or semicolons (;).
labels:
# An example label
#label_name: label_value
# Application Environments
# ------------------------------------------
# Environment specific settings are in this section.
# You can use the environment to override the default settings.
# For example, to change the app_name setting.
# Use -Dnewrelic.environment=<environment> on the Java startup command line
# to set the environment.
# The default environment is production.
# NOTE if your application has other named environments, you should
# provide configuration settings for these environments here.
development:
<<: *default_settings
app_name: Git Bridge (Development)
test:
<<: *default_settings
app_name: Git Bridge (Test)
production:
<<: *default_settings
staging:
<<: *default_settings
app_name: Git Bridge (Staging)

24
services/git-bridge/pom.xml
View File

@@ -140,5 +140,29 @@
<artifactId>mockito-core</artifactId>
<version>1.10.19</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.11.28</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.5</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-compress -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.12</version>
</dependency>
</dependencies>
</project>

4
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/GitBridgeApp.java
View File

@@ -22,10 +22,10 @@ public class GitBridgeApp implements Runnable {
public static final int EXIT_CODE_FAILED = 1;
private static final String USAGE_MESSAGE =
"usage: writelatex-git-bridge config_file";
"usage: writelatex-git-bridge [config_file]";
private String configFilePath;
private Config config;
Config config;
private GitBridgeServer server;
/**

51
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java
View File

@@ -4,12 +4,16 @@ import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import uk.ac.ic.wlgitbridge.application.exception.ConfigFileException;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStoreConfig;
import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
import uk.ac.ic.wlgitbridge.util.Instance;
import javax.annotation.Nullable;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.Optional;
/**
* Created by Winston on 05/12/14.
@@ -25,7 +29,9 @@ public class Config implements JSONSource {
config.apiBaseURL,
config.postbackURL,
config.serviceName,
Oauth2.asSanitised(config.oauth2)
Oauth2.asSanitised(config.oauth2),
SwapStoreConfig.sanitisedCopy(config.swapStore),
config.swapJob
);
}
@@ -36,7 +42,12 @@ public class Config implements JSONSource {
private String apiBaseURL;
private String postbackURL;
private String serviceName;
@Nullable
private Oauth2 oauth2;
@Nullable
private SwapStoreConfig swapStore;
@Nullable
private SwapJobConfig swapJob;
public Config(String configFilePath) throws ConfigFileException,
IOException {
@@ -47,14 +58,18 @@ public class Config implements JSONSource {
fromJSON(new Gson().fromJson(reader, JsonElement.class));
}
public Config(int port,
String rootGitDirectory,
String username,
String password,
String apiBaseURL,
String postbackURL,
String serviceName,
Oauth2 oauth2) {
public Config(
int port,
String rootGitDirectory,
String username,
String password,
String apiBaseURL,
String postbackURL,
String serviceName,
Oauth2 oauth2,
SwapStoreConfig swapStore,
SwapJobConfig swapJob
) {
this.port = port;
this.rootGitDirectory = rootGitDirectory;
this.username = username;
@@ -63,6 +78,8 @@ public class Config implements JSONSource {
this.postbackURL = postbackURL;
this.serviceName = serviceName;
this.oauth2 = oauth2;
this.swapStore = swapStore;
this.swapJob = swapJob;
}
@Override
@@ -89,6 +106,14 @@ public class Config implements JSONSource {
postbackURL += "/";
}
oauth2 = new Gson().fromJson(configObject.get("oauth2"), Oauth2.class);
swapStore = new Gson().fromJson(
configObject.get("swapStore"),
SwapStoreConfig.class
);
swapJob = new Gson().fromJson(
configObject.get("swapJob"),
SwapJobConfig.class
);
}
public String getSanitisedString() {
@@ -134,6 +159,14 @@ public class Config implements JSONSource {
return oauth2;
}
public Optional<SwapStoreConfig> getSwapStore() {
return Optional.ofNullable(swapStore);
}
public Optional<SwapJobConfig> getSwapJob() {
return Optional.ofNullable(swapJob);
}
private JsonElement getElement(JsonObject configObject, String name) {
JsonElement element = configObject.get(name);
if (element == null) {

222
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java
View File

@@ -3,15 +3,17 @@ package uk.ac.ic.wlgitbridge.bridge;
import com.google.api.client.auth.oauth2.Credential;
import org.eclipse.jgit.transport.ServiceMayNotContinueException;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache;
import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotAPI;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapJobImpl;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
import uk.ac.ic.wlgitbridge.data.ProjectLockImpl;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
@@ -29,7 +31,10 @@ import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.*;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.File;
import java.io.IOException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.util.*;
/**
@@ -42,18 +47,19 @@ public class Bridge {
private final RepoStore repoStore;
private final DBStore dbStore;
private final SwapStore swapStore;
private final SwapJob swapJob;
private final SnapshotAPI snapshotAPI;
private final ResourceCache resourceCache;
private final SwapJob swapJob;
private final PostbackManager postbackManager;
public static Bridge make(
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore
SwapStore swapStore,
Optional<SwapJobConfig> swapJobConfig
) {
ProjectLock lock = new ProjectLockImpl((int threads) ->
Log.info("Waiting for " + threads + " projects...")
@@ -63,14 +69,15 @@ public class Bridge {
repoStore,
dbStore,
swapStore,
new NetSnapshotAPI(),
new UrlResourceCache(dbStore),
new SwapJobImpl(
SwapJob.fromConfig(
swapJobConfig,
lock,
repoStore,
dbStore,
swapStore
)
),
new NetSnapshotAPI(),
new UrlResourceCache(dbStore)
);
}
@@ -79,9 +86,9 @@ public class Bridge {
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore,
SwapJob swapJob,
SnapshotAPI snapshotAPI,
ResourceCache resourceCache,
SwapJob swapJob
ResourceCache resourceCache
) {
this.lock = lock;
this.repoStore = repoStore;
@@ -104,66 +111,140 @@ public class Bridge {
Log.info("Bye");
}
public void startSwapJob(int intervalMillis) {
swapJob.start(intervalMillis);
public void startSwapJob() {
swapJob.start();
}
/* TODO: Remove these when WLBridged is moved into RepoStore */
public void lockForProject(String projectName) {
lock.lockForProject(projectName);
public void checkDB() {
Log.info("Checking DB");
File rootDir = repoStore.getRootDirectory();
for (File f : rootDir.listFiles()) {
if (f.getName().equals(".wlgb")) {
continue;
}
String projName = f.getName();
try (LockGuard __ = lock.lockGuard(projName)) {
File dotGit = new File(f, ".git");
if (!dotGit.exists()) {
Log.warn("Project: {} has no .git", projName);
continue;
}
ProjectState state = dbStore.getProjectState(projName);
if (state != ProjectState.NOT_PRESENT) {
continue;
}
Log.warn("Project: {} not in swap_store, adding", projName);
dbStore.setLastAccessedTime(
projName,
new Timestamp(dotGit.lastModified())
);
}
}
}
public void unlockForProject(String projectName) {
lock.unlockForProject(projectName);
}
public boolean repositoryExists(Credential oauth2, String projectName)
throws ServiceMayNotContinueException, GitUserException {
lockForProject(projectName);
GetDocRequest getDocRequest = new GetDocRequest(oauth2, projectName);
getDocRequest.request();
try {
public boolean projectExists(
Credential oauth2,
String projectName
) throws ServiceMayNotContinueException,
GitUserException {
try (LockGuard __ = lock.lockGuard(projectName)) {
GetDocRequest getDocRequest = new GetDocRequest(
oauth2,
projectName
);
getDocRequest.request();
getDocRequest.getResult().getVersionID();
return true;
} catch (InvalidProjectException e) {
return false;
} finally {
unlockForProject(projectName);
}
return true;
}
public void getWritableRepositories(
public void updateRepository(
Credential oauth2,
ProjectRepo repo
) throws IOException,
GitUserException {
Log.info("[{}] Fetching", repo.getProjectName());
updateProjectWithName(oauth2, repo);
) throws IOException, GitUserException {
String projectName = repo.getProjectName();
try (LockGuard __ = lock.lockGuard(projectName)) {
Log.info("[{}] Updating", projectName);
updateRepositoryCritical(oauth2, repo);
}
}
public void
putDirectoryContentsToProjectWithName(Credential oauth2,
String projectName,
RawDirectory directoryContents,
RawDirectory oldDirectoryContents,
String hostname)
throws SnapshotPostException, IOException, ForbiddenException {
lock.lockForProject(projectName);
CandidateSnapshot candidate = null;
try {
Log.info("[{}] Pushing", projectName);
String postbackKey = postbackManager.makeKeyForProject(projectName);
Log.info(
"[{}] Created postback key: {}",
private void updateRepositoryCritical(
Credential oauth2,
ProjectRepo repo
) throws IOException, GitUserException {
String projectName = repo.getProjectName();
ProjectState state = dbStore.getProjectState(projectName);
switch (state) {
case NOT_PRESENT:
repo.initRepo(repoStore);
break;
case SWAPPED:
swapJob.restore(projectName);
/* Fallthrough */
default:
repo.useExistingRepository(repoStore);
}
updateProject(oauth2, repo);
dbStore.setLastAccessedTime(
projectName,
Timestamp.valueOf(LocalDateTime.now())
);
}
public void putDirectoryContentsToProjectWithName(
Credential oauth2,
String projectName,
RawDirectory directoryContents,
RawDirectory oldDirectoryContents,
String hostname
) throws SnapshotPostException, IOException, ForbiddenException {
try (LockGuard __ = lock.lockGuard(projectName)) {
pushToProjectCritical(
oauth2,
projectName,
postbackKey
directoryContents,
oldDirectoryContents
);
candidate =
createCandidateSnapshot(
projectName,
directoryContents,
oldDirectoryContents
);
} catch (SevereSnapshotPostException e) {
Log.warn("[" + projectName + "] Failed to put to Overleaf", e);
throw e;
} catch (SnapshotPostException e) {
/* Stack trace should be printed further up */
Log.warn(
"[{}] Exception when waiting for postback: {}",
projectName,
e.getClass().getSimpleName()
);
throw e;
} catch (IOException e) {
Log.warn("[{}] IOException on put", projectName);
throw e;
}
}
private void pushToProjectCritical(
Credential oauth2,
String projectName,
RawDirectory directoryContents,
RawDirectory oldDirectoryContents
) throws IOException, ForbiddenException, SnapshotPostException {
Log.info("[{}] Pushing", projectName);
String postbackKey = postbackManager.makeKeyForProject(projectName);
Log.info(
"[{}] Created postback key: {}",
projectName,
postbackKey
);
try (
CandidateSnapshot candidate = createCandidateSnapshot(
projectName,
directoryContents,
oldDirectoryContents
);
) {
Log.info(
"[{}] Candindate snapshot created: {}",
projectName,
@@ -195,6 +276,10 @@ public class Bridge {
projectName,
versionID
);
dbStore.setLastAccessedTime(
projectName,
Timestamp.valueOf(LocalDateTime.now())
);
} else {
Log.warn(
"[{}] Went out of date while waiting for push",
@@ -202,31 +287,6 @@ public class Bridge {
);
throw new OutOfDateException();
}
} catch (SevereSnapshotPostException e) {
Log.warn("[" + projectName + "] Failed to put to Overleaf", e);
throw e;
} catch (SnapshotPostException e) {
/* Stack trace should be printed further up */
Log.warn(
"[{}] Exception when waiting for postback: {}",
projectName,
e.getClass().getSimpleName()
);
throw e;
} catch (IOException e) {
Log.warn("[{}] IOException on put", projectName);
throw e;
} finally {
if (candidate != null) {
candidate.deleteServletFiles();
} else {
Log.error(
"[{}] Candidate snapshot was null: " +
"this should never happen.",
projectName
);
}
lock.unlockForProject(projectName);
}
}
@@ -266,7 +326,7 @@ public class Bridge {
/* PRIVATE */
private void updateProjectWithName(
private void updateProject(
Credential oauth2,
ProjectRepo repo
) throws IOException, GitUserException {

64
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/GitProjectRepo.java
View File

@@ -1,33 +1,36 @@
package uk.ac.ic.wlgitbridge.bridge;
import com.google.common.base.Preconditions;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.Project;
import uk.ac.ic.wlgitbridge.util.Util;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
import java.util.*;
/**
* Created by winston on 20/08/2016.
*/
public class GitProjectRepo implements ProjectRepo {
private final Repository repository;
private final String projectName;
private Optional<Repository> repository;
public GitProjectRepo(Repository repository, String projectName) {
this.repository = repository;
public GitProjectRepo(String projectName) {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
this.projectName = projectName;
repository = Optional.empty();
}
@Override
@@ -35,11 +38,34 @@ public class GitProjectRepo implements ProjectRepo {
return projectName;
}
@Override
public void initRepo(
RepoStore repoStore
) throws IOException {
initRepositoryField(repoStore);
Preconditions.checkState(repository.isPresent());
Repository repo = this.repository.get();
Preconditions.checkState(!repo.getObjectDatabase().exists());
repo.create();
}
@Override
public void useExistingRepository(
RepoStore repoStore
) throws IOException {
initRepositoryField(repoStore);
Preconditions.checkState(repository.isPresent());
Preconditions.checkState(
repository.get().getObjectDatabase().exists()
);
}
@Override
public Map<String, RawFile> getFiles()
throws IOException, GitUserException {
Preconditions.checkState(repository.isPresent());
return new RepositoryObjectTreeWalker(
repository
repository.get()
).getDirectoryContents().getFileTable();
}
@@ -54,13 +80,33 @@ public class GitProjectRepo implements ProjectRepo {
}
}
public Repository getJGitRepository() {
return repository.get();
}
private void initRepositoryField(RepoStore repoStore) throws IOException {
Preconditions.checkNotNull(repoStore);
Preconditions.checkArgument(Project.isValidProjectName(projectName));
Preconditions.checkState(!repository.isPresent());
repository = Optional.of(createJGitRepository(repoStore, projectName));
}
private Repository createJGitRepository(
RepoStore repoStore,
String projName
) throws IOException {
File repoDir = new File(repoStore.getRootDirectory(), projName);
return new FileRepositoryBuilder().setWorkTree(repoDir).build();
}
private Collection<String> doCommitAndGetMissing(
GitDirectoryContents contents
) throws IOException, GitAPIException {
Preconditions.checkState(repository.isPresent());
String name = getProjectName();
Log.info("[{}] Writing commit", name);
contents.write();
Git git = new Git(repository);
Git git = new Git(repository.get());
Log.info("[{}] Getting missing files", name);
Set<String> missingFiles = git.status().call().getMissing();
for (String missing : missingFiles) {

9
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/ProjectRepo.java
View File

@@ -1,5 +1,6 @@
package uk.ac.ic.wlgitbridge.bridge;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
@@ -15,6 +16,14 @@ public interface ProjectRepo {
String getProjectName();
void initRepo(
RepoStore repoStore
) throws IOException;
void useExistingRepository(
RepoStore repoStore
) throws IOException;
Map<String, RawFile> getFiles() throws IOException, GitUserException;
Collection<String> commitAndGetMissing(

73
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/WLBridgedProject.java
View File

@@ -1,73 +0,0 @@
package uk.ac.ic.wlgitbridge.bridge;
import com.google.api.client.auth.oauth2.Credential;
import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.ServiceMayNotContinueException;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.exception.InvalidProjectException;
import java.io.IOException;
/**
* Created by Winston on 05/11/14.
*/
public class WLBridgedProject {
private final Repository repository;
private final String name;
private final Bridge bridgeAPI;
public WLBridgedProject(Repository repository, String name, Bridge bridgeAPI) {
this.repository = repository;
this.name = name;
this.bridgeAPI = bridgeAPI;
}
public void buildRepository(Credential oauth2) throws RepositoryNotFoundException, ServiceMayNotContinueException, GitUserException {
bridgeAPI.lockForProject(name);
try {
if (repository.getObjectDatabase().exists()) {
updateRepositoryFromSnapshots(oauth2, repository);
} else {
buildRepositoryFromScratch(oauth2, repository);
}
} catch (RuntimeException e) {
e.printStackTrace();
throw new ServiceMayNotContinueException(e);
} finally {
bridgeAPI.unlockForProject(name);
}
}
private void updateRepositoryFromSnapshots(
Credential oauth2,
Repository repository
) throws RepositoryNotFoundException,
ServiceMayNotContinueException,
GitUserException {
try {
bridgeAPI.getWritableRepositories(
oauth2,
new GitProjectRepo(repository, name)
);
} catch (InvalidProjectException e) {
throw new RepositoryNotFoundException(name);
} catch (IOException e) {
throw new ServiceMayNotContinueException(e);
}
}
private void buildRepositoryFromScratch(Credential oauth2, Repository repository) throws RepositoryNotFoundException, ServiceMayNotContinueException, GitUserException {
if (!bridgeAPI.repositoryExists(oauth2, name)) {
throw new RepositoryNotFoundException(name);
}
try {
repository.create();
} catch (IOException e) {
throw new ServiceMayNotContinueException(e);
}
updateRepositoryFromSnapshots(oauth2, repository);
}
}

20
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBInitException.java Normal file
View File

@@ -0,0 +1,20 @@
package uk.ac.ic.wlgitbridge.bridge.db;
/**
* Created by winston on 23/08/2016.
*/
public class DBInitException extends RuntimeException {
public DBInitException(String message) {
super(message);
}
public DBInitException(String message, Throwable cause) {
super(message, cause);
}
public DBInitException(Throwable cause) {
super(cause);
}
}

16
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java
View File

@@ -1,5 +1,6 @@
package uk.ac.ic.wlgitbridge.bridge.db;
import java.sql.Timestamp;
import java.util.List;
/**
@@ -7,6 +8,8 @@ import java.util.List;
*/
public interface DBStore {
int getNumProjects();
List<String> getProjectNames();
void setLatestVersionForProject(String project, int versionID);
@@ -19,4 +22,17 @@ public interface DBStore {
String getPathForURLInProject(String projectName, String url);
String getOldestUnswappedProject();
int getNumUnswappedProjects();
ProjectState getProjectState(String projectName);
/**
* Sets the last accessed time for the given project name.
* @param projectName the project's name
* @param time the time, or null if the project is to be swapped
*/
void setLastAccessedTime(String projectName, Timestamp time);
}

12
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/ProjectState.java Normal file
View File

@@ -0,0 +1,12 @@
package uk.ac.ic.wlgitbridge.bridge.db;
/**
* Created by winston on 24/08/2016.
*/
public enum ProjectState {
NOT_PRESENT,
PRESENT,
SWAPPED
}

87
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/SqliteDBStore.java
View File

@@ -1,87 +0,0 @@
package uk.ac.ic.wlgitbridge.bridge.db;
import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLiteWLDatabase;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.File;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.List;
/**
* Created by winston on 20/08/2016.
*/
public class SqliteDBStore implements DBStore {
private final SQLiteWLDatabase database;
public SqliteDBStore(File rootDirectory) {
try {
database = new SQLiteWLDatabase(rootDirectory);
} catch (SQLException e) {
throw new RuntimeException(e);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
}
@Override
public List<String> getProjectNames() {
try {
return database.getProjectNames();
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
@Override
public void setLatestVersionForProject(String project, int versionID) {
try {
database.setVersionIDForProject(project, versionID);
Log.info("[{}] Wrote latest versionId: {}", project, versionID);
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
@Override
public int getLatestVersionForProject(String project) {
try {
return database.getVersionIDForProjectName(project);
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
@Override
public void addURLIndexForProject(String projectName, String url, String path) {
try {
database.addURLIndex(projectName, url, path);
Log.info("[{}] Wrote url index: {} -> {}", projectName, url, path);
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
@Override
public void deleteFilesForProject(String project, String... files) {
try {
database.deleteFilesForProject(project, files);
Log.info(
"[{}] Deleting from url index: {}",
project,
Arrays.toString(files)
);
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
@Override
public String getPathForURLInProject(String projectName, String url) {
try {
return database.getPathForURLInProject(projectName, url);
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
}

2
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/SQLQuery.java → services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SQLQuery.java
View File

@@ -1,4 +1,4 @@
package uk.ac.ic.wlgitbridge.data.model.db.sql;
package uk.ac.ic.wlgitbridge.bridge.db.sqlite;
import java.sql.ResultSet;
import java.sql.SQLException;

18
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SQLUpdate.java Normal file
View File

@@ -0,0 +1,18 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite;
import java.sql.PreparedStatement;
import java.sql.SQLException;
/**
* Created by Winston on 20/11/14.
*/
public interface SQLUpdate {
String getSQL();
default void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
}
}

204
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java Normal file
View File

@@ -0,0 +1,204 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite;
import com.google.common.base.Preconditions;
import uk.ac.ic.wlgitbridge.bridge.db.DBInitException;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.query.*;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter.ProjectsAddLastAccessed;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create.CreateIndexURLIndexStore;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create.CreateProjectsIndexLastAccessed;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create.CreateProjectsTableSQLUpdate;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create.CreateURLIndexStoreSQLUpdate;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete.DeleteFilesForProjectSQLUpdate;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert.AddURLIndexSQLUpdate;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert.SetProjectLastAccessedTime;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert.SetProjectSQLUpdate;
import java.io.File;
import java.sql.*;
import java.util.List;
import java.util.stream.Stream;
/**
* Created by Winston on 17/11/14.
*/
public class SqliteDBStore implements DBStore {
private final Connection connection;
public SqliteDBStore(File dbFile) {
try {
connection = openConnectionTo(dbFile);
createTables();
} catch (Throwable t) {
throw new DBInitException(t);
}
}
@Override
public int getNumProjects() {
return query(new GetNumProjects());
}
@Override
public List<String> getProjectNames() {
return query(new GetProjectNamesSQLQuery());
}
@Override
public void setLatestVersionForProject(
String projectName,
int versionID
) {
update(new SetProjectSQLUpdate(projectName, versionID));
}
@Override
public int getLatestVersionForProject(
String projectName
) {
return query(new GetLatestVersionForProjectSQLQuery(projectName));
}
@Override
public void addURLIndexForProject(
String projectName,
String url,
String path
) {
update(new AddURLIndexSQLUpdate(projectName, url, path));
}
@Override
public void deleteFilesForProject(
String projectName,
String... paths
) {
update(new DeleteFilesForProjectSQLUpdate(projectName, paths));
}
@Override
public String getPathForURLInProject(
String projectName,
String url
) {
return query(new GetPathForURLInProjectSQLQuery(projectName, url));
}
@Override
public String getOldestUnswappedProject() {
return query(new GetOldestProjectName());
}
@Override
public int getNumUnswappedProjects() {
return query(new GetNumUnswappedProjects());
}
@Override
public ProjectState getProjectState(String projectName) {
return query(new GetProjectState(projectName));
}
@Override
public void setLastAccessedTime(
String projectName,
Timestamp lastAccessed
) {
update(new SetProjectLastAccessedTime(projectName, lastAccessed));
}
private Connection openConnectionTo(File dbFile) {
File parentDir = dbFile.getParentFile();
if (!parentDir.exists() && !parentDir.mkdirs()) {
throw new DBInitException(
parentDir.getAbsolutePath() + " directory didn't exist, " +
"and unable to create. Check your permissions."
);
}
try {
Class.forName("org.sqlite.JDBC");
} catch (ClassNotFoundException e) {
throw new DBInitException(e);
}
try {
return DriverManager.getConnection(
"jdbc:sqlite:" + dbFile.getAbsolutePath()
);
} catch (SQLException e) {
throw new DBInitException("Unable to connect to DB", e);
}
}
private void createTables() {
try {
doUpdate(new ProjectsAddLastAccessed());
} catch (SQLException ignore) {
/* We need to eat exceptions from here */
}
Stream.of(
new CreateProjectsTableSQLUpdate(),
new CreateProjectsIndexLastAccessed(),
new CreateURLIndexStoreSQLUpdate(),
new CreateIndexURLIndexStore()
).forEach(this::update);
/* In the case of needing to change the schema, we need to check that
ProjectsAddLastAccessed didn't just fail */
Preconditions.checkState(query(new LastAccessedColumnExists()));
}
private void update(SQLUpdate update) {
try {
doUpdate(update);
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
private <T> T query(SQLQuery<T> query) {
try {
return doQuery(query);
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
private void doUpdate(SQLUpdate update) throws SQLException {
PreparedStatement statement = null;
try {
statement = connection.prepareStatement(update.getSQL());
update.addParametersToStatement(statement);
statement.executeUpdate();
} catch (SQLException e) {
throw e;
} finally {
try {
statement.close();
} catch (Throwable t) {
throw new SQLException(t);
}
}
}
private <T> T doQuery(SQLQuery<T> query) throws SQLException {
PreparedStatement statement = null;
ResultSet results = null;
try {
statement = connection.prepareStatement(query.getSQL());
query.addParametersToStatement(statement);
results = statement.executeQuery();
return query.processResultSet(results);
} catch (SQLException e) {
throw e;
} finally {
if (statement != null) {
statement.close();
}
if (results != null) {
results.close();
}
}
}
}

4
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/query/GetLatestVersionForProjectSQLQuery.java → services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetLatestVersionForProjectSQLQuery.java
View File

@@ -1,6 +1,6 @@
package uk.ac.ic.wlgitbridge.data.model.db.sql.query;
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLQuery;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.PreparedStatement;
import java.sql.ResultSet;

30
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumProjects.java Normal file
View File

@@ -0,0 +1,30 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by winston on 24/08/2016.
*/
public class GetNumProjects implements SQLQuery<Integer> {
private static final String GET_NUM_PROJECTS =
"SELECT COUNT(*)\n" +
" FROM `projects`";
@Override
public String getSQL() {
return GET_NUM_PROJECTS;
}
@Override
public Integer processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
return resultSet.getInt("COUNT(*)");
}
throw new IllegalStateException("Count always returns results");
}
}

31
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumUnswappedProjects.java Normal file
View File

@@ -0,0 +1,31 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by winston on 24/08/2016.
*/
public class GetNumUnswappedProjects implements SQLQuery<Integer> {
private static final String GET_NUM_UNSWAPPED_PROJECTS =
"SELECT COUNT(*)\n" +
" FROM `projects`\n" +
" WHERE `last_accessed` IS NOT NULL";
@Override
public String getSQL() {
return GET_NUM_UNSWAPPED_PROJECTS;
}
@Override
public Integer processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
return resultSet.getInt("COUNT(*)");
}
throw new IllegalStateException("Count always returns results");
}
}

30
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetOldestProjectName.java Normal file
View File

@@ -0,0 +1,30 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by winston on 23/08/2016.
*/
public class GetOldestProjectName implements SQLQuery<String> {
private static final String GET_OLDEST_PROJECT_NAME =
"SELECT `name`, MIN(`last_accessed`)\n" +
" FROM `projects`";
@Override
public String getSQL() {
return GET_OLDEST_PROJECT_NAME;
}
@Override
public String processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
return resultSet.getString("name");
}
return null;
}
}

4
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/query/GetPathForURLInProjectSQLQuery.java → services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetPathForURLInProjectSQLQuery.java
View File

@@ -1,6 +1,6 @@
package uk.ac.ic.wlgitbridge.data.model.db.sql.query;
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLQuery;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.PreparedStatement;
import java.sql.ResultSet;

10
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/query/GetProjectNamesSQLQuery.java → services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetProjectNamesSQLQuery.java
View File

@@ -1,8 +1,7 @@
package uk.ac.ic.wlgitbridge.data.model.db.sql.query;
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLQuery;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.LinkedList;
@@ -30,9 +29,4 @@ public class GetProjectNamesSQLQuery implements SQLQuery<List<String>> {
return GET_URL_INDEXES_FOR_PROJECT_NAME;
}
@Override
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
}
}

47
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetProjectState.java Normal file
View File

@@ -0,0 +1,47 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by winston on 24/08/2016.
*/
public class GetProjectState implements SQLQuery<ProjectState> {
private static final String GET_PROJECT_STATE =
"SELECT `last_accessed`\n" +
" FROM `projects`\n" +
" WHERE `name` = ?";
private final String projectName;
public GetProjectState(String projectName) {
this.projectName = projectName;
}
@Override
public String getSQL() {
return GET_PROJECT_STATE;
}
@Override
public ProjectState processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getTimestamp("last_accessed") == null) {
return ProjectState.SWAPPED;
}
return ProjectState.PRESENT;
}
return ProjectState.NOT_PRESENT;
}
@Override
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
statement.setString(1, projectName);
}
}

31
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/LastAccessedColumnExists.java Normal file
View File

@@ -0,0 +1,31 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by winston on 04/09/2016.
*/
public class LastAccessedColumnExists implements SQLQuery<Boolean> {
private static final String LAST_ACCESSED_COLUMN_EXISTS =
"PRAGMA table_info(`projects`)";
@Override
public String getSQL() {
return LAST_ACCESSED_COLUMN_EXISTS;
}
@Override
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getString(2).equals("last_accessed")) {
return true;
}
}
return false;
}
}

19
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddLastAccessed.java Normal file
View File

@@ -0,0 +1,19 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
* Created by winston on 03/09/2016.
*/
public class ProjectsAddLastAccessed implements SQLUpdate {
private static final String PROJECTS_ADD_LAST_ACCESSED =
"ALTER TABLE `projects`\n" +
"ADD COLUMN `last_accessed` DATETIME NULL DEFAULT 0";
@Override
public String getSQL() {
return PROJECTS_ADD_LAST_ACCESSED;
}
}

19
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateIndexURLIndexStore.java Normal file
View File

@@ -0,0 +1,19 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
* Created by Winston on 21/02/15.
*/
public class CreateIndexURLIndexStore implements SQLUpdate {
public static final String CREATE_INDEX_URL_INDEX_STORE =
"CREATE UNIQUE INDEX IF NOT EXISTS `project_path_index` " +
"ON `url_index_store`(`project_name`, `path`);\n";
@Override
public String getSQL() {
return CREATE_INDEX_URL_INDEX_STORE;
}
}

19
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsIndexLastAccessed.java Normal file
View File

@@ -0,0 +1,19 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
* Created by winston on 23/08/2016.
*/
public class CreateProjectsIndexLastAccessed implements SQLUpdate {
private static final String CREATE_PROJECTS_INDEX_LAST_ACCESSED =
"CREATE INDEX IF NOT EXISTS `projects_index_last_accessed`\n" +
" ON `projects`(`last_accessed`)";
@Override
public String getSQL() {
return CREATE_PROJECTS_INDEX_LAST_ACCESSED;
}
}

23
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsTableSQLUpdate.java Normal file
View File

@@ -0,0 +1,23 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
* Created by Winston on 20/11/14.
*/
public class CreateProjectsTableSQLUpdate implements SQLUpdate {
private static final String CREATE_PROJECTS_TABLE =
"CREATE TABLE IF NOT EXISTS `projects` (\n" +
" `name` VARCHAR NOT NULL DEFAULT '',\n" +
" `version_id` INT NOT NULL DEFAULT 0,\n" +
" `last_accessed` DATETIME NULL DEFAULT 0,\n" +
" PRIMARY KEY (`name`)\n" +
")";
@Override
public String getSQL() {
return CREATE_PROJECTS_TABLE;
}
}

18
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/create/CreateURLIndexStoreSQLUpdate.java → services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateURLIndexStoreSQLUpdate.java
View File

@@ -1,9 +1,6 @@
package uk.ac.ic.wlgitbridge.data.model.db.sql.update.create;
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
* Created by Winston on 20/11/14.
@@ -16,7 +13,11 @@ public class CreateURLIndexStoreSQLUpdate implements SQLUpdate {
" `url` text NOT NULL,\n"+
" `path` text NOT NULL,\n"+
" PRIMARY KEY (`project_name`,`url`),\n"+
" CONSTRAINT `url_index_store_ibfk_1` FOREIGN KEY (`project_name`) REFERENCES `projects` (`name`) ON DELETE CASCADE ON UPDATE CASCADE\n"+
" CONSTRAINT `url_index_store_ibfk_1` " +
"FOREIGN KEY (`project_name`) " +
"REFERENCES `projects` (`name`) " +
"ON DELETE CASCADE " +
"ON UPDATE CASCADE\n"+
");\n";
@Override
@@ -24,9 +25,4 @@ public class CreateURLIndexStoreSQLUpdate implements SQLUpdate {
return CREATE_URL_INDEX_STORE;
}
@Override
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
}
}

15
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/delete/DeleteFilesForProjectSQLUpdate.java → services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteFilesForProjectSQLUpdate.java
View File

@@ -1,6 +1,6 @@
package uk.ac.ic.wlgitbridge.data.model.db.sql.update.delete;
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete;
import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLUpdate;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
@@ -11,7 +11,8 @@ import java.sql.SQLException;
public class DeleteFilesForProjectSQLUpdate implements SQLUpdate {
private static final String DELETE_URL_INDEXES_FOR_PROJECT_NAME =
"DELETE FROM `url_index_store` WHERE `project_name` = ? AND path IN (";
"DELETE FROM `url_index_store` " +
"WHERE `project_name` = ? AND path IN (";
private final String projectName;
private final String[] paths;
@@ -23,7 +24,9 @@ public class DeleteFilesForProjectSQLUpdate implements SQLUpdate {
@Override
public String getSQL() {
StringBuilder sb = new StringBuilder(DELETE_URL_INDEXES_FOR_PROJECT_NAME);
StringBuilder sb = new StringBuilder(
DELETE_URL_INDEXES_FOR_PROJECT_NAME
);
for (int i = 0; i < paths.length; i++) {
sb.append("?");
if (i < paths.length - 1) {
@@ -35,7 +38,9 @@ public class DeleteFilesForProjectSQLUpdate implements SQLUpdate {
}
@Override
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setString(1, projectName);
for (int i = 0; i < paths.length; i++) {
statement.setString(i + 2, paths[i]);

15
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/insert/AddURLIndexSQLUpdate.java → services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/AddURLIndexSQLUpdate.java
View File

@@ -1,6 +1,6 @@
package uk.ac.ic.wlgitbridge.data.model.db.sql.update.insert;
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLUpdate;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
@@ -11,7 +11,12 @@ import java.sql.SQLException;
public class AddURLIndexSQLUpdate implements SQLUpdate {
private static final String ADD_URL_INDEX =
"INSERT OR REPLACE INTO `url_index_store` (`project_name`, `url`, `path`) VALUES (?, ?, ?);\n";
"INSERT OR REPLACE INTO `url_index_store`(" +
"`project_name`, " +
"`url`, " +
"`path`" +
") VALUES " +
"(?, ?, ?)\n";
private final String projectName;
private final String url;
@@ -29,7 +34,9 @@ public class AddURLIndexSQLUpdate implements SQLUpdate {
}
@Override
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setString(1, projectName);
statement.setString(2, url);
statement.setString(3, path);

43
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/SetProjectLastAccessedTime.java Normal file
View File

@@ -0,0 +1,43 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
/**
* Created by winston on 23/08/2016.
*/
public class SetProjectLastAccessedTime implements SQLUpdate {
private static final String SET_PROJECT_LAST_ACCESSED_TIME =
"UPDATE `projects`\n" +
"SET `last_accessed` = ?\n" +
"WHERE `name` = ?";
private final String projectName;
private final Timestamp lastAccessed;
public SetProjectLastAccessedTime(
String projectName,
Timestamp lastAccessed
) {
this.projectName = projectName;
this.lastAccessed = lastAccessed;
}
@Override
public String getSQL() {
return SET_PROJECT_LAST_ACCESSED_TIME;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setTimestamp(1, lastAccessed);
statement.setString(2, projectName);
}
}

7
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/insert/SetProjectSQLUpdate.java → services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/SetProjectSQLUpdate.java
View File

@@ -1,6 +1,6 @@
package uk.ac.ic.wlgitbridge.data.model.db.sql.update.insert;
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLUpdate;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
@@ -11,7 +11,8 @@ import java.sql.SQLException;
public class SetProjectSQLUpdate implements SQLUpdate {
private static final String SET_PROJECT =
"INSERT OR REPLACE INTO `projects`(`name`, `version_id`) VALUES (?, ?);\n";
"INSERT OR REPLACE INTO `projects`(`name`, `version_id`, `last_accessed`) " +
"VALUES (?, ?, DATETIME('now'));\n";
private final String projectName;
private final int versionID;

10
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/LockGuard.java Normal file
View File

@@ -0,0 +1,10 @@
package uk.ac.ic.wlgitbridge.bridge.lock;
/**
* Created by winston on 24/08/2016.
*/
public interface LockGuard extends AutoCloseable {
void close();
}

8
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/ProjectLock.java
View File

@@ -4,9 +4,17 @@ package uk.ac.ic.wlgitbridge.bridge.lock;
* Created by winston on 20/08/2016.
*/
public interface ProjectLock {
void lockAll();
void lockForProject(String projectName);
void unlockForProject(String projectName);
/* RAII hahaha */
default LockGuard lockGuard(String projectName) {
lockForProject(projectName);
return () -> unlockForProject(projectName);
}
}

112
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java Normal file
View File

@@ -0,0 +1,112 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import uk.ac.ic.wlgitbridge.util.Project;
import uk.ac.ic.wlgitbridge.util.Tar;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import static uk.ac.ic.wlgitbridge.util.Util.deleteInDirectoryApartFrom;
/**
* Created by winston on 20/08/2016.
*/
public class FSGitRepoStore implements RepoStore {
private final String repoStorePath;
private final File rootDirectory;
public FSGitRepoStore(String repoStorePath) {
this.repoStorePath = repoStorePath;
rootDirectory = initRootGitDirectory(repoStorePath);
}
@Override
public String getRepoStorePath() {
return repoStorePath;
}
@Override
public File getRootDirectory() {
return rootDirectory;
}
/* TODO: Perhaps we should just delete bad directories on the fly. */
@Override
public void purgeNonexistentProjects(
Collection<String> existingProjectNames
) {
List<String> excludedFromDeletion =
new ArrayList<>(existingProjectNames);
excludedFromDeletion.add(".wlgb");
deleteInDirectoryApartFrom(
rootDirectory,
excludedFromDeletion.toArray(new String[] {})
);
}
@Override
public long totalSize() {
return FileUtils.sizeOfDirectory(rootDirectory);
}
@Override
public InputStream bzip2Project(
String projectName,
long[] sizePtr
) throws IOException {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
return Tar.bz2.zip(getDotGitForProject(projectName), sizePtr);
}
@Override
public void remove(String projectName) throws IOException {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
FileUtils.deleteDirectory(new File(rootDirectory, projectName));
}
@Override
public void unbzip2Project(
String projectName,
InputStream dataStream
) throws IOException {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
Preconditions.checkState(getDirForProject(projectName).mkdirs());
Tar.bz2.unzip(dataStream, getDirForProject(projectName));
}
private File getDirForProject(String projectName) {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
return Paths.get(
rootDirectory.getAbsolutePath()
).resolve(
projectName
).toFile();
}
private File getDotGitForProject(String projectName) {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
return Paths.get(
rootDirectory.getAbsolutePath()
).resolve(
projectName
).resolve(
".git"
).toFile();
}
private File initRootGitDirectory(String rootGitDirectoryPath) {
File rootGitDirectory = new File(rootGitDirectoryPath);
rootGitDirectory.mkdirs();
Preconditions.checkArgument(rootGitDirectory.isDirectory());
return rootGitDirectory;
}
}

52
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java
View File

@@ -1,52 +0,0 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import uk.ac.ic.wlgitbridge.util.Util;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
/**
* Created by winston on 20/08/2016.
*/
public class FSRepoStore implements RepoStore {
private final String repoStorePath;
private final File rootDirectory;
public FSRepoStore(String repoStorePath) {
this.repoStorePath = repoStorePath;
rootDirectory = initRootGitDirectory(repoStorePath);
}
@Override
public String getRepoStorePath() {
return repoStorePath;
}
@Override
public File getRootDirectory() {
return rootDirectory;
}
@Override
public void purgeNonexistentProjects(
Collection<String> existingProjectNames
) {
List<String> excludedFromDeletion =
new ArrayList<>(existingProjectNames);
excludedFromDeletion.add(".wlgb");
Util.deleteInDirectoryApartFrom(
rootDirectory,
excludedFromDeletion.toArray(new String[] {})
);
}
private File initRootGitDirectory(String rootGitDirectoryPath) {
File rootGitDirectory = new File(rootGitDirectoryPath);
rootGitDirectory.mkdirs();
return rootGitDirectory;
}
}

40
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java
View File

@@ -1,8 +1,8 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
/**
@@ -10,13 +10,47 @@ import java.util.Collection;
*/
public interface RepoStore {
/* Still need to get rid of these two methods.
Main dependency: GitRepoStore needs a Repository which needs a directory.
Instead, use a visitor or something. */
String getRepoStorePath();
File getRootDirectory();
void purgeNonexistentProjects(
Collection<String> existingProjectNames
);
long totalSize();
/**
* Tars and bzip2s the .git directory of the given project. Throws an
* IOException if the project doesn't exist. The returned stream is a copy
* of the original .git directory, which must be deleted using remove().
*/
InputStream bzip2Project(
String projectName,
long[] sizePtr
) throws IOException;
default InputStream bzip2Project(
String projectName
) throws IOException {
return bzip2Project(projectName, null);
}
void remove(String projectName) throws IOException;
/**
* Unbzip2s the given data stream into a .git directory for projectName.
* Creates the project directory.
* If projectName already exists, throws an IOException.
* @param projectName the name of the project, e.g. abc123
* @param dataStream the data stream containing the bzipped contents.
*/
void unbzip2Project(
String projectName,
InputStream dataStream
) throws IOException;
}

12
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java
View File

@@ -1,12 +0,0 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapJob {
void start(int intervalMillis);
void stop();
}

53
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java
View File

@@ -1,53 +0,0 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.util.Util;
import java.util.Timer;
/**
* Created by winston on 20/08/2016.
*/
public class SwapJobImpl implements SwapJob {
private final ProjectLock lock;
private final RepoStore repoStore;
private final SwapStore swapStore;
private final DBStore dbStore;
private final Timer timer;
public SwapJobImpl(
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore, SwapStore swapStore
) {
this.lock = lock;
this.repoStore = repoStore;
this.swapStore = swapStore;
this.dbStore = dbStore;
timer = new Timer();
}
@Override
public void start(int intervalMillis) {
timer.scheduleAtFixedRate(
Util.makeTimerTask(this::doSwap),
0,
intervalMillis
);
}
@Override
public void stop() {
timer.cancel();
}
private void doSwap() {
throw new UnsupportedOperationException();
}
}

7
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java
View File

@@ -1,7 +0,0 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapStore {
}

30
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/NoopSwapJob.java Normal file
View File

@@ -0,0 +1,30 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
import java.io.IOException;
/**
* Created by winston on 24/08/2016.
*/
public class NoopSwapJob implements SwapJob {
@Override
public void start() {
}
@Override
public void stop() {
}
@Override
public void evict(String projName) throws IOException {
}
@Override
public void restore(String projName) throws IOException {
}
}

42
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java Normal file
View File

@@ -0,0 +1,42 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import java.io.IOException;
import java.util.Optional;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapJob {
static SwapJob fromConfig(
Optional<SwapJobConfig> cfg,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore
) {
if (cfg.isPresent()) {
return new SwapJobImpl(
cfg.get(),
lock,
repoStore,
dbStore,
swapStore
);
}
return new NoopSwapJob();
}
void start();
void stop();
void evict(String projName) throws IOException;
void restore(String projName) throws IOException;
}

41
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java Normal file
View File

@@ -0,0 +1,41 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
/**
* Created by winston on 23/08/2016.
*/
public class SwapJobConfig {
private final int minProjects;
private final int lowGiB;
private final int highGiB;
private final long intervalMillis;
public SwapJobConfig(
int minProjects,
int lowGiB,
int highGiB,
long intervalMillis
) {
this.minProjects = minProjects;
this.lowGiB = lowGiB;
this.highGiB = highGiB;
this.intervalMillis = intervalMillis;
}
public int getMinProjects() {
return minProjects;
}
public int getLowGiB() {
return lowGiB;
}
public int getHighGiB() {
return highGiB;
}
public long getIntervalMillis() {
return intervalMillis;
}
}

175
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java Normal file
View File

@@ -0,0 +1,175 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.IOException;
import java.io.InputStream;
import java.sql.Timestamp;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.Timer;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Created by winston on 20/08/2016.
*/
public class SwapJobImpl implements SwapJob {
private static final long GiB = (1l << 30);
int minProjects;
long lowWatermarkBytes;
long highWatermarkBytes;
Duration interval;
private final ProjectLock lock;
private final RepoStore repoStore;
private final DBStore dbStore;
private final SwapStore swapStore;
private final Timer timer;
final AtomicInteger swaps;
public SwapJobImpl(
SwapJobConfig cfg,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore
) {
this(
cfg.getMinProjects(),
GiB * cfg.getLowGiB(),
GiB * cfg.getHighGiB(),
Duration.ofMillis(cfg.getIntervalMillis()),
lock,
repoStore,
dbStore,
swapStore
);
}
SwapJobImpl(
int minProjects,
long lowWatermarkBytes,
long highWatermarkBytes,
Duration interval,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore
) {
this.minProjects = minProjects;
this.lowWatermarkBytes = lowWatermarkBytes;
this.highWatermarkBytes = highWatermarkBytes;
this.interval = interval;
this.lock = lock;
this.repoStore = repoStore;
this.dbStore = dbStore;
this.swapStore = swapStore;
timer = new Timer();
swaps = new AtomicInteger(0);
}
@Override
public void start() {
timer.schedule(
uk.ac.ic.wlgitbridge.util.Timer.makeTimerTask(this::doSwap),
0
);
}
@Override
public void stop() {
timer.cancel();
}
private void doSwap() {
try {
doSwap_();
} catch (Throwable t) {
Log.warn("Exception thrown during swap job", t);
}
timer.schedule(
uk.ac.ic.wlgitbridge.util.Timer.makeTimerTask(this::doSwap),
interval.toMillis()
);
}
private void doSwap_() {
Log.info("Running swap number {}", swaps.get() + 1);
long totalSize = repoStore.totalSize();
Log.info("Size is {}/{} (high)", totalSize, highWatermarkBytes);
if (totalSize < highWatermarkBytes) {
Log.info("No need to swap.");
swaps.incrementAndGet();
return;
}
int numProjects = dbStore.getNumProjects();
while (
(totalSize = repoStore.totalSize()) > lowWatermarkBytes &&
(numProjects = dbStore.getNumUnswappedProjects()) > minProjects
) {
try {
evict(dbStore.getOldestUnswappedProject());
} catch (IOException e) {
Log.warn("Exception while swapping, giving up", e);
}
}
if (totalSize > lowWatermarkBytes) {
Log.warn(
"Finished swapping, but total size is still too high."
);
}
Log.info(
"Size: {}/{} (low), " +
"{} (high), " +
"projects on disk: {}/{}, " +
"min projects on disk: {}",
totalSize,
lowWatermarkBytes,
highWatermarkBytes,
numProjects,
dbStore.getNumProjects(),
minProjects
);
swaps.incrementAndGet();
}
@Override
public void evict(String projName) throws IOException {
Preconditions.checkNotNull(projName);
Log.info("Evicting project: {}", projName);
try (LockGuard __ = lock.lockGuard(projName)) {
long[] sizePtr = new long[1];
InputStream bzipped = repoStore.bzip2Project(projName, sizePtr);
swapStore.upload(projName, bzipped, sizePtr[0]);
dbStore.setLastAccessedTime(projName, null);
repoStore.remove(projName);
}
Log.info("Evicted project: {}", projName);
}
@Override
public void restore(String projName) throws IOException {
try (LockGuard __ = lock.lockGuard(projName)) {
repoStore.unbzip2Project(
projName,
swapStore.openDownloadStream(projName)
);
swapStore.remove(projName);
dbStore.setLastAccessedTime(
projName,
Timestamp.valueOf(LocalDateTime.now())
);
}
}
}

54
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStore.java Normal file
View File

@@ -0,0 +1,54 @@
package uk.ac.ic.wlgitbridge.bridge.swap.store;
import org.apache.commons.io.IOUtils;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
/**
* Created by winston on 23/08/2016.
*/
public class InMemorySwapStore implements SwapStore {
private final Map<String, byte[]> store;
public InMemorySwapStore() {
store = new HashMap<>();
}
public InMemorySwapStore(SwapStoreConfig __) {
this();
}
@Override
public void upload(
String projectName,
InputStream uploadStream,
long contentLength
) throws IOException {
store.put(
projectName,
IOUtils.toByteArray(uploadStream, contentLength)
);
}
@Override
public InputStream openDownloadStream(String projectName) {
byte[] buf = store.get(projectName);
if (buf == null) {
throw new IllegalArgumentException(
"no such project in swap store: " + projectName
);
}
return new ByteArrayInputStream(buf);
}
@Override
public void remove(String projectName) {
store.remove(projectName);
}
}

35
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/NoopSwapStore.java Normal file
View File

@@ -0,0 +1,35 @@
package uk.ac.ic.wlgitbridge.bridge.swap.store;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
/**
* Created by winston on 24/08/2016.
*/
public class NoopSwapStore implements SwapStore {
public NoopSwapStore(SwapStoreConfig config) {
}
@Override
public void upload(
String projectName,
InputStream uploadStream,
long contentLength
) throws IOException {
}
@Override
public InputStream openDownloadStream(String projectName) {
return new ByteArrayInputStream(new byte[0]);
}
@Override
public void remove(String projectName) {
}
}

72
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStore.java Normal file
View File

@@ -0,0 +1,72 @@
package uk.ac.ic.wlgitbridge.bridge.swap.store;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.*;
import java.io.InputStream;
/**
* Created by winston on 21/08/2016.
*/
public class S3SwapStore implements SwapStore {
private final AmazonS3 s3;
private final String bucketName;
public S3SwapStore(SwapStoreConfig cfg) {
this(
cfg.getAwsAccessKey(),
cfg.getAwsSecret(),
cfg.getS3BucketName()
);
}
S3SwapStore(
String accessKey,
String secret,
String bucketName
) {
s3 = new AmazonS3Client(new BasicAWSCredentials(accessKey, secret));
this.bucketName = bucketName;
}
@Override
public void upload(
String projectName,
InputStream uploadStream,
long contentLength
) {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(contentLength);
PutObjectRequest put = new PutObjectRequest(
bucketName,
projectName,
uploadStream,
metadata
);
PutObjectResult res = s3.putObject(put);
}
@Override
public InputStream openDownloadStream(String projectName) {
GetObjectRequest get = new GetObjectRequest(
bucketName,
projectName
);
S3Object res = s3.getObject(get);
return res.getObjectContent();
}
@Override
public void remove(String projectName) {
DeleteObjectRequest del = new DeleteObjectRequest(
bucketName,
projectName
);
s3.deleteObject(del);
}
}

44
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStore.java Normal file
View File

@@ -0,0 +1,44 @@
package uk.ac.ic.wlgitbridge.bridge.swap.store;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapStore {
Map<String, Function<SwapStoreConfig, SwapStore>> swapStores =
new HashMap<String, Function<SwapStoreConfig, SwapStore>>() {
{
put("noop", NoopSwapStore::new);
put("memory", InMemorySwapStore::new);
put("s3", S3SwapStore::new);
}
};
static SwapStore fromConfig(
Optional<SwapStoreConfig> cfg
) {
SwapStoreConfig cfg_ = cfg.orElse(SwapStoreConfig.NOOP);
String type = cfg_.getType();
return swapStores.get(type).apply(cfg_);
}
void upload(
String projectName,
InputStream uploadStream,
long contentLength
) throws IOException;
InputStream openDownloadStream(String projectName);
void remove(String projectName);
}

76
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStoreConfig.java Normal file
View File

@@ -0,0 +1,76 @@
package uk.ac.ic.wlgitbridge.bridge.swap.store;
/**
* Created by winston on 24/08/2016.
*/
public class SwapStoreConfig {
public static final SwapStoreConfig NOOP = new SwapStoreConfig(
"noop",
null,
null,
null
);
private String type;
private String awsAccessKey;
private String awsSecret;
private String s3BucketName;
public SwapStoreConfig() {}
public SwapStoreConfig(
String awsAccessKey,
String awsSecret,
String s3BucketName
) {
this(
"s3",
awsAccessKey,
awsSecret,
s3BucketName
);
}
SwapStoreConfig(
String type,
String awsAccessKey,
String awsSecret,
String s3BucketName
) {
this.type = type;
this.awsAccessKey = awsAccessKey;
this.awsSecret = awsSecret;
this.s3BucketName = s3BucketName;
}
public String getType() {
return type;
}
public String getAwsAccessKey() {
return awsAccessKey;
}
public String getAwsSecret() {
return awsSecret;
}
public String getS3BucketName() {
return s3BucketName;
}
public SwapStoreConfig sanitisedCopy() {
return new SwapStoreConfig(
type,
awsAccessKey == null ? null : "<awsAccessKey>",
awsSecret == null ? null : "<awsSecret>",
s3BucketName
);
}
public static SwapStoreConfig sanitisedCopy(SwapStoreConfig swapStore) {
return swapStore == null ? null : swapStore.sanitisedCopy();
}
}

45
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/CandidateSnapshot.java
View File

@@ -17,7 +17,7 @@ import java.util.Map.Entry;
/**
* Created by Winston on 16/11/14.
*/
public class CandidateSnapshot {
public class CandidateSnapshot implements AutoCloseable {
private final String projectName;
private final int currentVersion;
@@ -25,14 +25,22 @@ public class CandidateSnapshot {
private final List<String> deleted;
private File attsDirectory;
public CandidateSnapshot(String projectName, int currentVersion, RawDirectory directoryContents, RawDirectory oldDirectoryContents) {
public CandidateSnapshot(
String projectName,
int currentVersion,
RawDirectory directoryContents,
RawDirectory oldDirectoryContents
) {
this.projectName = projectName;
this.currentVersion = currentVersion;
files = diff(directoryContents, oldDirectoryContents);
deleted = deleted(directoryContents, oldDirectoryContents);
}
private List<ServletFile> diff(RawDirectory directoryContents, RawDirectory oldDirectoryContents) {
private List<ServletFile> diff(
RawDirectory directoryContents,
RawDirectory oldDirectoryContents
) {
List<ServletFile> files = new LinkedList<ServletFile>();
Map<String, RawFile> fileTable = directoryContents.getFileTable();
Map<String, RawFile> oldFileTable = oldDirectoryContents.getFileTable();
@@ -43,10 +51,16 @@ public class CandidateSnapshot {
return files;
}
private List<String> deleted(RawDirectory directoryContents, RawDirectory oldDirectoryContents) {
private List<String> deleted(
RawDirectory directoryContents,
RawDirectory oldDirectoryContents
) {
List<String> deleted = new LinkedList<String>();
Map<String, RawFile> fileTable = directoryContents.getFileTable();
for (Entry<String, RawFile> entry : oldDirectoryContents.getFileTable().entrySet()) {
for (
Entry<String, RawFile> entry :
oldDirectoryContents.getFileTable().entrySet()
) {
String path = entry.getKey();
RawFile newFile = fileTable.get(path);
if (newFile == null) {
@@ -57,7 +71,10 @@ public class CandidateSnapshot {
}
public void writeServletFiles(File rootGitDirectory) throws IOException {
attsDirectory = new File(rootGitDirectory, ".wlgb/atts/" + projectName);
attsDirectory = new File(
rootGitDirectory,
".wlgb/atts/" + projectName
);
for (ServletFile file : files) {
if (file.isChanged()) {
file.writeToDisk(attsDirectory);
@@ -90,11 +107,18 @@ public class CandidateSnapshot {
return filesArray;
}
private JsonObject getFileAsJson(ServletFile file, String projectURL, String postbackKey) {
private JsonObject getFileAsJson(
ServletFile file,
String projectURL,
String postbackKey
) {
JsonObject jsonFile = new JsonObject();
jsonFile.addProperty("name", file.getPath());
if (file.isChanged()) {
jsonFile.addProperty("url", projectURL + "/" + file.getPath() + "?key=" + postbackKey);
jsonFile.addProperty(
"url",
projectURL + "/" + file.getPath() + "?key=" + postbackKey
);
}
return jsonFile;
}
@@ -119,4 +143,9 @@ public class CandidateSnapshot {
return sb.toString();
}
@Override
public void close() throws IOException {
deleteServletFiles();
}
}

2
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ProjectLockImpl.java
View File

@@ -33,11 +33,13 @@ public class ProjectLockImpl implements ProjectLock {
setWaiter(waiter);
}
@Override
public void lockForProject(String projectName) {
getLockForProjectName(projectName).lock();
rlock.lock();
}
@Override
public void unlockForProject(String projectName) {
getLockForProjectName(projectName).unlock();
rlock.unlock();

53
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/SnapshotRepositoryBuilder.java
View File

@@ -1,53 +0,0 @@
package uk.ac.ic.wlgitbridge.data;
import com.google.api.client.auth.oauth2.Credential;
import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
import org.eclipse.jgit.transport.ServiceMayNotContinueException;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.bridge.WLBridgedProject;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.InternalErrorException;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.File;
import java.io.IOException;
/**
* Created by Winston on 03/11/14.
*/
public class SnapshotRepositoryBuilder {
private final Bridge bridgeAPI;
public SnapshotRepositoryBuilder(Bridge bridgeAPI) {
this.bridgeAPI = bridgeAPI;
}
public Repository getRepositoryWithNameAtRootDirectory(String name, File rootDirectory, Credential oauth2) throws RepositoryNotFoundException, ServiceMayNotContinueException, GitUserException {
if (!bridgeAPI.repositoryExists(oauth2, name)) {
throw new RepositoryNotFoundException(name);
}
File repositoryDirectory = new File(rootDirectory, name);
Repository repository = null;
try {
repository = new FileRepositoryBuilder().setWorkTree(repositoryDirectory).build();
new WLBridgedProject(repository, name, bridgeAPI).buildRepository(oauth2);
} catch (IOException e) {
Log.warn(
"IOException when trying to get repo: " +
name +
", at: "
+ rootDirectory.getAbsolutePath(),
e
);
throw new ServiceMayNotContinueException(
new InternalErrorException().getDescriptionLines().get(0)
);
}
return repository;
}
}

20
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/PersistentStore.java
View File

@@ -1,20 +0,0 @@
package uk.ac.ic.wlgitbridge.data.model.db;
import java.util.List;
/**
* Created by m on 20/11/15.
*/
public interface PersistentStore {
List<String> getProjectNames();
void setLatestVersionForProject(String project, int versionID);
int getLatestVersionForProject(String project);
void addURLIndexForProject(String projectName, String url, String path);
void deleteFilesForProject(String project, String... files);
String getPathForURLInProject(String projectName, String url);
}

14
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/SQLUpdate.java
View File

@@ -1,14 +0,0 @@
package uk.ac.ic.wlgitbridge.data.model.db.sql;
import java.sql.PreparedStatement;
import java.sql.SQLException;
/**
* Created by Winston on 20/11/14.
*/
public interface SQLUpdate {
public String getSQL();
public void addParametersToStatement(PreparedStatement statement) throws SQLException;
}

107
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/SQLiteWLDatabase.java
View File

@@ -1,107 +0,0 @@
package uk.ac.ic.wlgitbridge.data.model.db.sql;
import uk.ac.ic.wlgitbridge.data.model.db.sql.query.GetLatestVersionForProjectSQLQuery;
import uk.ac.ic.wlgitbridge.data.model.db.sql.query.GetPathForURLInProjectSQLQuery;
import uk.ac.ic.wlgitbridge.data.model.db.sql.query.GetProjectNamesSQLQuery;
import uk.ac.ic.wlgitbridge.data.model.db.sql.update.create.CreateIndexURLIndexStore;
import uk.ac.ic.wlgitbridge.data.model.db.sql.update.create.CreateProjectsTableSQLUpdate;
import uk.ac.ic.wlgitbridge.data.model.db.sql.update.create.CreateURLIndexStoreSQLUpdate;
import uk.ac.ic.wlgitbridge.data.model.db.sql.update.delete.DeleteFilesForProjectSQLUpdate;
import uk.ac.ic.wlgitbridge.data.model.db.sql.update.insert.AddURLIndexSQLUpdate;
import uk.ac.ic.wlgitbridge.data.model.db.sql.update.insert.SetProjectSQLUpdate;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.File;
import java.sql.*;
import java.util.List;
/**
* Created by Winston on 17/11/14.
*/
public class SQLiteWLDatabase {
private final Connection connection;
public SQLiteWLDatabase(File rootGitDirectory) throws SQLException, ClassNotFoundException {
File databaseFile = new File(rootGitDirectory, "/.wlgb/wlgb.db");
File dotWlgbDir = databaseFile.getParentFile();
if (!dotWlgbDir.exists()) {
if (!dotWlgbDir.mkdirs()) {
Log.error("{} directory didn't exist, and unable to create. Check your permissions", dotWlgbDir.getAbsolutePath());
}
}
Class.forName("org.sqlite.JDBC");
connection = DriverManager.getConnection("jdbc:sqlite:" + databaseFile.getAbsolutePath());
createTables();
}
public void setVersionIDForProject(String projectName, int versionID) throws SQLException {
update(new SetProjectSQLUpdate(projectName, versionID));
}
public void addURLIndex(String projectName, String url, String path) throws SQLException {
update(new AddURLIndexSQLUpdate(projectName, url, path));
}
public void deleteFilesForProject(String projectName, String... paths) throws SQLException {
update(new DeleteFilesForProjectSQLUpdate(projectName, paths));
}
public int getVersionIDForProjectName(String projectName) throws SQLException {
return query(new GetLatestVersionForProjectSQLQuery(projectName));
}
public String getPathForURLInProject(String projectName, String url) throws SQLException {
return query(new GetPathForURLInProjectSQLQuery(projectName, url));
}
public List<String> getProjectNames() throws SQLException {
return query(new GetProjectNamesSQLQuery());
}
private void createTables() throws SQLException {
final SQLUpdate[] createTableUpdates = {
new CreateProjectsTableSQLUpdate(),
new CreateURLIndexStoreSQLUpdate(),
new CreateIndexURLIndexStore()
};
for (SQLUpdate update : createTableUpdates) {
update(update);
}
}
private void update(SQLUpdate update) throws SQLException {
PreparedStatement statement = null;
try {
statement = connection.prepareStatement(update.getSQL());
update.addParametersToStatement(statement);
statement.executeUpdate();
} catch (SQLException e) {
throw e;
} finally {
statement.close();
}
}
private <T> T query(SQLQuery<T> query) throws SQLException {
PreparedStatement statement = null;
ResultSet results = null;
try {
statement = connection.prepareStatement(query.getSQL());
query.addParametersToStatement(statement);
results = statement.executeQuery();
return query.processResultSet(results);
} catch (SQLException e) {
throw e;
} finally {
if (statement != null) {
statement.close();
}
if (results != null) {
results.close();
}
}
}
}

26
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/create/CreateIndexURLIndexStore.java
View File

@@ -1,26 +0,0 @@
package uk.ac.ic.wlgitbridge.data.model.db.sql.update.create;
import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
/**
* Created by Winston on 21/02/15.
*/
public class CreateIndexURLIndexStore implements SQLUpdate {
public static final String CREATE_INDEX_URL_INDEX_STORE =
"CREATE UNIQUE INDEX IF NOT EXISTS `project_path_index` ON `url_index_store`(`project_name`, `path`);\n";
@Override
public String getSQL() {
return CREATE_INDEX_URL_INDEX_STORE;
}
@Override
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
}
}

29
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/create/CreateProjectsTableSQLUpdate.java
View File

@@ -1,29 +0,0 @@
package uk.ac.ic.wlgitbridge.data.model.db.sql.update.create;
import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
/**
* Created by Winston on 20/11/14.
*/
public class CreateProjectsTableSQLUpdate implements SQLUpdate {
private static final String CREATE_PROJECTS_TABLE =
"CREATE TABLE IF NOT EXISTS `projects` (\n" +
" `name` varchar(10) NOT NULL DEFAULT '',\n" +
" `version_id` int(11) NOT NULL DEFAULT 0,\n" +
" PRIMARY KEY (`name`)\n" +
")";
@Override
public String getSQL() {
return CREATE_PROJECTS_TABLE;
}
@Override
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
}
}

8
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java
View File

@@ -19,10 +19,10 @@ import javax.servlet.http.HttpServletRequest;
/* */
public class WLReceivePackFactory implements ReceivePackFactory<HttpServletRequest> {
private final Bridge bridgeAPI;
private final Bridge bridge;
public WLReceivePackFactory(Bridge bridgeAPI) {
this.bridgeAPI = bridgeAPI;
public WLReceivePackFactory(Bridge bridge) {
this.bridge = bridge;
}
@Override
@@ -33,7 +33,7 @@ public class WLReceivePackFactory implements ReceivePackFactory<HttpServletReque
if (hostname == null) {
hostname = httpServletRequest.getLocalName();
}
receivePack.setPreReceiveHook(new WriteLatexPutHook(bridgeAPI, hostname, oauth2));
receivePack.setPreReceiveHook(new WriteLatexPutHook(bridge, hostname, oauth2));
return receivePack;
}

61
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLRepositoryResolver.java
View File

@@ -7,35 +7,48 @@ import org.eclipse.jgit.transport.ServiceMayNotContinueException;
import org.eclipse.jgit.transport.resolver.RepositoryResolver;
import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException;
import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException;
import uk.ac.ic.wlgitbridge.data.SnapshotRepositoryBuilder;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.bridge.GitProjectRepo;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.exception.InvalidRootDirectoryPathException;
import uk.ac.ic.wlgitbridge.server.Oauth2Filter;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.Util;
import javax.servlet.http.HttpServletRequest;
import java.io.File;
import java.io.IOException;
/**
* Created by Winston on 02/11/14.
*/
public class WLRepositoryResolver implements RepositoryResolver<HttpServletRequest> {
public class WLRepositoryResolver
implements RepositoryResolver<HttpServletRequest> {
private File rootGitDirectory;
private SnapshotRepositoryBuilder snapshotRepositoryBuilder;
private final Bridge bridge;
public WLRepositoryResolver(String rootGitDirectoryPath, SnapshotRepositoryBuilder repositorySource) throws InvalidRootDirectoryPathException {
this.snapshotRepositoryBuilder = repositorySource;
initRootGitDirectory(rootGitDirectoryPath);
public WLRepositoryResolver(Bridge bridge) {
this.bridge = bridge;
}
@Override
public Repository open(HttpServletRequest httpServletRequest, String name) throws RepositoryNotFoundException, ServiceNotAuthorizedException, ServiceNotEnabledException, ServiceMayNotContinueException {
Credential oauth2 = (Credential) httpServletRequest.getAttribute(Oauth2Filter.ATTRIBUTE_KEY);
public Repository open(
HttpServletRequest httpServletRequest,
String name
) throws RepositoryNotFoundException,
ServiceNotAuthorizedException,
ServiceNotEnabledException,
ServiceMayNotContinueException {
Credential oauth2 = (Credential) httpServletRequest.getAttribute(
Oauth2Filter.ATTRIBUTE_KEY
);
String projName = Util.removeAllSuffixes(name, "/", ".git");
try {
return snapshotRepositoryBuilder.getRepositoryWithNameAtRootDirectory(Util.removeAllSuffixes(name, "/", ".git"), rootGitDirectory, oauth2);
if (!bridge.projectExists(oauth2, projName)) {
throw new RepositoryNotFoundException(projName);
}
GitProjectRepo repo = new GitProjectRepo(projName);
bridge.updateRepository(oauth2, repo);
return repo.getJGitRepository();
} catch (RepositoryNotFoundException e) {
Log.info("Repository not found: " + name);
throw e;
@@ -45,25 +58,25 @@ public class WLRepositoryResolver implements RepositoryResolver<HttpServletReque
} catch (ServiceNotEnabledException e) {
cannot occur
*/
} catch (ServiceMayNotContinueException e) { /* Such as FailedConnectionException */
} catch (ServiceMayNotContinueException e) {
/* Such as FailedConnectionException */
throw e;
} catch (RuntimeException e) {
Log.warn("Runtime exception when trying to open repo", e);
Log.warn(
"Runtime exception when trying to open repo: " + projName,
e
);
throw new ServiceMayNotContinueException(e);
} catch (ForbiddenException e) {
throw new ServiceNotAuthorizedException();
} catch (GitUserException e) {
throw new ServiceMayNotContinueException(e.getMessage(), e);
}
}
private void initRootGitDirectory(String rootGitDirectoryPath) throws InvalidRootDirectoryPathException {
rootGitDirectory = new File(rootGitDirectoryPath);
/* throws SecurityException */
rootGitDirectory.mkdirs();
rootGitDirectory.getAbsolutePath();
if (!rootGitDirectory.isDirectory()) {
throw new InvalidRootDirectoryPathException();
} catch (IOException e) {
Log.warn(
"IOException when trying to open repo: " + projName,
e
);
throw new ServiceMayNotContinueException("Internal server error.");
}
}

8
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/WriteLatexPutHook.java
View File

@@ -26,12 +26,12 @@ import java.util.Iterator;
*/
public class WriteLatexPutHook implements PreReceiveHook {
private final Bridge bridgeAPI;
private final Bridge bridge;
private final String hostname;
private final Credential oauth2;
public WriteLatexPutHook(Bridge bridgeAPI, String hostname, Credential oauth2) {
this.bridgeAPI = bridgeAPI;
public WriteLatexPutHook(Bridge bridge, String hostname, Credential oauth2) {
this.bridge = bridge;
this.hostname = hostname;
this.oauth2 = oauth2;
}
@@ -75,7 +75,7 @@ public class WriteLatexPutHook implements PreReceiveHook {
private void handleReceiveCommand(Credential oauth2, Repository repository, ReceiveCommand receiveCommand) throws IOException, GitUserException {
checkBranch(receiveCommand);
checkForcedPush(receiveCommand);
bridgeAPI.putDirectoryContentsToProjectWithName(
bridge.putDirectoryContentsToProjectWithName(
oauth2,
repository.getWorkTree().getName(),
getPushedDirectoryContents(repository,

14
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/servlet/WLGitServlet.java
View File

@@ -2,12 +2,11 @@ package uk.ac.ic.wlgitbridge.git.servlet;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jgit.http.server.GitServlet;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.git.exception.InvalidRootDirectoryPathException;
import uk.ac.ic.wlgitbridge.git.handler.WLReceivePackFactory;
import uk.ac.ic.wlgitbridge.git.handler.WLRepositoryResolver;
import uk.ac.ic.wlgitbridge.git.handler.WLUploadPackFactory;
import uk.ac.ic.wlgitbridge.data.SnapshotRepositoryBuilder;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import javax.servlet.ServletException;
@@ -16,11 +15,14 @@ import javax.servlet.ServletException;
*/
public class WLGitServlet extends GitServlet {
public WLGitServlet(ServletContextHandler servletContextHandler, Bridge bridgeAPI, String rootGitDirectoryPath) throws ServletException, InvalidRootDirectoryPathException {
setRepositoryResolver(new WLRepositoryResolver(rootGitDirectoryPath, new SnapshotRepositoryBuilder(bridgeAPI)));
setReceivePackFactory(new WLReceivePackFactory(bridgeAPI));
public WLGitServlet(
ServletContextHandler ctxHandler,
Bridge bridge
) throws ServletException, InvalidRootDirectoryPathException {
setRepositoryResolver(new WLRepositoryResolver(bridge));
setReceivePackFactory(new WLReceivePackFactory(bridge));
setUploadPackFactory(new WLUploadPackFactory());
init(new WLGitServletConfig(servletContextHandler));
init(new WLGitServletConfig(ctxHandler));
}
}

4
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/servlet/WLGitServletConfig.java
View File

@@ -15,8 +15,8 @@ public class WLGitServletConfig implements ServletConfig {
private ServletContext servletContext;
public WLGitServletConfig(ServletContextHandler servletContextHandler) {
servletContext = servletContextHandler.getServletContext();
public WLGitServletConfig(ServletContextHandler ctxHandler) {
servletContext = ctxHandler.getServletContext();
}
@Override

66
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java
View File

@@ -10,10 +10,10 @@ import uk.ac.ic.wlgitbridge.application.config.Config;
import uk.ac.ic.wlgitbridge.application.jetty.NullLogger;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.SqliteDBStore;
import uk.ac.ic.wlgitbridge.bridge.repo.FSRepoStore;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore;
import uk.ac.ic.wlgitbridge.bridge.repo.FSGitRepoStore;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import uk.ac.ic.wlgitbridge.git.exception.InvalidRootDirectoryPathException;
import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet;
import uk.ac.ic.wlgitbridge.snapshot.base.SnapshotAPIRequest;
@@ -25,6 +25,7 @@ import javax.servlet.Filter;
import javax.servlet.ServletException;
import java.io.File;
import java.net.BindException;
import java.nio.file.Paths;
import java.util.EnumSet;
/**
@@ -36,7 +37,7 @@ import java.util.EnumSet;
*/
public class GitBridgeServer {
private final Bridge bridgeAPI;
private final Bridge bridge;
private final Server jettyServer;
@@ -44,21 +45,31 @@ public class GitBridgeServer {
private String rootGitDirectoryPath;
private String apiBaseURL;
public GitBridgeServer(Config config) throws ServletException, InvalidRootDirectoryPathException {
public GitBridgeServer(
Config config
) throws ServletException, InvalidRootDirectoryPathException {
org.eclipse.jetty.util.log.Log.setLog(new NullLogger());
this.port = config.getPort();
this.rootGitDirectoryPath = config.getRootGitDirectory();
RepoStore repoStore = new FSRepoStore(rootGitDirectoryPath);
DBStore dbStore = new SqliteDBStore(repoStore.getRootDirectory());
SwapStore swapStore = new SwapStore() {};
bridgeAPI = Bridge.make(
RepoStore repoStore = new FSGitRepoStore(rootGitDirectoryPath);
DBStore dbStore = new SqliteDBStore(
Paths.get(
repoStore.getRootDirectory().getAbsolutePath()
).resolve(".wlgb").resolve("wlgb.db").toFile()
);
SwapStore swapStore = SwapStore.fromConfig(config.getSwapStore());
bridge = Bridge.make(
repoStore,
dbStore,
swapStore
swapStore,
config.getSwapJob()
);
jettyServer = new Server(port);
configureJettyServer(config);
SnapshotAPIRequest.setBasicAuth(config.getUsername(), config.getPassword());
SnapshotAPIRequest.setBasicAuth(
config.getUsername(),
config.getPassword()
);
apiBaseURL = config.getAPIBaseURL();
SnapshotAPIRequest.setBaseURL(apiBaseURL);
Util.setServiceName(config.getServiceName());
@@ -71,7 +82,9 @@ public class GitBridgeServer {
*/
public void start() {
try {
bridge.checkDB();
jettyServer.start();
bridge.startSwapJob();
Log.info(Util.getServiceName() + "-Git Bridge server started");
Log.info("Listening on port: " + port);
Log.info("Bridged to: " + apiBaseURL);
@@ -92,7 +105,9 @@ public class GitBridgeServer {
}
}
private void configureJettyServer(Config config) throws ServletException, InvalidRootDirectoryPathException {
private void configureJettyServer(
Config config
) throws ServletException, InvalidRootDirectoryPathException {
HandlerCollection handlers = new HandlerList();
handlers.addHandler(initApiHandler());
handlers.addHandler(initGitHandler(config));
@@ -105,31 +120,44 @@ public class GitBridgeServer {
HandlerCollection handlers = new HandlerList();
handlers.addHandler(initResourceHandler());
handlers.addHandler(new PostbackHandler(bridgeAPI));
handlers.addHandler(new PostbackHandler(bridge));
handlers.addHandler(new DefaultHandler());
api.setHandler(handlers);
return api;
}
private Handler initGitHandler(Config config) throws ServletException, InvalidRootDirectoryPathException {
final ServletContextHandler servletContextHandler = new ServletContextHandler(ServletContextHandler.SESSIONS);
private Handler initGitHandler(
Config config
) throws ServletException, InvalidRootDirectoryPathException {
final ServletContextHandler servletContextHandler =
new ServletContextHandler(ServletContextHandler.SESSIONS);
if (config.isUsingOauth2()) {
Filter filter = new Oauth2Filter(config.getOauth2());
servletContextHandler.addFilter(new FilterHolder(filter), "/*", EnumSet.of(DispatcherType.REQUEST));
servletContextHandler.addFilter(
new FilterHolder(filter),
"/*",
EnumSet.of(DispatcherType.REQUEST)
);
}
servletContextHandler.setContextPath("/");
servletContextHandler.addServlet(
new ServletHolder(
new WLGitServlet(servletContextHandler, bridgeAPI, rootGitDirectoryPath)),
new WLGitServlet(
servletContextHandler,
bridge
)
),
"/*"
);
return servletContextHandler;
}
private Handler initResourceHandler() {
ResourceHandler resourceHandler = new FileHandler(bridgeAPI);
resourceHandler.setResourceBase(new File(rootGitDirectoryPath, ".wlgb/atts").getAbsolutePath());
ResourceHandler resourceHandler = new FileHandler(bridge);
resourceHandler.setResourceBase(
new File(rootGitDirectoryPath, ".wlgb/atts").getAbsolutePath()
);
return resourceHandler;
}
}

10
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/PostbackContents.java
View File

@@ -17,7 +17,7 @@ public class PostbackContents implements JSONSource {
private static final String CODE_SUCCESS = "upToDate";
private final Bridge bridgeAPI;
private final Bridge bridge;
private final String projectName;
private final String postbackKey;
@@ -26,8 +26,8 @@ public class PostbackContents implements JSONSource {
private int versionID;
private SnapshotPostException exception;
public PostbackContents(Bridge bridgeAPI, String projectName, String postbackKey, String contents) {
this.bridgeAPI = bridgeAPI;
public PostbackContents(Bridge bridge, String projectName, String postbackKey, String contents) {
this.bridge = bridge;
this.projectName = projectName;
this.postbackKey = postbackKey;
snapshotPostExceptionBuilder = new SnapshotPostExceptionBuilder();
@@ -43,9 +43,9 @@ public class PostbackContents implements JSONSource {
public void processPostback() throws UnexpectedPostbackException {
if (exception == null) {
bridgeAPI.postbackReceivedSuccessfully(projectName, postbackKey, versionID);
bridge.postbackReceivedSuccessfully(projectName, postbackKey, versionID);
} else {
bridgeAPI.postbackReceivedWithException(projectName, postbackKey, exception);
bridge.postbackReceivedWithException(projectName, postbackKey, exception);
}
}

8
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/PostbackHandler.java
View File

@@ -19,10 +19,10 @@ import java.io.IOException;
*/
public class PostbackHandler extends AbstractHandler {
private final Bridge bridgeAPI;
private final Bridge bridge;
public PostbackHandler(Bridge bridgeAPI) {
this.bridgeAPI = bridgeAPI;
public PostbackHandler(Bridge bridge) {
this.bridge = bridge;
}
@Override
@@ -39,7 +39,7 @@ public class PostbackHandler extends AbstractHandler {
String projectName = parts[1];
String postbackKey = parts[2];
Log.info(baseRequest.getMethod() + " <- " + baseRequest.getHttpURI());
PostbackContents postbackContents = new PostbackContents(bridgeAPI, projectName, postbackKey, contents);
PostbackContents postbackContents = new PostbackContents(bridge, projectName, postbackKey, contents);
JsonObject body = new JsonObject();
try {

127
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Files.java Normal file
View File

@@ -0,0 +1,127 @@
package uk.ac.ic.wlgitbridge.util;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.TrueFileFilter;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Created by winston on 23/08/2016.
*/
public class Files {
private Files() {}
public static boolean contentsAreEqual(
File f0,
File f1
) throws IOException {
try {
return uncheckedContentsAreEqual(f0, f1);
} catch (UncheckedIOException e) {
throw e.getCause();
}
}
public static void renameAll(
File fileOrDir,
String from,
String to
) {
if (fileOrDir.isDirectory()) {
File f = doRename(fileOrDir, from, to);
for (File c : f.listFiles()) {
renameAll(c, from, to);
}
} else if (fileOrDir.isFile()) {
doRename(fileOrDir, from, to);
} else {
throw new IllegalArgumentException(
"not a file or dir: " + fileOrDir
);
}
}
private static File doRename(File fileOrDir, String from, String to) {
if (!fileOrDir.getName().equals(from)) {
return fileOrDir;
}
File renamed = new File(fileOrDir.getParent(), to);
Preconditions.checkState(fileOrDir.renameTo(renamed));
return renamed;
}
private static boolean uncheckedContentsAreEqual(File f0, File f1) throws IOException {
if (f0.equals(f1)) {
return true;
}
if (!f0.isDirectory() || !f1.isDirectory()) {
return !f0.isDirectory() && !f1.isDirectory() &&
Arrays.equals(
FileUtils.readFileToByteArray(f0),
FileUtils.readFileToByteArray(f1)
);
}
Path f0Base = Paths.get(f0.getAbsolutePath());
Path f1Base = Paths.get(f1.getAbsolutePath());
Set<Path> children0 = getChildren(f0, f0Base);
Set<Path> children1 = getChildren(f1, f1Base);
if (children0.size() != children1.size()) {
return false;
}
return children0.stream(
).allMatch(c0 ->
children1.contains(c0) && childEquals(c0, f0Base, f1Base)
);
}
private static Set<Path> getChildren(File f0, Path f0Base) {
return FileUtils.listFilesAndDirs(
f0,
TrueFileFilter.TRUE,
TrueFileFilter.TRUE
).stream(
).map(
File::getAbsolutePath
).map(
Paths::get
).map(p ->
f0Base.relativize(p)
).filter(p ->
!p.toString().isEmpty()
).collect(
Collectors.toSet()
);
}
private static boolean childEquals(
Path child,
Path f0Base,
Path f1Base
) throws UncheckedIOException {
File c0 = f0Base.resolve(child).toFile();
File c1 = f1Base.resolve(child).toFile();
boolean c0IsDir = c0.isDirectory();
boolean c1IsDir = c1.isDirectory();
if (c0IsDir || c1IsDir) {
return c0IsDir && c1IsDir;
}
try {
return c0.isFile() && c1.isFile() && Arrays.equals(
FileUtils.readFileToByteArray(c0),
FileUtils.readFileToByteArray(c1)
);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}

13
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Project.java Normal file
View File

@@ -0,0 +1,13 @@
package uk.ac.ic.wlgitbridge.util;
/**
* Created by winston on 23/08/2016.
*/
public class Project {
public static boolean isValidProjectName(String projectName) {
return projectName != null && !projectName.isEmpty()
&& !projectName.startsWith(".");
}
}

151
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java Normal file
View File

@@ -0,0 +1,151 @@
package uk.ac.ic.wlgitbridge.util;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
import org.apache.commons.compress.utils.IOUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.output.ByteArrayOutputStream;
import java.io.*;
import java.nio.file.Path;
import java.nio.file.Paths;
/**
* Created by winston on 23/08/2016.
*/
public class Tar {
public static class bz2 {
public static InputStream zip(
File fileOrDir
) throws IOException {
return zip(fileOrDir, null);
}
public static InputStream zip(
File fileOrDir,
long[] sizePtr
) throws IOException {
ByteArrayOutputStream target = new ByteArrayOutputStream();
try (OutputStream bzip2 = new BZip2CompressorOutputStream(target)) {
tarTo(fileOrDir, bzip2);
}
if (sizePtr != null) {
sizePtr[0] = target.size();
}
return target.toInputStream();
}
public static void unzip(
InputStream tarbz2,
File parentDir
) throws IOException {
try (InputStream tar = new BZip2CompressorInputStream(tarbz2)) {
untar(tar, parentDir);
}
}
}
private Tar() {}
public static InputStream tar(File fileOrDir) throws IOException {
ByteArrayOutputStream target = new ByteArrayOutputStream();
tarTo(fileOrDir, target);
return target.toInputStream();
}
public static void tarTo(
File fileOrDir,
OutputStream target
) throws IOException {
try (TarArchiveOutputStream tout = new TarArchiveOutputStream(target)) {
addTarEntry(
tout,
Paths.get(fileOrDir.getParentFile().getAbsolutePath()),
fileOrDir
);
}
}
public static void untar(
InputStream tar,
File parentDir
) throws IOException {
TarArchiveInputStream tin = new TarArchiveInputStream(tar);
ArchiveEntry e;
while ((e = tin.getNextEntry()) != null) {
File f = new File(parentDir, e.getName());
f.setLastModified(e.getLastModifiedDate().getTime());
f.getParentFile().mkdirs();
if (e.isDirectory()) {
f.mkdir();
continue;
}
long size = e.getSize();
Preconditions.checkArgument(
size > 0 && size < Integer.MAX_VALUE,
"file too big: tarTo should have thrown an IOException"
);
try (OutputStream out = new FileOutputStream(f)) {
/* TarInputStream pretends each
entry's EOF is the stream's EOF */
IOUtils.copy(tin, out);
}
}
}
private static void addTarEntry(
TarArchiveOutputStream tout,
Path base,
File fileOrDir
) throws IOException {
if (fileOrDir.isDirectory()) {
addTarDir(tout, base, fileOrDir);
} else if (fileOrDir.isFile()) {
addTarFile(tout, base, fileOrDir);
} else {
throw new IllegalArgumentException(
"invalid file or dir: " + fileOrDir
);
}
}
private static void addTarDir(
TarArchiveOutputStream tout,
Path base,
File dir
) throws IOException {
Preconditions.checkArgument(dir.isDirectory());
String name = base.relativize(
Paths.get(dir.getAbsolutePath())
).toString();
ArchiveEntry entry = tout.createArchiveEntry(dir, name);
tout.putArchiveEntry(entry);
tout.closeArchiveEntry();
for (File f : dir.listFiles()) {
addTarEntry(tout, base, f);
}
}
private static void addTarFile(
TarArchiveOutputStream tout,
Path base,
File file
) throws IOException {
Preconditions.checkArgument(file.isFile());
String name = base.relativize(
Paths.get(file.getAbsolutePath())
).toString();
ArchiveEntry entry = tout.createArchiveEntry(file, name);
tout.putArchiveEntry(entry);
tout.write(FileUtils.readFileToByteArray(file));
tout.closeArchiveEntry();
}
}

19
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Timer.java Normal file
View File

@@ -0,0 +1,19 @@
package uk.ac.ic.wlgitbridge.util;
import java.util.TimerTask;
/**
* Created by winston on 23/08/2016.
*/
public class Timer {
public static TimerTask makeTimerTask(Runnable lamb) {
return new TimerTask() {
@Override
public void run() {
lamb.run();
}
};
}
}

9
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Util.java
View File

@@ -19,15 +19,6 @@ public class Util {
private static String POSTBACK_URL;
private static final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSS");
public static TimerTask makeTimerTask(Runnable lamb) {
return new TimerTask() {
@Override
public void run() {
lamb.run();
}
};
}
public static String entries(int entries) {
if (entries == 1) {
return "entry";

132
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest.java → services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java
View File

@@ -1,13 +1,14 @@
package uk.ac.ic.wlgitbridge;
package uk.ac.ic.wlgitbridge.application;
import com.ning.http.client.AsyncHttpClient;
import com.ning.http.client.Response;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import uk.ac.ic.wlgitbridge.application.GitBridgeApp;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig;
import uk.ac.ic.wlgitbridge.snapshot.servermock.server.MockSnapshotServer;
import uk.ac.ic.wlgitbridge.snapshot.servermock.state.SnapshotAPIState;
import uk.ac.ic.wlgitbridge.snapshot.servermock.state.SnapshotAPIStateBuilder;
@@ -15,6 +16,7 @@ import uk.ac.ic.wlgitbridge.snapshot.servermock.util.FileUtil;
import uk.ac.ic.wlgitbridge.util.Util;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
@@ -102,6 +104,9 @@ public class WLGitBridgeIntegrationTest {
put("canServePushedFiles", new HashMap<String, SnapshotAPIState>() {{
put("state", new SnapshotAPIStateBuilder(getResourceAsStream("/canServePushedFiles/state/state.json")).build());
}});
put("wlgbCanSwapProjects", new HashMap<String, SnapshotAPIState>() {{
put("state", new SnapshotAPIStateBuilder(getResourceAsStream("/wlgbCanSwapProjects/state/state.json")).build());
}});
}};
@Rule
@@ -587,14 +592,56 @@ public class WLGitBridgeIntegrationTest {
wlgb.stop();
}
@Test
public void wlgbCanSwapProjects(
) throws IOException, GitAPIException, InterruptedException {
MockSnapshotServer server = new MockSnapshotServer(
3874,
getResource("/wlgbCanSwapProjects").toFile()
);
server.start();
server.setState(states.get("wlgbCanSwapProjects").get("state"));
GitBridgeApp wlgb = new GitBridgeApp(new String[] {
makeConfigFile(33874, 3874, new SwapJobConfig(1, 0, 0, 250))
});
wlgb.run();
File dir = folder.newFolder();
File rootGitDir = new File(wlgb.config.getRootGitDirectory());
File testProj1ServerDir = new File(rootGitDir, "testproj1");
File testProj2ServerDir = new File(rootGitDir, "testproj2");
File testProj1Dir = cloneRepository("testproj1", 33874, dir);
assertTrue(testProj1ServerDir.exists());
assertFalse(testProj2ServerDir.exists());
cloneRepository("testproj2", 33874, dir);
while (testProj1ServerDir.exists());
assertFalse(testProj1ServerDir.exists());
assertTrue(testProj2ServerDir.exists());
FileUtils.deleteDirectory(testProj1Dir);
cloneRepository("testproj1", 33874, dir);
while (testProj2ServerDir.exists());
assertTrue(testProj1ServerDir.exists());
assertFalse(testProj2ServerDir.exists());
wlgb.stop();
}
private File cloneRepository(String repositoryName, int port, File dir) throws IOException, InterruptedException {
String repo = "git clone http://127.0.0.1:" + port + "/" + repositoryName + ".git";
Process gitProcess = runtime.exec(repo, null, dir);
int exitCode = gitProcess.waitFor();
if (exitCode != 0) {
System.err.println("git clone failed. Dumping stderr and stdout.");
System.err.println(IOUtils.toString(gitProcess.getErrorStream()));
System.err.println(IOUtils.toString(gitProcess.getInputStream()));
System.err.println(
IOUtils.toString(
gitProcess.getErrorStream(),
StandardCharsets.UTF_8
)
);
System.err.println(
IOUtils.toString(
gitProcess.getInputStream(),
StandardCharsets.UTF_8
)
);
fail("git clone failed");
}
File repositoryDir = new File(dir, repositoryName);
@@ -606,30 +653,75 @@ public class WLGitBridgeIntegrationTest {
return repositoryDir;
}
private String makeConfigFile(int port, int apiPort) throws IOException {
private String makeConfigFile(
int port,
int apiPort
) throws IOException {
return makeConfigFile(port, apiPort, null);
}
private String makeConfigFile(
int port,
int apiPort,
SwapJobConfig swapCfg
) throws IOException {
File wlgb = folder.newFolder();
File config = folder.newFile();
PrintWriter writer = new PrintWriter(config);
writer.println("{\n" +
"\t\"port\": " + port + ",\n" +
"\t\"rootGitDirectory\": \"" + wlgb.getAbsolutePath() + "\",\n" +
"\t\"apiBaseUrl\": \"http://127.0.0.1:" + apiPort + "/api/v0\",\n" +
"\t\"username\": \"\",\n" +
"\t\"password\": \"\",\n" +
"\t\"postbackBaseUrl\": \"http://127.0.0.1:" + port + "\",\n" +
"\t\"serviceName\": \"Overleaf\"\n," +
" \"oauth2\": {\n" +
" \"oauth2ClientID\": \"clientID\",\n" +
" \"oauth2ClientSecret\": \"oauth2 client secret\",\n" +
" \"oauth2Server\": \"https://www.overleaf.com\"\n" +
" }\n" +
"}\n");
String cfgStr =
"{\n" +
" \"port\": " + port + ",\n" +
" \"rootGitDirectory\": \"" +
wlgb.getAbsolutePath() +
"\",\n" +
" \"apiBaseUrl\": \"http://127.0.0.1:" +
apiPort +
"/api/v0\",\n" +
" \"username\": \"\",\n" +
" \"password\": \"\",\n" +
" \"postbackBaseUrl\": \"http://127.0.0.1:" +
port +
"\",\n" +
" \"serviceName\": \"Overleaf\",\n" +
" \"oauth2\": {\n" +
" \"oauth2ClientID\": \"clientID\",\n" +
" \"oauth2ClientSecret\": \"oauth2 client secret\",\n" +
" \"oauth2Server\": \"https://www.overleaf.com\"\n" +
" }";
if (swapCfg != null) {
cfgStr += ",\n" +
" \"swapStore\": {\n" +
" \"type\": \"memory\",\n" +
" \"awsAccessKey\": null,\n" +
" \"awsSecret\": null,\n" +
" \"s3BucketName\": \"com.overleaf.testbucket\"\n" +
" },\n" +
" \"swapJob\": {\n" +
" \"minProjects\": " +
swapCfg.getMinProjects() +
",\n" +
" \"lowGiB\": " +
swapCfg.getLowGiB() +
",\n" +
" \"highGiB\": " +
swapCfg.getHighGiB() +
",\n" +
" \"intervalMillis\": " +
swapCfg.getIntervalMillis() +
"\n" +
" }\n";
}
cfgStr += "}\n";
writer.print(cfgStr);
writer.close();
return config.getAbsolutePath();
}
private Path getResource(String path) {
return Paths.get("src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest" + path);
return Paths.get(
"src/test/resources/" +
"uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest" + path
);
}
private InputStream getResourceAsStream(String path) {

40
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java
View File

@@ -3,15 +3,25 @@ package uk.ac.ic.wlgitbridge.bridge;
import org.junit.Before;
import org.junit.Test;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import java.io.IOException;
import java.util.ArrayDeque;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Created by winston on 20/08/2016.
@@ -35,24 +45,42 @@ public class BridgeTest {
dbStore = mock(DBStore.class);
swapStore = mock(SwapStore.class);
snapshotAPI = mock(SnapshotAPI.class);
resourceCache = mock(ResourceCache.class);
swapJob = mock(SwapJob.class);
bridge = new Bridge(
lock,
repoStore,
dbStore,
swapStore,
swapJob,
snapshotAPI,
resourceCache,
swapJob
resourceCache
);
}
@Test
public void shutdownStopsSwapJob() {
bridge.startSwapJob(1000);
bridge.startSwapJob();
bridge.doShutdown();
verify(swapJob).start(1000);
verify(swapJob).start();
verify(swapJob).stop();
}
@Test
public void updatingRepositorySetsLastAccessedTime(
) throws IOException, GitUserException {
ProjectRepo repo = mock(ProjectRepo.class);
when(repo.getProjectName()).thenReturn("asdf");
when(dbStore.getProjectState("asdf")).thenReturn(ProjectState.PRESENT);
when(
snapshotAPI.getSnapshotsForProjectAfterVersion(
any(),
any(),
anyInt()
)
).thenReturn(new ArrayDeque<Snapshot>());
bridge.updateRepository(null, repo);
verify(dbStore).setLastAccessedTime(eq("asdf"), any());
}
}

150
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java Normal file
View File

@@ -0,0 +1,150 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite;
import org.junit.Before;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
import java.io.IOException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
/**
* Created by winston on 23/08/2016.
*/
public class SqliteDBStoreTest {
private SqliteDBStore dbStore;
@Before
public void setup() throws IOException {
TemporaryFolder tmpFolder = new TemporaryFolder();
tmpFolder.create();
dbStore = new SqliteDBStore(tmpFolder.newFile("dbStore.db"));
}
@Test
public void testGetNumProjects() {
assertEquals(0, dbStore.getNumProjects());
dbStore.setLatestVersionForProject("asdf", 1);
assertEquals(1, dbStore.getNumProjects());
dbStore.setLatestVersionForProject("asdf1", 2);
assertEquals(2, dbStore.getNumProjects());
dbStore.setLatestVersionForProject("asdf1", 3);
assertEquals(2, dbStore.getNumProjects());
}
@Test
public void swapTableStartsOutEmpty() {
assertNull(dbStore.getOldestUnswappedProject());
}
@Test
public void testGetOldestUnswappedProject() {
dbStore.setLatestVersionForProject("older", 3);
dbStore.setLastAccessedTime(
"older",
Timestamp.valueOf(
LocalDateTime.now().minus(5, ChronoUnit.SECONDS)
)
);
dbStore.setLatestVersionForProject("asdf", 1);
dbStore.setLastAccessedTime(
"asdf",
Timestamp.valueOf(
LocalDateTime.now().minus(1, ChronoUnit.SECONDS)
)
);
assertEquals("older", dbStore.getOldestUnswappedProject());
dbStore.setLastAccessedTime(
"older",
Timestamp.valueOf(
LocalDateTime.now()
)
);
assertEquals("asdf", dbStore.getOldestUnswappedProject());
}
@Test
public void noOldestProjectIfAllEvicted() {
dbStore.setLatestVersionForProject("older", 3);
dbStore.setLastAccessedTime("older", null);
assertNull(dbStore.getOldestUnswappedProject());
}
@Test
public void nullLastAccessedTimesDoNotCount() {
dbStore.setLatestVersionForProject("older", 2);
dbStore.setLastAccessedTime(
"older",
Timestamp.valueOf(
LocalDateTime.now().minus(5, ChronoUnit.SECONDS)
)
);
dbStore.setLatestVersionForProject("newer", 3);
dbStore.setLastAccessedTime(
"newer",
Timestamp.valueOf(
LocalDateTime.now()
)
);
assertEquals("older", dbStore.getOldestUnswappedProject());
dbStore.setLastAccessedTime("older", null);
assertEquals("newer", dbStore.getOldestUnswappedProject());
}
@Test
public void missingProjectLastAccessedTimeCanBeSet() {
dbStore.setLatestVersionForProject("asdf", 1);
dbStore.setLastAccessedTime(
"asdf",
Timestamp.valueOf(LocalDateTime.now())
);
assertEquals("asdf", dbStore.getOldestUnswappedProject());
}
@Test
public void testGetNumUnswappedProjects() {
dbStore.setLatestVersionForProject("asdf", 1);
dbStore.setLastAccessedTime(
"asdf",
Timestamp.valueOf(LocalDateTime.now())
);
assertEquals(1, dbStore.getNumUnswappedProjects());
dbStore.setLastAccessedTime(
"asdf",
null
);
assertEquals(0, dbStore.getNumUnswappedProjects());
}
@Test
public void projectStateIsNotPresentIfNotInDBAtAll() {
assertEquals(
ProjectState.NOT_PRESENT,
dbStore.getProjectState("asdf")
);
}
@Test
public void projectStateIsPresentIfProjectHasLastAccessed() {
dbStore.setLatestVersionForProject("asdf", 1);
dbStore.setLastAccessedTime(
"asdf",
Timestamp.valueOf(LocalDateTime.now())
);
assertEquals(ProjectState.PRESENT, dbStore.getProjectState("asdf"));
}
@Test
public void projectStateIsSwappedIfLastAccessedIsNull() {
dbStore.setLatestVersionForProject("asdf", 1);
dbStore.setLastAccessedTime("asdf", null);
assertEquals(ProjectState.SWAPPED, dbStore.getProjectState("asdf"));
}
}

24
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteFilesForProjectSQLUpdateTest.java Normal file
View File

@@ -0,0 +1,24 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete;
import org.junit.Test;
import static org.junit.Assert.*;
public class DeleteFilesForProjectSQLUpdateTest {
@Test
public void testGetSQL() {
DeleteFilesForProjectSQLUpdate update =
new DeleteFilesForProjectSQLUpdate(
"projname",
"path1",
"path2"
);
assertEquals(
"DELETE FROM `url_index_store` " +
"WHERE `project_name` = ? " +
"AND path IN (?, ?);\n",
update.getSQL()
);
}
}

83
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest.java Normal file
View File

@@ -0,0 +1,83 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import org.apache.commons.io.FileUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import uk.ac.ic.wlgitbridge.util.Files;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import static org.junit.Assert.assertEquals;
/**
* Created by winston on 23/08/2016.
*/
public class FSGitRepoStoreTest {
public static File makeTempRepoDir(
TemporaryFolder tmpFolder,
String name
) throws IOException {
File tmp = tmpFolder.newFolder(name);
Path rootdir = Paths.get(
"src/test/resources/uk/ac/ic/wlgitbridge/"
+ "bridge/repo/FSGitRepoStoreTest/rootdir"
);
FileUtils.copyDirectory(rootdir.toFile(), tmp);
Files.renameAll(tmp, "DOTgit", ".git");
return tmp;
}
private FSGitRepoStore repoStore;
private File original;
@Before
public void setup() throws IOException {
TemporaryFolder tmpFolder = new TemporaryFolder();
tmpFolder.create();
File tmp = makeTempRepoDir(tmpFolder, "rootdir");
original = tmpFolder.newFolder("original");
FileUtils.copyDirectory(tmp, original);
repoStore = new FSGitRepoStore(tmp.getAbsolutePath());
}
@Test
public void testPurgeNonexistentProjects() {
File toDelete = new File(repoStore.getRootDirectory(), "idontexist");
File wlgb = new File(repoStore.getRootDirectory(), ".wlgb");
Assert.assertTrue(toDelete.exists());
Assert.assertTrue(wlgb.exists());
repoStore.purgeNonexistentProjects(Arrays.asList("proj1", "proj2"));
Assert.assertFalse(toDelete.exists());
Assert.assertTrue(wlgb.exists());
}
@Test
public void testTotalSize() {
assertEquals(31860, repoStore.totalSize());
}
@Test
public void zipAndUnzipShouldBeTheSame() throws IOException {
long beforeSize = repoStore.totalSize();
InputStream zipped = repoStore.bzip2Project("proj1");
repoStore.remove("proj1");
Assert.assertTrue(beforeSize > repoStore.totalSize());
repoStore.unbzip2Project("proj1", zipped);
Assert.assertEquals(beforeSize, repoStore.totalSize());
Assert.assertTrue(
Files.contentsAreEqual(
original,
repoStore.getRootDirectory()
)
);
}
}

119
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java Normal file
View File

@@ -0,0 +1,119 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
import org.junit.Before;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.FSGitRepoStore;
import uk.ac.ic.wlgitbridge.bridge.repo.FSGitRepoStoreTest;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.swap.store.InMemorySwapStore;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import uk.ac.ic.wlgitbridge.data.ProjectLockImpl;
import java.io.IOException;
import java.sql.Timestamp;
import java.time.Duration;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
* Created by winston on 20/08/2016.
*/
public class SwapJobImplTest {
private SwapJobImpl swapJob;
private ProjectLock lock;
private RepoStore repoStore;
private DBStore dbStore;
private SwapStore swapStore;
@Before
public void setup() throws IOException {
TemporaryFolder tmpFolder = new TemporaryFolder();
tmpFolder.create();
lock = new ProjectLockImpl();
repoStore = new FSGitRepoStore(
FSGitRepoStoreTest.makeTempRepoDir(
tmpFolder,
"repostore"
).getAbsolutePath()
);
dbStore = new SqliteDBStore(tmpFolder.newFile());
dbStore.setLatestVersionForProject("proj1", 0);
dbStore.setLatestVersionForProject("proj2", 0);
dbStore.setLastAccessedTime(
"proj1",
Timestamp.valueOf(LocalDateTime.now())
);
dbStore.setLastAccessedTime(
"proj2",
Timestamp.valueOf(
LocalDateTime.now().minus(1, ChronoUnit.SECONDS)
)
);
swapStore = new InMemorySwapStore();
swapJob = new SwapJobImpl(
1,
15000,
30000,
Duration.ofMillis(100),
lock,
repoStore,
dbStore,
swapStore
);
}
@Test
public void startingTimerAlwaysCausesASwap() {
swapJob.lowWatermarkBytes = 16384;
swapJob.interval = Duration.ofHours(1);
assertEquals(0, swapJob.swaps.get());
swapJob.start();
while (swapJob.swaps.get() <= 0);
assertTrue(swapJob.swaps.get() > 0);
}
@Test
public void swapsHappenEveryInterval() {
swapJob.lowWatermarkBytes = 16384;
assertEquals(0, swapJob.swaps.get());
swapJob.start();
while (swapJob.swaps.get() <= 1);
assertTrue(swapJob.swaps.get() > 1);
}
@Test
public void noProjectsGetSwappedWhenUnderHighWatermark() {
swapJob.highWatermarkBytes = 65536;
assertEquals(2, dbStore.getNumUnswappedProjects());
swapJob.start();
while (swapJob.swaps.get() < 1);
assertEquals(2, dbStore.getNumUnswappedProjects());
}
@Test
public void correctProjGetSwappedWhenOverHighWatermark(
) throws IOException {
swapJob.lowWatermarkBytes = 16384;
assertEquals(2, dbStore.getNumUnswappedProjects());
assertEquals("proj2", dbStore.getOldestUnswappedProject());
swapJob.start();
while (swapJob.swaps.get() < 1);
assertEquals(1, dbStore.getNumUnswappedProjects());
assertEquals("proj1", dbStore.getOldestUnswappedProject());
swapJob.restore("proj2");
int numSwaps = swapJob.swaps.get();
while (swapJob.swaps.get() <= numSwaps);
assertEquals(1, dbStore.getNumUnswappedProjects());
assertEquals("proj2", dbStore.getOldestUnswappedProject());
}
}

96
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStoreTest.java Normal file
View File

@@ -0,0 +1,96 @@
package uk.ac.ic.wlgitbridge.bridge.swap.store;
import org.apache.commons.io.IOUtils;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import uk.ac.ic.wlgitbridge.bridge.swap.store.InMemorySwapStore;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import static org.junit.Assert.assertArrayEquals;
/**
* Created by winston on 23/08/2016.
*/
public class InMemorySwapStoreTest {
private final InMemorySwapStore swapStore = new InMemorySwapStore();
@Rule
public final ExpectedException exception = ExpectedException.none();
@Test
public void downloadingNonExistentFileThrows() {
exception.expect(IllegalArgumentException.class);
swapStore.openDownloadStream("asdf");
}
@Test
public void canDownloadUploadedFiles() throws IOException {
byte[] proj1Contents = "helloproj1".getBytes();
byte[] proj2Contents = "asdfproj2".getBytes();
swapStore.upload(
"proj1",
new ByteArrayInputStream(proj1Contents),
proj1Contents.length
);
swapStore.upload(
"proj2",
new ByteArrayInputStream(proj2Contents),
proj2Contents.length
);
assertArrayEquals(
proj1Contents,
IOUtils.toByteArray(swapStore.openDownloadStream("proj1"))
);
assertArrayEquals(
proj2Contents,
IOUtils.toByteArray(swapStore.openDownloadStream("proj2"))
);
}
@Test
public void uploadingForTheSameProjectOverwritesTheFile(
) throws IOException {
byte[] proj1Contents = "helloproj1".getBytes();
byte[] proj1NewContents = "goodbyeproj1".getBytes();
swapStore.upload(
"proj1",
new ByteArrayInputStream(proj1Contents),
proj1Contents.length
);
assertArrayEquals(
proj1Contents,
IOUtils.toByteArray(swapStore.openDownloadStream("proj1"))
);
swapStore.upload(
"proj1",
new ByteArrayInputStream(proj1NewContents),
proj1NewContents.length
);
assertArrayEquals(
proj1NewContents,
IOUtils.toByteArray(swapStore.openDownloadStream("proj1"))
);
}
@Test
public void canRemoveFiles() throws IOException {
byte[] projContents = "total garbage".getBytes();
swapStore.upload(
"proj",
new ByteArrayInputStream(projContents),
projContents.length
);
assertArrayEquals(
projContents,
IOUtils.toByteArray(swapStore.openDownloadStream("proj"))
);
swapStore.remove("proj");
exception.expect(IllegalArgumentException.class);
swapStore.openDownloadStream("proj");
}
}

41
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStoreTest.java Normal file
View File

@@ -0,0 +1,41 @@
package uk.ac.ic.wlgitbridge.bridge.swap.store;
import org.junit.Before;
/**
* Created by winston on 21/08/2016.
*/
public class S3SwapStoreTest {
private static final String accessKey = null;
private static final String secret = null;
private static final String bucketName = "com.overleaf.testbucket";
private S3SwapStore s3;
@Before
public void setup() {
if (accessKey == null || secret == null) {
s3 = null;
return;
}
s3 = new S3SwapStore(accessKey, secret, bucketName);
}
// @Ignore
// @Test
// public void testUploadDownloadDelete() throws Exception {
// assumeNotNull(s3);
// String projName = "abc123";
// byte[] contents = "hello".getBytes();
// s3.upload(
// projName,
// new ByteArrayInputStream(contents),
// contents.length
// );
// InputStream down = s3.openDownloadStream(projName);
// s3.remove(projName);
// assertArrayEquals(contents, IOUtils.toByteArray(down));
// }
}

14
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/delete/DeleteFilesForProjectSQLUpdateTest.java
View File

@@ -1,14 +0,0 @@
package uk.ac.ic.wlgitbridge.data.model.db.sql.update.delete;
import org.junit.Test;
import static org.junit.Assert.*;
public class DeleteFilesForProjectSQLUpdateTest {
@Test
public void testGetSQL() {
DeleteFilesForProjectSQLUpdate update = new DeleteFilesForProjectSQLUpdate("projname", "path1", "path2");
assertEquals("DELETE FROM `url_index_store` WHERE `project_name` = ? AND path IN (?, ?);\n", update.getSQL());
}
}

18
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/ProjectTest.java Normal file
View File

@@ -0,0 +1,18 @@
package uk.ac.ic.wlgitbridge.util;
import org.junit.Assert;
import org.junit.Test;
/**
* Created by winston on 23/08/2016.
*/
public class ProjectTest {
@Test
public void testValidProjectNames() {
Assert.assertFalse(Project.isValidProjectName(null));
Assert.assertFalse(Project.isValidProjectName(""));
Assert.assertFalse(Project.isValidProjectName(".wlgb"));
}
}

52
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TarTest.java Normal file
View File

@@ -0,0 +1,52 @@
package uk.ac.ic.wlgitbridge.util;
import org.apache.commons.io.FileUtils;
import org.junit.Before;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import static org.junit.Assert.assertTrue;
/**
* Created by winston on 23/08/2016.
*/
public class TarTest {
private File testDir;
private File tmpDir;
@Before
public void setup() throws IOException {
TemporaryFolder tmpFolder = new TemporaryFolder();
tmpFolder.create();
testDir = tmpFolder.newFolder("testdir");
Path resdir = Paths.get(
"src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir"
);
FileUtils.copyDirectory(resdir.toFile(), testDir);
tmpDir = tmpFolder.newFolder();
}
@Test
public void tarAndUntarProducesTheSameResult() throws IOException {
InputStream tar = Tar.tar(testDir);
Tar.untar(tar, tmpDir);
File untarred = new File(tmpDir, "testdir");
assertTrue(Files.contentsAreEqual(testDir, untarred));
}
@Test
public void tarbz2AndUntarbz2ProducesTheSameResult() throws IOException {
InputStream tarbz2 = Tar.bz2.zip(testDir);
Tar.bz2.unzip(tarbz2, tmpDir);
File unzipped = new File(tmpDir, "testdir");
assertTrue(Files.contentsAreEqual(testDir, unzipped));
}
}

19
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TimerTest.java Normal file
View File

@@ -0,0 +1,19 @@
package uk.ac.ic.wlgitbridge.util;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
/**
* Created by winston on 23/08/2016.
*/
public class TimerTest {
@Test
public void testMakeTimerTask() {
int[] iPtr = new int[] { 3 };
Timer.makeTimerTask(() -> iPtr[0] = 5).run();
assertEquals(5, iPtr[0]);
}
}

6
services/git-bridge/src/test/resources/logback-test.xml
View File

@@ -1,7 +1,7 @@
<configuration>
<!-- Log everything (subject to logger and root levels set below) to stdout. -->
<appender name="tempfile" class="ch.qos.logback.core.FileAppender">
<file>${java.io.tmpdir}/git-bridge-test.log</file>
<appender name="stderr" class="ch.qos.logback.core.ConsoleAppender">
<target>System.err</target>
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%thread] %logger{0}: %msg%n</pattern>
</encoder>
@@ -12,6 +12,6 @@
<!-- The root log level determines how much our dependencies put in the logs. -->
<root level="WARN">
<appender-ref ref="tempfile" />
<appender-ref ref="stderr" />
</root>
</configuration>

90
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/state.json Normal file
View File

@@ -0,0 +1,90 @@
[
{
"project": "testproj1",
"getDoc": {
"versionID": 1,
"createdAt": "2014-11-30T18:40:58.123Z",
"email": "jdleesmiller+1@gmail.com",
"name": "John+1"
},
"getSavedVers": [
{
"versionID": 1,
"comment": "added more info on doc GET and error details",
"email": "jdleesmiller+1@gmail.com",
"name": "John+1",
"createdAt": "2014-11-30T18:47:01.456Z"
}
],
"getForVers": [
{
"versionID": 1,
"srcs": [
{
"content": "content\n",
"path": "main.tex"
},
{
"content": "This text is from another file.",
"path": "foo/bar/test.tex"
}
],
"atts": [
{
"url": "http://127.0.0.1:3874/state/testproj1/overleaf-white-410.png",
"path": "overleaf-white-410.png"
}
]
}
],
"push": "success",
"postback": {
"type": "success",
"versionID": 2
}
},
{
"project": "testproj2",
"getDoc": {
"versionID": 1,
"createdAt": "2014-11-30T18:40:58.123Z",
"email": "jdleesmiller+1@gmail.com",
"name": "John+1"
},
"getSavedVers": [
{
"versionID": 1,
"comment": "added more info on doc GET and error details",
"email": "jdleesmiller+1@gmail.com",
"name": "John+1",
"createdAt": "2014-11-30T18:47:01.456Z"
}
],
"getForVers": [
{
"versionID": 1,
"srcs": [
{
"content": "different content\n",
"path": "main.tex"
},
{
"content": "a different one",
"path": "foo/bar/test.tex"
}
],
"atts": [
{
"url": "http://127.0.0.1:3874/state/testproj2/editor-versions-a7e4de19d015c3e7477e3f7eaa6c418e.png",
"path": "editor-versions-a7e4de19d015c3e7477e3f7eaa6c418e.png"
}
]
}
],
"push": "success",
"postback": {
"type": "success",
"versionID": 2
}
}
]

1
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj1/foo/bar/test.tex Normal file
View File

@@ -0,0 +1 @@
This text is from another file.

1
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj1/main.tex Normal file
View File

@@ -0,0 +1 @@
content

BIN
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj1/overleaf-white-410.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 8.6 KiB

BIN
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj2/editor-versions-a7e4de19d015c3e7477e3f7eaa6c418e.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.5 KiB

1
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj2/foo/bar/test.tex Normal file
View File

@@ -0,0 +1 @@
a different one

1
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj2/main.tex Normal file
View File

@@ -0,0 +1 @@
different content

0
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/.wlgb/wlgb.db Normal file
View File

Some files were not shown because too many files have changed in this diff Show More