diff --git a/services/git-bridge/.gitignore b/services/git-bridge/.gitignore index d198a6a13c..43696eb643 100644 --- a/services/git-bridge/.gitignore +++ b/services/git-bridge/.gitignore @@ -1,6 +1,11 @@ # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 +# Let's not share anything because we're using Maven. + +.idea +*.iml + # User-specific stuff: .idea/workspace.xml .idea/tasks.xml diff --git a/services/git-bridge/.idea/compiler.xml b/services/git-bridge/.idea/compiler.xml deleted file mode 100644 index 13e26ea226..0000000000 --- a/services/git-bridge/.idea/compiler.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/services/git-bridge/.idea/copyright/profiles_settings.xml b/services/git-bridge/.idea/copyright/profiles_settings.xml deleted file mode 100644 index e7bedf3377..0000000000 --- a/services/git-bridge/.idea/copyright/profiles_settings.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - \ No newline at end of file diff --git a/services/git-bridge/.idea/encodings.xml b/services/git-bridge/.idea/encodings.xml deleted file mode 100644 index c0bce70846..0000000000 --- a/services/git-bridge/.idea/encodings.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/services/git-bridge/.idea/misc.xml b/services/git-bridge/.idea/misc.xml deleted file mode 100644 index dfca8c47e5..0000000000 --- a/services/git-bridge/.idea/misc.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - 1.8 - - - - - - - - \ No newline at end of file diff --git a/services/git-bridge/.idea/modules.xml b/services/git-bridge/.idea/modules.xml deleted file mode 100644 index 4cebdcf6c0..0000000000 --- a/services/git-bridge/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/services/git-bridge/README.md b/services/git-bridge/README.md index fa8c76f7db..ba9208c4a2 100644 --- a/services/git-bridge/README.md +++ b/services/git-bridge/README.md @@ -3,9 +3,23 @@ writelatex-git-bridge Required -------- - * `maven` (for building) + * `maven` (for building, running tests and packaging) * `jdk-8` (for compiling and running) +Commands +-------- + +To be run from the base directory: + +**Build jar**: +`mvn package` + +**Run tests**: +`mvn test` + +**Clean**: +`mvn clean` + Installation ------------ @@ -32,16 +46,46 @@ The configuration file is in `.json` format. { "port" (int): the port number, - "rootGitDirectory" (string): the directory in which to store git repos and the db/atts, + "rootGitDirectory" (string): the directory in which to store + git repos and the db/atts, "apiBaseUrl" (string): base url for the snapshot api, "username" (string, optional): username for http basic auth, "password" (string, optional): password for http basic auth, "postbackBaseUrl" (string): the postback url, - "serviceName" (string): current name of writeLaTeX in case it ever changes - "oauth2" (object): { /* null or missing if oauth2 shouldn't be used */ - "oauth2ClientID" (string): oauth2 client ID - "oauth2ClientSecret" (string): oauth2 client secret - "oauth2Server" (string): oauth2 server, with protocol and without trailing slash + "serviceName" (string): current name of writeLaTeX + in case it ever changes, + "oauth2" (object): { null or missing if oauth2 shouldn't be used + "oauth2ClientID" (string): oauth2 client ID, + "oauth2ClientSecret" (string): oauth2 client secret, + "oauth2Server" (string): oauth2 server, + with protocol and + without trailing slash + }, + "swapStore" (object, optional): { the place to swap projects to. + if null, type defaults to + "noop" + "type" (string): "s3", "memory", "noop" (not recommended), + "awsAccessKey" (string, optional): only for s3, + "awsSecret" (string, optional): only for s3, + "s3BucketName" (string, optional): only for s3 + }, + "swapJob" (object, optional): { configure the project + swapping job. + if null, defaults to no-op + "minProjects" (int64): lower bound on number of projects + present. The swap job will never go + below this, regardless of what the + watermark shows. Regardless, if + minProjects prevents an eviction, + the swap job will WARN, + "lowGiB" (int32): the low watermark for swapping, + i.e. swap until disk usage is below this, + "highGiB" (int32): the high watermark for swapping, + i.e. start swapping when + disk usage becomes this, + "intervalMillis" (int64): amount of time in between running + swap job and checking watermarks. + 3600000 is 1 hour } } diff --git a/services/git-bridge/conf/example_config.json b/services/git-bridge/conf/example_config.json new file mode 100644 index 0000000000..c7f1193811 --- /dev/null +++ b/services/git-bridge/conf/example_config.json @@ -0,0 +1,26 @@ +{ + "port": 8080, + "rootGitDirectory": "/tmp/wlgb", + "apiBaseUrl": "https://localhost/api/v0", + "username": "user", + "password": "pass", + "postbackBaseUrl": "https://localhost", + "serviceName": "Overleaf", + "oauth2": { + "oauth2ClientID": "asdf", + "oauth2ClientSecret": "asdf", + "oauth2Server": "https://localhost" + }, + "swapStore": { + "type": "s3", + "awsAccessKey": "asdf", + "awsSecret": "asdf", + "s3BucketName": "com.overleaf.testbucket" + }, + "swapJob": { + "minProjects": 50, + "lowGiB": 128, + "highGiB": 256, + "intervalMillis": 3600000 + } +} diff --git a/services/git-bridge/lib/newrelic.jar b/services/git-bridge/lib/newrelic.jar new file mode 100644 index 0000000000..f02c7055e4 Binary files /dev/null and b/services/git-bridge/lib/newrelic.jar differ diff --git a/services/git-bridge/newrelic.yml b/services/git-bridge/newrelic.yml new file mode 100644 index 0000000000..ceac8b2e2f --- /dev/null +++ b/services/git-bridge/newrelic.yml @@ -0,0 +1,300 @@ +# This file configures the New Relic Agent. New Relic monitors +# Java applications with deep visibility and low overhead. For more details and additional +# configuration options visit https://docs.newrelic.com/docs/java/java-agent-configuration. +# +# This configuration file is custom generated for Winston +# +# This section is for settings common to all environments. +# Do not add anything above this next line. +common: &default_settings + + # ============================== LICENSE KEY =============================== + # You must specify the license key associated with your New Relic + # account. For example, if your license key is 12345 use this: + # license_key: '12345' + # The key binds your Agent's data to your account in the New Relic service. + license_key: '' + + # Agent Enabled + # Use this setting to disable the agent instead of removing it from the startup command. + # Default is true. + agent_enabled: true + + # Set the name of your application as you'd like it show up in New Relic. + # If enable_auto_app_naming is false, the agent reports all data to this application. + # Otherwise, the agent reports only background tasks (transactions for non-web applications) + # to this application. To report data to more than one application + # (useful for rollup reporting), separate the application names with ";". + # For example, to report data to "My Application" and "My Application 2" use this: + # app_name: My Application;My Application 2 + # This setting is required. Up to 3 different application names can be specified. + # The first application name must be unique. + app_name: Git Bridge + + # To enable high security, set this property to true. When in high + # security mode, the agent will use SSL and obfuscated SQL. Additionally, + # request parameters and message parameters will not be sent to New Relic. + high_security: false + + # Set to true to enable support for auto app naming. + # The name of each web app is detected automatically + # and the agent reports data separately for each one. + # This provides a finer-grained performance breakdown for + # web apps in New Relic. + # Default is false. + enable_auto_app_naming: false + + # Set to true to enable component-based transaction naming. + # Set to false to use the URI of a web request as the name of the transaction. + # Default is true. + enable_auto_transaction_naming: true + + # The agent uses its own log file to keep its logging + # separate from that of your application. Specify the log level here. + # This setting is dynamic, so changes do not require restarting your application. + # The levels in increasing order of verboseness are: + # off, severe, warning, info, fine, finer, finest + # Default is info. + log_level: info + + # Log all data sent to and from New Relic in plain text. + # This setting is dynamic, so changes do not require restarting your application. + # Default is false. + audit_mode: false + + # The number of backup log files to save. + # Default is 1. + log_file_count: 1 + + # The maximum number of kbytes to write to any one log file. + # The log_file_count must be set greater than 1. + # Default is 0 (no limit). + log_limit_in_kbytes: 0 + + # Override other log rolling configuration and roll the logs daily. + # Default is false. + log_daily: false + + # The name of the log file. + # Default is newrelic_agent.log. + log_file_name: newrelic_agent.log + + # The log file directory. + # Default is the logs directory in the newrelic.jar parent directory. + #log_file_path: + + # The agent communicates with New Relic via https by + # default. If you want to communicate with newrelic via http, + # then turn off SSL by setting this value to false. + # This work is done asynchronously to the threads that process your + # application code, so response times will not be directly affected + # by this change. + # Default is true. + ssl: true + + # Proxy settings for connecting to the New Relic server: + # If a proxy is used, the host setting is required. Other settings + # are optional. Default port is 8080. The username and password + # settings will be used to authenticate to Basic Auth challenges + # from a proxy server. + #proxy_host: hostname + #proxy_port: 8080 + #proxy_user: username + #proxy_password: password + + # Limits the number of lines to capture for each stack trace. + # Default is 30 + max_stack_trace_lines: 30 + + # Provides the ability to configure the attributes sent to New Relic. These + # attributes can be found in transaction traces, traced errors, Insight's + # transaction events, and Insight's page views. + attributes: + + # When true, attributes will be sent to New Relic. The default is true. + enabled: true + + #A comma separated list of attribute keys whose values should + # be sent to New Relic. + #include: + + # A comma separated list of attribute keys whose values should + # not be sent to New Relic. + #exclude: + + + # Transaction tracer captures deep information about slow + # transactions and sends this to the New Relic service once a + # minute. Included in the transaction is the exact call sequence of + # the transactions including any SQL statements issued. + transaction_tracer: + + # Transaction tracer is enabled by default. Set this to false to turn it off. + # This feature is not available to Lite accounts and is automatically disabled. + # Default is true. + enabled: true + + # Threshold in seconds for when to collect a transaction + # trace. When the response time of a controller action exceeds + # this threshold, a transaction trace will be recorded and sent to + # New Relic. Valid values are any float value, or (default) "apdex_f", + # which will use the threshold for the "Frustrated" Apdex level + # (greater than four times the apdex_t value). + # Default is apdex_f. + transaction_threshold: apdex_f + + # When transaction tracer is on, SQL statements can optionally be + # recorded. The recorder has three modes, "off" which sends no + # SQL, "raw" which sends the SQL statement in its original form, + # and "obfuscated", which strips out numeric and string literals. + # Default is obfuscated. + record_sql: obfuscated + + # Set this to true to log SQL statements instead of recording them. + # SQL is logged using the record_sql mode. + # Default is false. + log_sql: false + + # Threshold in seconds for when to collect stack trace for a SQL + # call. In other words, when SQL statements exceed this threshold, + # then capture and send to New Relic the current stack trace. This is + # helpful for pinpointing where long SQL calls originate from. + # Default is 0.5 seconds. + stack_trace_threshold: 0.5 + + # Determines whether the agent will capture query plans for slow + # SQL queries. Only supported for MySQL and PostgreSQL. + # Default is true. + explain_enabled: true + + # Threshold for query execution time below which query plans will not + # not be captured. Relevant only when `explain_enabled` is true. + # Default is 0.5 seconds. + explain_threshold: 0.5 + + # Use this setting to control the variety of transaction traces. + # The higher the setting, the greater the variety. + # Set this to 0 to always report the slowest transaction trace. + # Default is 20. + top_n: 20 + + # Error collector captures information about uncaught exceptions and + # sends them to New Relic for viewing. + error_collector: + + # This property enables the collection of errors. If the property is not + # set or the property is set to false, then errors will not be collected. + # Default is true. + enabled: true + + # Use this property to exclude specific exceptions from being reported as errors + # by providing a comma separated list of full class names. + # The default is to exclude akka.actor.ActorKilledException. If you want to override + # this, you must provide any new value as an empty list is ignored. + ignore_errors: akka.actor.ActorKilledException + + # Use this property to exclude specific http status codes from being reported as errors + # by providing a comma separated list of status codes. + # The default is to exclude 404s. If you want to override + # this, you must provide any new value as an empty list is ignored. + ignore_status_codes: 404 + + # Transaction Events are used for Histograms and Percentiles. Unaggregated data is collected + # for each web transaction and sent to the server on harvest. + transaction_events: + + # Set to false to disable transaction events. + # Default is true. + enabled: true + + # Events are collected up to the configured amount. Afterwards, events are sampled to + # maintain an even distribution across the harvest cycle. + # Default is 2000. Setting to 0 will disable. + max_samples_stored: 2000 + + # Cross Application Tracing adds request and response headers to + # external calls using supported HTTP libraries to provide better + # performance data when calling applications monitored by other New Relic Agents. + cross_application_tracer: + + # Set to false to disable cross application tracing. + # Default is true. + enabled: true + + # Thread profiler measures wall clock time, CPU time, and method call counts + # in your application's threads as they run. + # This feature is not available to Lite accounts and is automatically disabled. + thread_profiler: + + # Set to false to disable the thread profiler. + # Default is true. + enabled: true + + # New Relic Real User Monitoring gives you insight into the performance real users are + # experiencing with your website. This is accomplished by measuring the time it takes for + # your users' browsers to download and render your web pages by injecting a small amount + # of JavaScript code into the header and footer of each page. + browser_monitoring: + + # By default the agent automatically inserts API calls in compiled JSPs to + # inject the monitoring JavaScript into web pages. Not all rendering engines are supported. + # See https://docs.newrelic.com/docs/java/real-user-monitoring-in-java#manual_instrumentation + # for instructions to add these manually to your pages. + # Set this attribute to false to turn off this behavior. + auto_instrument: true + + class_transformer: + # This instrumentation reports the name of the user principal returned from + # HttpServletRequest.getUserPrincipal() when servlets and filters are invoked. + com.newrelic.instrumentation.servlet-user: + enabled: false + + com.newrelic.instrumentation.spring-aop-2: + enabled: false + + # Classes loaded by classloaders in this list will not be instrumented. + # This is a useful optimization for runtimes which use classloaders to + # load dynamic classes which the agent would not instrument. + classloader_excludes: + groovy.lang.GroovyClassLoader$InnerLoader, + org.codehaus.groovy.runtime.callsite.CallSiteClassLoader, + com.collaxa.cube.engine.deployment.BPELClassLoader, + org.springframework.data.convert.ClassGeneratingEntityInstantiator$ObjectInstantiatorClassGenerator, + org.mvel2.optimizers.impl.asm.ASMAccessorOptimizer$ContextClassLoader, + gw.internal.gosu.compiler.SingleServingGosuClassLoader, + + # User-configurable custom labels for this agent. Labels are name-value pairs. + # There is a maximum of 64 labels per agent. Names and values are limited to 255 characters. + # Names and values may not contain colons (:) or semicolons (;). + labels: + + # An example label + #label_name: label_value + + +# Application Environments +# ------------------------------------------ +# Environment specific settings are in this section. +# You can use the environment to override the default settings. +# For example, to change the app_name setting. +# Use -Dnewrelic.environment= on the Java startup command line +# to set the environment. +# The default environment is production. + +# NOTE if your application has other named environments, you should +# provide configuration settings for these environments here. + +development: + <<: *default_settings + app_name: Git Bridge (Development) + +test: + <<: *default_settings + app_name: Git Bridge (Test) + +production: + <<: *default_settings + +staging: + <<: *default_settings + app_name: Git Bridge (Staging) diff --git a/services/git-bridge/pom.xml b/services/git-bridge/pom.xml index 5666ad925b..07db011f45 100644 --- a/services/git-bridge/pom.xml +++ b/services/git-bridge/pom.xml @@ -140,5 +140,29 @@ mockito-core 1.10.19 + + + com.amazonaws + aws-java-sdk + 1.11.28 + + + + org.apache.httpcomponents + httpclient + 4.5.2 + + + + commons-io + commons-io + 2.5 + + + + org.apache.commons + commons-compress + 1.12 + diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/GitBridgeApp.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/GitBridgeApp.java index 6e3f049cf6..f354ceb0f2 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/GitBridgeApp.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/GitBridgeApp.java @@ -22,10 +22,10 @@ public class GitBridgeApp implements Runnable { public static final int EXIT_CODE_FAILED = 1; private static final String USAGE_MESSAGE = - "usage: writelatex-git-bridge config_file"; + "usage: writelatex-git-bridge [config_file]"; private String configFilePath; - private Config config; + Config config; private GitBridgeServer server; /** diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java index f434437606..53c9ad7d46 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java @@ -4,12 +4,16 @@ import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import uk.ac.ic.wlgitbridge.application.exception.ConfigFileException; +import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig; +import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStoreConfig; import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource; import uk.ac.ic.wlgitbridge.util.Instance; +import javax.annotation.Nullable; import java.io.FileReader; import java.io.IOException; import java.io.Reader; +import java.util.Optional; /** * Created by Winston on 05/12/14. @@ -25,7 +29,9 @@ public class Config implements JSONSource { config.apiBaseURL, config.postbackURL, config.serviceName, - Oauth2.asSanitised(config.oauth2) + Oauth2.asSanitised(config.oauth2), + SwapStoreConfig.sanitisedCopy(config.swapStore), + config.swapJob ); } @@ -36,7 +42,12 @@ public class Config implements JSONSource { private String apiBaseURL; private String postbackURL; private String serviceName; + @Nullable private Oauth2 oauth2; + @Nullable + private SwapStoreConfig swapStore; + @Nullable + private SwapJobConfig swapJob; public Config(String configFilePath) throws ConfigFileException, IOException { @@ -47,14 +58,18 @@ public class Config implements JSONSource { fromJSON(new Gson().fromJson(reader, JsonElement.class)); } - public Config(int port, - String rootGitDirectory, - String username, - String password, - String apiBaseURL, - String postbackURL, - String serviceName, - Oauth2 oauth2) { + public Config( + int port, + String rootGitDirectory, + String username, + String password, + String apiBaseURL, + String postbackURL, + String serviceName, + Oauth2 oauth2, + SwapStoreConfig swapStore, + SwapJobConfig swapJob + ) { this.port = port; this.rootGitDirectory = rootGitDirectory; this.username = username; @@ -63,6 +78,8 @@ public class Config implements JSONSource { this.postbackURL = postbackURL; this.serviceName = serviceName; this.oauth2 = oauth2; + this.swapStore = swapStore; + this.swapJob = swapJob; } @Override @@ -89,6 +106,14 @@ public class Config implements JSONSource { postbackURL += "/"; } oauth2 = new Gson().fromJson(configObject.get("oauth2"), Oauth2.class); + swapStore = new Gson().fromJson( + configObject.get("swapStore"), + SwapStoreConfig.class + ); + swapJob = new Gson().fromJson( + configObject.get("swapJob"), + SwapJobConfig.class + ); } public String getSanitisedString() { @@ -134,6 +159,14 @@ public class Config implements JSONSource { return oauth2; } + public Optional getSwapStore() { + return Optional.ofNullable(swapStore); + } + + public Optional getSwapJob() { + return Optional.ofNullable(swapJob); + } + private JsonElement getElement(JsonObject configObject, String name) { JsonElement element = configObject.get(name); if (element == null) { diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java index 294a5df50a..2e98d546ec 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java @@ -3,15 +3,17 @@ package uk.ac.ic.wlgitbridge.bridge; import com.google.api.client.auth.oauth2.Credential; import org.eclipse.jgit.transport.ServiceMayNotContinueException; import uk.ac.ic.wlgitbridge.bridge.db.DBStore; +import uk.ac.ic.wlgitbridge.bridge.db.ProjectState; +import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard; import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock; import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache; import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache; import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotAPI; import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI; -import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob; -import uk.ac.ic.wlgitbridge.bridge.swap.SwapJobImpl; -import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore; +import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob; +import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig; +import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore; import uk.ac.ic.wlgitbridge.data.CandidateSnapshot; import uk.ac.ic.wlgitbridge.data.ProjectLockImpl; import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents; @@ -29,7 +31,10 @@ import uk.ac.ic.wlgitbridge.snapshot.push.PushResult; import uk.ac.ic.wlgitbridge.snapshot.push.exception.*; import uk.ac.ic.wlgitbridge.util.Log; +import java.io.File; import java.io.IOException; +import java.sql.Timestamp; +import java.time.LocalDateTime; import java.util.*; /** @@ -42,18 +47,19 @@ public class Bridge { private final RepoStore repoStore; private final DBStore dbStore; private final SwapStore swapStore; + private final SwapJob swapJob; private final SnapshotAPI snapshotAPI; private final ResourceCache resourceCache; - private final SwapJob swapJob; private final PostbackManager postbackManager; public static Bridge make( RepoStore repoStore, DBStore dbStore, - SwapStore swapStore + SwapStore swapStore, + Optional swapJobConfig ) { ProjectLock lock = new ProjectLockImpl((int threads) -> Log.info("Waiting for " + threads + " projects...") @@ -63,14 +69,15 @@ public class Bridge { repoStore, dbStore, swapStore, - new NetSnapshotAPI(), - new UrlResourceCache(dbStore), - new SwapJobImpl( + SwapJob.fromConfig( + swapJobConfig, lock, repoStore, dbStore, swapStore - ) + ), + new NetSnapshotAPI(), + new UrlResourceCache(dbStore) ); } @@ -79,9 +86,9 @@ public class Bridge { RepoStore repoStore, DBStore dbStore, SwapStore swapStore, + SwapJob swapJob, SnapshotAPI snapshotAPI, - ResourceCache resourceCache, - SwapJob swapJob + ResourceCache resourceCache ) { this.lock = lock; this.repoStore = repoStore; @@ -104,66 +111,140 @@ public class Bridge { Log.info("Bye"); } - public void startSwapJob(int intervalMillis) { - swapJob.start(intervalMillis); + public void startSwapJob() { + swapJob.start(); } - /* TODO: Remove these when WLBridged is moved into RepoStore */ - public void lockForProject(String projectName) { - lock.lockForProject(projectName); + public void checkDB() { + Log.info("Checking DB"); + File rootDir = repoStore.getRootDirectory(); + for (File f : rootDir.listFiles()) { + if (f.getName().equals(".wlgb")) { + continue; + } + String projName = f.getName(); + try (LockGuard __ = lock.lockGuard(projName)) { + File dotGit = new File(f, ".git"); + if (!dotGit.exists()) { + Log.warn("Project: {} has no .git", projName); + continue; + } + ProjectState state = dbStore.getProjectState(projName); + if (state != ProjectState.NOT_PRESENT) { + continue; + } + Log.warn("Project: {} not in swap_store, adding", projName); + dbStore.setLastAccessedTime( + projName, + new Timestamp(dotGit.lastModified()) + ); + } + } } - public void unlockForProject(String projectName) { - lock.unlockForProject(projectName); - } - - public boolean repositoryExists(Credential oauth2, String projectName) - throws ServiceMayNotContinueException, GitUserException { - lockForProject(projectName); - GetDocRequest getDocRequest = new GetDocRequest(oauth2, projectName); - getDocRequest.request(); - try { + public boolean projectExists( + Credential oauth2, + String projectName + ) throws ServiceMayNotContinueException, + GitUserException { + try (LockGuard __ = lock.lockGuard(projectName)) { + GetDocRequest getDocRequest = new GetDocRequest( + oauth2, + projectName + ); + getDocRequest.request(); getDocRequest.getResult().getVersionID(); + return true; } catch (InvalidProjectException e) { return false; - } finally { - unlockForProject(projectName); } - return true; } - public void getWritableRepositories( + public void updateRepository( Credential oauth2, ProjectRepo repo - ) throws IOException, - GitUserException { - Log.info("[{}] Fetching", repo.getProjectName()); - updateProjectWithName(oauth2, repo); + ) throws IOException, GitUserException { + String projectName = repo.getProjectName(); + try (LockGuard __ = lock.lockGuard(projectName)) { + Log.info("[{}] Updating", projectName); + updateRepositoryCritical(oauth2, repo); + } } - public void - putDirectoryContentsToProjectWithName(Credential oauth2, - String projectName, - RawDirectory directoryContents, - RawDirectory oldDirectoryContents, - String hostname) - throws SnapshotPostException, IOException, ForbiddenException { - lock.lockForProject(projectName); - CandidateSnapshot candidate = null; - try { - Log.info("[{}] Pushing", projectName); - String postbackKey = postbackManager.makeKeyForProject(projectName); - Log.info( - "[{}] Created postback key: {}", + private void updateRepositoryCritical( + Credential oauth2, + ProjectRepo repo + ) throws IOException, GitUserException { + String projectName = repo.getProjectName(); + ProjectState state = dbStore.getProjectState(projectName); + switch (state) { + case NOT_PRESENT: + repo.initRepo(repoStore); + break; + case SWAPPED: + swapJob.restore(projectName); + /* Fallthrough */ + default: + repo.useExistingRepository(repoStore); + } + updateProject(oauth2, repo); + dbStore.setLastAccessedTime( + projectName, + Timestamp.valueOf(LocalDateTime.now()) + ); + } + + public void putDirectoryContentsToProjectWithName( + Credential oauth2, + String projectName, + RawDirectory directoryContents, + RawDirectory oldDirectoryContents, + String hostname + ) throws SnapshotPostException, IOException, ForbiddenException { + try (LockGuard __ = lock.lockGuard(projectName)) { + pushToProjectCritical( + oauth2, projectName, - postbackKey + directoryContents, + oldDirectoryContents ); - candidate = - createCandidateSnapshot( - projectName, - directoryContents, - oldDirectoryContents - ); + } catch (SevereSnapshotPostException e) { + Log.warn("[" + projectName + "] Failed to put to Overleaf", e); + throw e; + } catch (SnapshotPostException e) { + /* Stack trace should be printed further up */ + Log.warn( + "[{}] Exception when waiting for postback: {}", + projectName, + e.getClass().getSimpleName() + ); + throw e; + } catch (IOException e) { + Log.warn("[{}] IOException on put", projectName); + throw e; + } + } + + private void pushToProjectCritical( + Credential oauth2, + String projectName, + RawDirectory directoryContents, + RawDirectory oldDirectoryContents + ) throws IOException, ForbiddenException, SnapshotPostException { + Log.info("[{}] Pushing", projectName); + String postbackKey = postbackManager.makeKeyForProject(projectName); + Log.info( + "[{}] Created postback key: {}", + projectName, + postbackKey + ); + try ( + CandidateSnapshot candidate = createCandidateSnapshot( + projectName, + directoryContents, + oldDirectoryContents + ); + ) { Log.info( "[{}] Candindate snapshot created: {}", projectName, @@ -195,6 +276,10 @@ public class Bridge { projectName, versionID ); + dbStore.setLastAccessedTime( + projectName, + Timestamp.valueOf(LocalDateTime.now()) + ); } else { Log.warn( "[{}] Went out of date while waiting for push", @@ -202,31 +287,6 @@ public class Bridge { ); throw new OutOfDateException(); } - } catch (SevereSnapshotPostException e) { - Log.warn("[" + projectName + "] Failed to put to Overleaf", e); - throw e; - } catch (SnapshotPostException e) { - /* Stack trace should be printed further up */ - Log.warn( - "[{}] Exception when waiting for postback: {}", - projectName, - e.getClass().getSimpleName() - ); - throw e; - } catch (IOException e) { - Log.warn("[{}] IOException on put", projectName); - throw e; - } finally { - if (candidate != null) { - candidate.deleteServletFiles(); - } else { - Log.error( - "[{}] Candidate snapshot was null: " + - "this should never happen.", - projectName - ); - } - lock.unlockForProject(projectName); } } @@ -266,7 +326,7 @@ public class Bridge { /* PRIVATE */ - private void updateProjectWithName( + private void updateProject( Credential oauth2, ProjectRepo repo ) throws IOException, GitUserException { diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/GitProjectRepo.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/GitProjectRepo.java index aa6aefeaa8..941d67a316 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/GitProjectRepo.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/GitProjectRepo.java @@ -1,33 +1,36 @@ package uk.ac.ic.wlgitbridge.bridge; +import com.google.common.base.Preconditions; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.storage.file.FileRepositoryBuilder; +import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents; import uk.ac.ic.wlgitbridge.data.filestore.RawFile; import uk.ac.ic.wlgitbridge.git.exception.GitUserException; import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker; import uk.ac.ic.wlgitbridge.util.Log; +import uk.ac.ic.wlgitbridge.util.Project; import uk.ac.ic.wlgitbridge.util.Util; +import java.io.File; import java.io.IOException; -import java.util.Collection; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; +import java.util.*; /** * Created by winston on 20/08/2016. */ public class GitProjectRepo implements ProjectRepo { - private final Repository repository; private final String projectName; + private Optional repository; - public GitProjectRepo(Repository repository, String projectName) { - this.repository = repository; + public GitProjectRepo(String projectName) { + Preconditions.checkArgument(Project.isValidProjectName(projectName)); this.projectName = projectName; + repository = Optional.empty(); } @Override @@ -35,11 +38,34 @@ public class GitProjectRepo implements ProjectRepo { return projectName; } + @Override + public void initRepo( + RepoStore repoStore + ) throws IOException { + initRepositoryField(repoStore); + Preconditions.checkState(repository.isPresent()); + Repository repo = this.repository.get(); + Preconditions.checkState(!repo.getObjectDatabase().exists()); + repo.create(); + } + + @Override + public void useExistingRepository( + RepoStore repoStore + ) throws IOException { + initRepositoryField(repoStore); + Preconditions.checkState(repository.isPresent()); + Preconditions.checkState( + repository.get().getObjectDatabase().exists() + ); + } + @Override public Map getFiles() throws IOException, GitUserException { + Preconditions.checkState(repository.isPresent()); return new RepositoryObjectTreeWalker( - repository + repository.get() ).getDirectoryContents().getFileTable(); } @@ -54,13 +80,33 @@ public class GitProjectRepo implements ProjectRepo { } } + public Repository getJGitRepository() { + return repository.get(); + } + + private void initRepositoryField(RepoStore repoStore) throws IOException { + Preconditions.checkNotNull(repoStore); + Preconditions.checkArgument(Project.isValidProjectName(projectName)); + Preconditions.checkState(!repository.isPresent()); + repository = Optional.of(createJGitRepository(repoStore, projectName)); + } + + private Repository createJGitRepository( + RepoStore repoStore, + String projName + ) throws IOException { + File repoDir = new File(repoStore.getRootDirectory(), projName); + return new FileRepositoryBuilder().setWorkTree(repoDir).build(); + } + private Collection doCommitAndGetMissing( GitDirectoryContents contents ) throws IOException, GitAPIException { + Preconditions.checkState(repository.isPresent()); String name = getProjectName(); Log.info("[{}] Writing commit", name); contents.write(); - Git git = new Git(repository); + Git git = new Git(repository.get()); Log.info("[{}] Getting missing files", name); Set missingFiles = git.status().call().getMissing(); for (String missing : missingFiles) { diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/ProjectRepo.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/ProjectRepo.java index 068ad76bbb..259cac227e 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/ProjectRepo.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/ProjectRepo.java @@ -1,5 +1,6 @@ package uk.ac.ic.wlgitbridge.bridge; +import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents; import uk.ac.ic.wlgitbridge.data.filestore.RawFile; import uk.ac.ic.wlgitbridge.git.exception.GitUserException; @@ -15,6 +16,14 @@ public interface ProjectRepo { String getProjectName(); + void initRepo( + RepoStore repoStore + ) throws IOException; + + void useExistingRepository( + RepoStore repoStore + ) throws IOException; + Map getFiles() throws IOException, GitUserException; Collection commitAndGetMissing( diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/WLBridgedProject.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/WLBridgedProject.java deleted file mode 100644 index 7040d11373..0000000000 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/WLBridgedProject.java +++ /dev/null @@ -1,73 +0,0 @@ -package uk.ac.ic.wlgitbridge.bridge; - -import com.google.api.client.auth.oauth2.Credential; -import org.eclipse.jgit.errors.RepositoryNotFoundException; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.transport.ServiceMayNotContinueException; -import uk.ac.ic.wlgitbridge.git.exception.GitUserException; -import uk.ac.ic.wlgitbridge.snapshot.getdoc.exception.InvalidProjectException; - -import java.io.IOException; - -/** - * Created by Winston on 05/11/14. - */ -public class WLBridgedProject { - - private final Repository repository; - private final String name; - private final Bridge bridgeAPI; - - public WLBridgedProject(Repository repository, String name, Bridge bridgeAPI) { - this.repository = repository; - this.name = name; - this.bridgeAPI = bridgeAPI; - } - - public void buildRepository(Credential oauth2) throws RepositoryNotFoundException, ServiceMayNotContinueException, GitUserException { - bridgeAPI.lockForProject(name); - try { - if (repository.getObjectDatabase().exists()) { - updateRepositoryFromSnapshots(oauth2, repository); - } else { - buildRepositoryFromScratch(oauth2, repository); - } - } catch (RuntimeException e) { - e.printStackTrace(); - throw new ServiceMayNotContinueException(e); - } finally { - bridgeAPI.unlockForProject(name); - } - } - - private void updateRepositoryFromSnapshots( - Credential oauth2, - Repository repository - ) throws RepositoryNotFoundException, - ServiceMayNotContinueException, - GitUserException { - try { - bridgeAPI.getWritableRepositories( - oauth2, - new GitProjectRepo(repository, name) - ); - } catch (InvalidProjectException e) { - throw new RepositoryNotFoundException(name); - } catch (IOException e) { - throw new ServiceMayNotContinueException(e); - } - } - - private void buildRepositoryFromScratch(Credential oauth2, Repository repository) throws RepositoryNotFoundException, ServiceMayNotContinueException, GitUserException { - if (!bridgeAPI.repositoryExists(oauth2, name)) { - throw new RepositoryNotFoundException(name); - } - try { - repository.create(); - } catch (IOException e) { - throw new ServiceMayNotContinueException(e); - } - updateRepositoryFromSnapshots(oauth2, repository); - } - -} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBInitException.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBInitException.java new file mode 100644 index 0000000000..0c8a6d7b04 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBInitException.java @@ -0,0 +1,20 @@ +package uk.ac.ic.wlgitbridge.bridge.db; + +/** + * Created by winston on 23/08/2016. + */ +public class DBInitException extends RuntimeException { + + public DBInitException(String message) { + super(message); + } + + public DBInitException(String message, Throwable cause) { + super(message, cause); + } + + public DBInitException(Throwable cause) { + super(cause); + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java index 33075444cf..0a13b26844 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java @@ -1,5 +1,6 @@ package uk.ac.ic.wlgitbridge.bridge.db; +import java.sql.Timestamp; import java.util.List; /** @@ -7,6 +8,8 @@ import java.util.List; */ public interface DBStore { + int getNumProjects(); + List getProjectNames(); void setLatestVersionForProject(String project, int versionID); @@ -19,4 +22,17 @@ public interface DBStore { String getPathForURLInProject(String projectName, String url); + String getOldestUnswappedProject(); + + int getNumUnswappedProjects(); + + ProjectState getProjectState(String projectName); + + /** + * Sets the last accessed time for the given project name. + * @param projectName the project's name + * @param time the time, or null if the project is to be swapped + */ + void setLastAccessedTime(String projectName, Timestamp time); + } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/ProjectState.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/ProjectState.java new file mode 100644 index 0000000000..ad389f5ac1 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/ProjectState.java @@ -0,0 +1,12 @@ +package uk.ac.ic.wlgitbridge.bridge.db; + +/** + * Created by winston on 24/08/2016. + */ +public enum ProjectState { + + NOT_PRESENT, + PRESENT, + SWAPPED + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/SqliteDBStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/SqliteDBStore.java deleted file mode 100644 index 4838479b8d..0000000000 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/SqliteDBStore.java +++ /dev/null @@ -1,87 +0,0 @@ -package uk.ac.ic.wlgitbridge.bridge.db; - -import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLiteWLDatabase; -import uk.ac.ic.wlgitbridge.util.Log; - -import java.io.File; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.List; - -/** - * Created by winston on 20/08/2016. - */ -public class SqliteDBStore implements DBStore { - - private final SQLiteWLDatabase database; - - public SqliteDBStore(File rootDirectory) { - try { - database = new SQLiteWLDatabase(rootDirectory); - } catch (SQLException e) { - throw new RuntimeException(e); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } - } - - @Override - public List getProjectNames() { - try { - return database.getProjectNames(); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - @Override - public void setLatestVersionForProject(String project, int versionID) { - try { - database.setVersionIDForProject(project, versionID); - Log.info("[{}] Wrote latest versionId: {}", project, versionID); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - @Override - public int getLatestVersionForProject(String project) { - try { - return database.getVersionIDForProjectName(project); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - @Override - public void addURLIndexForProject(String projectName, String url, String path) { - try { - database.addURLIndex(projectName, url, path); - Log.info("[{}] Wrote url index: {} -> {}", projectName, url, path); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - @Override - public void deleteFilesForProject(String project, String... files) { - try { - database.deleteFilesForProject(project, files); - Log.info( - "[{}] Deleting from url index: {}", - project, - Arrays.toString(files) - ); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - @Override - public String getPathForURLInProject(String projectName, String url) { - try { - return database.getPathForURLInProject(projectName, url); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - -} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/SQLQuery.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SQLQuery.java similarity index 82% rename from services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/SQLQuery.java rename to services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SQLQuery.java index df0c2869d4..0873d133b8 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/SQLQuery.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SQLQuery.java @@ -1,4 +1,4 @@ -package uk.ac.ic.wlgitbridge.data.model.db.sql; +package uk.ac.ic.wlgitbridge.bridge.db.sqlite; import java.sql.ResultSet; import java.sql.SQLException; diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SQLUpdate.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SQLUpdate.java new file mode 100644 index 0000000000..167be212c0 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SQLUpdate.java @@ -0,0 +1,18 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite; + +import java.sql.PreparedStatement; +import java.sql.SQLException; + +/** + * Created by Winston on 20/11/14. + */ +public interface SQLUpdate { + + String getSQL(); + default void addParametersToStatement( + PreparedStatement statement + ) throws SQLException { + + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java new file mode 100644 index 0000000000..8c20356997 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java @@ -0,0 +1,204 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite; + +import com.google.common.base.Preconditions; +import uk.ac.ic.wlgitbridge.bridge.db.DBInitException; +import uk.ac.ic.wlgitbridge.bridge.db.DBStore; +import uk.ac.ic.wlgitbridge.bridge.db.ProjectState; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.query.*; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter.ProjectsAddLastAccessed; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create.CreateIndexURLIndexStore; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create.CreateProjectsIndexLastAccessed; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create.CreateProjectsTableSQLUpdate; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create.CreateURLIndexStoreSQLUpdate; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete.DeleteFilesForProjectSQLUpdate; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert.AddURLIndexSQLUpdate; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert.SetProjectLastAccessedTime; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert.SetProjectSQLUpdate; + +import java.io.File; +import java.sql.*; +import java.util.List; +import java.util.stream.Stream; + +/** + * Created by Winston on 17/11/14. + */ +public class SqliteDBStore implements DBStore { + + private final Connection connection; + + public SqliteDBStore(File dbFile) { + try { + connection = openConnectionTo(dbFile); + createTables(); + } catch (Throwable t) { + throw new DBInitException(t); + } + } + + @Override + public int getNumProjects() { + return query(new GetNumProjects()); + } + + @Override + public List getProjectNames() { + return query(new GetProjectNamesSQLQuery()); + } + + @Override + public void setLatestVersionForProject( + String projectName, + int versionID + ) { + update(new SetProjectSQLUpdate(projectName, versionID)); + } + + @Override + public int getLatestVersionForProject( + String projectName + ) { + return query(new GetLatestVersionForProjectSQLQuery(projectName)); + } + + @Override + public void addURLIndexForProject( + String projectName, + String url, + String path + ) { + update(new AddURLIndexSQLUpdate(projectName, url, path)); + } + + @Override + public void deleteFilesForProject( + String projectName, + String... paths + ) { + update(new DeleteFilesForProjectSQLUpdate(projectName, paths)); + } + + @Override + public String getPathForURLInProject( + String projectName, + String url + ) { + return query(new GetPathForURLInProjectSQLQuery(projectName, url)); + } + + @Override + public String getOldestUnswappedProject() { + return query(new GetOldestProjectName()); + } + + @Override + public int getNumUnswappedProjects() { + return query(new GetNumUnswappedProjects()); + } + + @Override + public ProjectState getProjectState(String projectName) { + return query(new GetProjectState(projectName)); + } + + @Override + public void setLastAccessedTime( + String projectName, + Timestamp lastAccessed + ) { + update(new SetProjectLastAccessedTime(projectName, lastAccessed)); + } + + private Connection openConnectionTo(File dbFile) { + File parentDir = dbFile.getParentFile(); + if (!parentDir.exists() && !parentDir.mkdirs()) { + throw new DBInitException( + parentDir.getAbsolutePath() + " directory didn't exist, " + + "and unable to create. Check your permissions." + ); + } + try { + Class.forName("org.sqlite.JDBC"); + } catch (ClassNotFoundException e) { + throw new DBInitException(e); + } + try { + return DriverManager.getConnection( + "jdbc:sqlite:" + dbFile.getAbsolutePath() + ); + } catch (SQLException e) { + throw new DBInitException("Unable to connect to DB", e); + } + } + + private void createTables() { + try { + doUpdate(new ProjectsAddLastAccessed()); + } catch (SQLException ignore) { + /* We need to eat exceptions from here */ + } + Stream.of( + new CreateProjectsTableSQLUpdate(), + new CreateProjectsIndexLastAccessed(), + new CreateURLIndexStoreSQLUpdate(), + new CreateIndexURLIndexStore() + ).forEach(this::update); + /* In the case of needing to change the schema, we need to check that + ProjectsAddLastAccessed didn't just fail */ + Preconditions.checkState(query(new LastAccessedColumnExists())); + } + + private void update(SQLUpdate update) { + try { + doUpdate(update); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + private T query(SQLQuery query) { + try { + return doQuery(query); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + private void doUpdate(SQLUpdate update) throws SQLException { + PreparedStatement statement = null; + try { + statement = connection.prepareStatement(update.getSQL()); + update.addParametersToStatement(statement); + statement.executeUpdate(); + } catch (SQLException e) { + throw e; + } finally { + try { + statement.close(); + } catch (Throwable t) { + throw new SQLException(t); + } + } + } + + private T doQuery(SQLQuery query) throws SQLException { + PreparedStatement statement = null; + ResultSet results = null; + try { + statement = connection.prepareStatement(query.getSQL()); + query.addParametersToStatement(statement); + results = statement.executeQuery(); + return query.processResultSet(results); + } catch (SQLException e) { + throw e; + } finally { + if (statement != null) { + statement.close(); + } + if (results != null) { + results.close(); + } + } + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/query/GetLatestVersionForProjectSQLQuery.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetLatestVersionForProjectSQLQuery.java similarity index 90% rename from services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/query/GetLatestVersionForProjectSQLQuery.java rename to services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetLatestVersionForProjectSQLQuery.java index 49c1a7d447..42802af1ae 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/query/GetLatestVersionForProjectSQLQuery.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetLatestVersionForProjectSQLQuery.java @@ -1,6 +1,6 @@ -package uk.ac.ic.wlgitbridge.data.model.db.sql.query; +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query; -import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLQuery; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery; import java.sql.PreparedStatement; import java.sql.ResultSet; diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumProjects.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumProjects.java new file mode 100644 index 0000000000..e5123c2ea1 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumProjects.java @@ -0,0 +1,30 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery; + +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Created by winston on 24/08/2016. + */ +public class GetNumProjects implements SQLQuery { + + private static final String GET_NUM_PROJECTS = + "SELECT COUNT(*)\n" + + " FROM `projects`"; + + @Override + public String getSQL() { + return GET_NUM_PROJECTS; + } + + @Override + public Integer processResultSet(ResultSet resultSet) throws SQLException { + while (resultSet.next()) { + return resultSet.getInt("COUNT(*)"); + } + throw new IllegalStateException("Count always returns results"); + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumUnswappedProjects.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumUnswappedProjects.java new file mode 100644 index 0000000000..ec7a89b980 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumUnswappedProjects.java @@ -0,0 +1,31 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery; + +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Created by winston on 24/08/2016. + */ +public class GetNumUnswappedProjects implements SQLQuery { + + private static final String GET_NUM_UNSWAPPED_PROJECTS = + "SELECT COUNT(*)\n" + + " FROM `projects`\n" + + " WHERE `last_accessed` IS NOT NULL"; + + @Override + public String getSQL() { + return GET_NUM_UNSWAPPED_PROJECTS; + } + + @Override + public Integer processResultSet(ResultSet resultSet) throws SQLException { + while (resultSet.next()) { + return resultSet.getInt("COUNT(*)"); + } + throw new IllegalStateException("Count always returns results"); + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetOldestProjectName.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetOldestProjectName.java new file mode 100644 index 0000000000..73507f2d73 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetOldestProjectName.java @@ -0,0 +1,30 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery; + +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Created by winston on 23/08/2016. + */ +public class GetOldestProjectName implements SQLQuery { + + private static final String GET_OLDEST_PROJECT_NAME = + "SELECT `name`, MIN(`last_accessed`)\n" + + " FROM `projects`"; + + @Override + public String getSQL() { + return GET_OLDEST_PROJECT_NAME; + } + + @Override + public String processResultSet(ResultSet resultSet) throws SQLException { + while (resultSet.next()) { + return resultSet.getString("name"); + } + return null; + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/query/GetPathForURLInProjectSQLQuery.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetPathForURLInProjectSQLQuery.java similarity index 91% rename from services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/query/GetPathForURLInProjectSQLQuery.java rename to services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetPathForURLInProjectSQLQuery.java index a449ecc8a5..b88ae81f4d 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/query/GetPathForURLInProjectSQLQuery.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetPathForURLInProjectSQLQuery.java @@ -1,6 +1,6 @@ -package uk.ac.ic.wlgitbridge.data.model.db.sql.query; +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query; -import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLQuery; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery; import java.sql.PreparedStatement; import java.sql.ResultSet; diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/query/GetProjectNamesSQLQuery.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetProjectNamesSQLQuery.java similarity index 74% rename from services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/query/GetProjectNamesSQLQuery.java rename to services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetProjectNamesSQLQuery.java index 0985c3402a..902b0244ea 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/query/GetProjectNamesSQLQuery.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetProjectNamesSQLQuery.java @@ -1,8 +1,7 @@ -package uk.ac.ic.wlgitbridge.data.model.db.sql.query; +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query; -import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLQuery; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery; -import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.LinkedList; @@ -30,9 +29,4 @@ public class GetProjectNamesSQLQuery implements SQLQuery> { return GET_URL_INDEXES_FOR_PROJECT_NAME; } - @Override - public void addParametersToStatement(PreparedStatement statement) throws SQLException { - - } - } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetProjectState.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetProjectState.java new file mode 100644 index 0000000000..d253f2fad0 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetProjectState.java @@ -0,0 +1,47 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query; + +import uk.ac.ic.wlgitbridge.bridge.db.ProjectState; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Created by winston on 24/08/2016. + */ +public class GetProjectState implements SQLQuery { + + private static final String GET_PROJECT_STATE = + "SELECT `last_accessed`\n" + + " FROM `projects`\n" + + " WHERE `name` = ?"; + + private final String projectName; + + public GetProjectState(String projectName) { + this.projectName = projectName; + } + + @Override + public String getSQL() { + return GET_PROJECT_STATE; + } + + @Override + public ProjectState processResultSet(ResultSet resultSet) throws SQLException { + while (resultSet.next()) { + if (resultSet.getTimestamp("last_accessed") == null) { + return ProjectState.SWAPPED; + } + return ProjectState.PRESENT; + } + return ProjectState.NOT_PRESENT; + } + + @Override + public void addParametersToStatement(PreparedStatement statement) throws SQLException { + statement.setString(1, projectName); + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/LastAccessedColumnExists.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/LastAccessedColumnExists.java new file mode 100644 index 0000000000..da3b525ff3 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/LastAccessedColumnExists.java @@ -0,0 +1,31 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery; + +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Created by winston on 04/09/2016. + */ +public class LastAccessedColumnExists implements SQLQuery { + + private static final String LAST_ACCESSED_COLUMN_EXISTS = + "PRAGMA table_info(`projects`)"; + + @Override + public String getSQL() { + return LAST_ACCESSED_COLUMN_EXISTS; + } + + @Override + public Boolean processResultSet(ResultSet resultSet) throws SQLException { + while (resultSet.next()) { + if (resultSet.getString(2).equals("last_accessed")) { + return true; + } + } + return false; + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddLastAccessed.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddLastAccessed.java new file mode 100644 index 0000000000..549d4403b2 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddLastAccessed.java @@ -0,0 +1,19 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate; + +/** + * Created by winston on 03/09/2016. + */ +public class ProjectsAddLastAccessed implements SQLUpdate { + + private static final String PROJECTS_ADD_LAST_ACCESSED = + "ALTER TABLE `projects`\n" + + "ADD COLUMN `last_accessed` DATETIME NULL DEFAULT 0"; + + @Override + public String getSQL() { + return PROJECTS_ADD_LAST_ACCESSED; + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateIndexURLIndexStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateIndexURLIndexStore.java new file mode 100644 index 0000000000..9b89199b65 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateIndexURLIndexStore.java @@ -0,0 +1,19 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate; + +/** + * Created by Winston on 21/02/15. + */ +public class CreateIndexURLIndexStore implements SQLUpdate { + + public static final String CREATE_INDEX_URL_INDEX_STORE = + "CREATE UNIQUE INDEX IF NOT EXISTS `project_path_index` " + + "ON `url_index_store`(`project_name`, `path`);\n"; + + @Override + public String getSQL() { + return CREATE_INDEX_URL_INDEX_STORE; + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsIndexLastAccessed.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsIndexLastAccessed.java new file mode 100644 index 0000000000..6e129d8fee --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsIndexLastAccessed.java @@ -0,0 +1,19 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate; + +/** + * Created by winston on 23/08/2016. + */ +public class CreateProjectsIndexLastAccessed implements SQLUpdate { + + private static final String CREATE_PROJECTS_INDEX_LAST_ACCESSED = + "CREATE INDEX IF NOT EXISTS `projects_index_last_accessed`\n" + + " ON `projects`(`last_accessed`)"; + + @Override + public String getSQL() { + return CREATE_PROJECTS_INDEX_LAST_ACCESSED; + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsTableSQLUpdate.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsTableSQLUpdate.java new file mode 100644 index 0000000000..c5f8d3f6e7 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsTableSQLUpdate.java @@ -0,0 +1,23 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate; + +/** + * Created by Winston on 20/11/14. + */ +public class CreateProjectsTableSQLUpdate implements SQLUpdate { + + private static final String CREATE_PROJECTS_TABLE = + "CREATE TABLE IF NOT EXISTS `projects` (\n" + + " `name` VARCHAR NOT NULL DEFAULT '',\n" + + " `version_id` INT NOT NULL DEFAULT 0,\n" + + " `last_accessed` DATETIME NULL DEFAULT 0,\n" + + " PRIMARY KEY (`name`)\n" + + ")"; + + @Override + public String getSQL() { + return CREATE_PROJECTS_TABLE; + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/create/CreateURLIndexStoreSQLUpdate.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateURLIndexStoreSQLUpdate.java similarity index 54% rename from services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/create/CreateURLIndexStoreSQLUpdate.java rename to services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateURLIndexStoreSQLUpdate.java index 0ecf77e6d9..3ca0e9cf4c 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/create/CreateURLIndexStoreSQLUpdate.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateURLIndexStoreSQLUpdate.java @@ -1,9 +1,6 @@ -package uk.ac.ic.wlgitbridge.data.model.db.sql.update.create; +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create; -import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLUpdate; - -import java.sql.PreparedStatement; -import java.sql.SQLException; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate; /** * Created by Winston on 20/11/14. @@ -16,7 +13,11 @@ public class CreateURLIndexStoreSQLUpdate implements SQLUpdate { " `url` text NOT NULL,\n"+ " `path` text NOT NULL,\n"+ " PRIMARY KEY (`project_name`,`url`),\n"+ - " CONSTRAINT `url_index_store_ibfk_1` FOREIGN KEY (`project_name`) REFERENCES `projects` (`name`) ON DELETE CASCADE ON UPDATE CASCADE\n"+ + " CONSTRAINT `url_index_store_ibfk_1` " + + "FOREIGN KEY (`project_name`) " + + "REFERENCES `projects` (`name`) " + + "ON DELETE CASCADE " + + "ON UPDATE CASCADE\n"+ ");\n"; @Override @@ -24,9 +25,4 @@ public class CreateURLIndexStoreSQLUpdate implements SQLUpdate { return CREATE_URL_INDEX_STORE; } - @Override - public void addParametersToStatement(PreparedStatement statement) throws SQLException { - - } - } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/delete/DeleteFilesForProjectSQLUpdate.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteFilesForProjectSQLUpdate.java similarity index 67% rename from services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/delete/DeleteFilesForProjectSQLUpdate.java rename to services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteFilesForProjectSQLUpdate.java index db9ef39f95..78d3e60d72 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/delete/DeleteFilesForProjectSQLUpdate.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteFilesForProjectSQLUpdate.java @@ -1,6 +1,6 @@ -package uk.ac.ic.wlgitbridge.data.model.db.sql.update.delete; +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete; -import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLUpdate; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -11,7 +11,8 @@ import java.sql.SQLException; public class DeleteFilesForProjectSQLUpdate implements SQLUpdate { private static final String DELETE_URL_INDEXES_FOR_PROJECT_NAME = - "DELETE FROM `url_index_store` WHERE `project_name` = ? AND path IN ("; + "DELETE FROM `url_index_store` " + + "WHERE `project_name` = ? AND path IN ("; private final String projectName; private final String[] paths; @@ -23,7 +24,9 @@ public class DeleteFilesForProjectSQLUpdate implements SQLUpdate { @Override public String getSQL() { - StringBuilder sb = new StringBuilder(DELETE_URL_INDEXES_FOR_PROJECT_NAME); + StringBuilder sb = new StringBuilder( + DELETE_URL_INDEXES_FOR_PROJECT_NAME + ); for (int i = 0; i < paths.length; i++) { sb.append("?"); if (i < paths.length - 1) { @@ -35,7 +38,9 @@ public class DeleteFilesForProjectSQLUpdate implements SQLUpdate { } @Override - public void addParametersToStatement(PreparedStatement statement) throws SQLException { + public void addParametersToStatement( + PreparedStatement statement + ) throws SQLException { statement.setString(1, projectName); for (int i = 0; i < paths.length; i++) { statement.setString(i + 2, paths[i]); diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/insert/AddURLIndexSQLUpdate.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/AddURLIndexSQLUpdate.java similarity index 61% rename from services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/insert/AddURLIndexSQLUpdate.java rename to services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/AddURLIndexSQLUpdate.java index 48e6badb0b..5c337bed7a 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/insert/AddURLIndexSQLUpdate.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/AddURLIndexSQLUpdate.java @@ -1,6 +1,6 @@ -package uk.ac.ic.wlgitbridge.data.model.db.sql.update.insert; +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert; -import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLUpdate; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -11,7 +11,12 @@ import java.sql.SQLException; public class AddURLIndexSQLUpdate implements SQLUpdate { private static final String ADD_URL_INDEX = - "INSERT OR REPLACE INTO `url_index_store` (`project_name`, `url`, `path`) VALUES (?, ?, ?);\n"; + "INSERT OR REPLACE INTO `url_index_store`(" + + "`project_name`, " + + "`url`, " + + "`path`" + + ") VALUES " + + "(?, ?, ?)\n"; private final String projectName; private final String url; @@ -29,7 +34,9 @@ public class AddURLIndexSQLUpdate implements SQLUpdate { } @Override - public void addParametersToStatement(PreparedStatement statement) throws SQLException { + public void addParametersToStatement( + PreparedStatement statement + ) throws SQLException { statement.setString(1, projectName); statement.setString(2, url); statement.setString(3, path); diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/SetProjectLastAccessedTime.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/SetProjectLastAccessedTime.java new file mode 100644 index 0000000000..7870822a50 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/SetProjectLastAccessedTime.java @@ -0,0 +1,43 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert; + +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; + +/** + * Created by winston on 23/08/2016. + */ +public class SetProjectLastAccessedTime implements SQLUpdate { + + private static final String SET_PROJECT_LAST_ACCESSED_TIME = + "UPDATE `projects`\n" + + "SET `last_accessed` = ?\n" + + "WHERE `name` = ?"; + + private final String projectName; + private final Timestamp lastAccessed; + + public SetProjectLastAccessedTime( + String projectName, + Timestamp lastAccessed + ) { + this.projectName = projectName; + this.lastAccessed = lastAccessed; + } + + @Override + public String getSQL() { + return SET_PROJECT_LAST_ACCESSED_TIME; + } + + @Override + public void addParametersToStatement( + PreparedStatement statement + ) throws SQLException { + statement.setTimestamp(1, lastAccessed); + statement.setString(2, projectName); + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/insert/SetProjectSQLUpdate.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/SetProjectSQLUpdate.java similarity index 80% rename from services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/insert/SetProjectSQLUpdate.java rename to services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/SetProjectSQLUpdate.java index 4db17c9d1b..f3b11e0bfd 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/insert/SetProjectSQLUpdate.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/SetProjectSQLUpdate.java @@ -1,6 +1,6 @@ -package uk.ac.ic.wlgitbridge.data.model.db.sql.update.insert; +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert; -import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLUpdate; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -11,7 +11,8 @@ import java.sql.SQLException; public class SetProjectSQLUpdate implements SQLUpdate { private static final String SET_PROJECT = - "INSERT OR REPLACE INTO `projects`(`name`, `version_id`) VALUES (?, ?);\n"; + "INSERT OR REPLACE INTO `projects`(`name`, `version_id`, `last_accessed`) " + + "VALUES (?, ?, DATETIME('now'));\n"; private final String projectName; private final int versionID; diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/LockGuard.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/LockGuard.java new file mode 100644 index 0000000000..73c8d253c1 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/LockGuard.java @@ -0,0 +1,10 @@ +package uk.ac.ic.wlgitbridge.bridge.lock; + +/** + * Created by winston on 24/08/2016. + */ +public interface LockGuard extends AutoCloseable { + + void close(); + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/ProjectLock.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/ProjectLock.java index a8157e86b6..624004946f 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/ProjectLock.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/ProjectLock.java @@ -4,9 +4,17 @@ package uk.ac.ic.wlgitbridge.bridge.lock; * Created by winston on 20/08/2016. */ public interface ProjectLock { + void lockAll(); void lockForProject(String projectName); void unlockForProject(String projectName); + + /* RAII hahaha */ + default LockGuard lockGuard(String projectName) { + lockForProject(projectName); + return () -> unlockForProject(projectName); + } + } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java new file mode 100644 index 0000000000..3635fa1836 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java @@ -0,0 +1,112 @@ +package uk.ac.ic.wlgitbridge.bridge.repo; + +import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import org.apache.commons.io.FileUtils; +import uk.ac.ic.wlgitbridge.util.Project; +import uk.ac.ic.wlgitbridge.util.Tar; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static uk.ac.ic.wlgitbridge.util.Util.deleteInDirectoryApartFrom; + +/** + * Created by winston on 20/08/2016. + */ +public class FSGitRepoStore implements RepoStore { + + private final String repoStorePath; + private final File rootDirectory; + + public FSGitRepoStore(String repoStorePath) { + this.repoStorePath = repoStorePath; + rootDirectory = initRootGitDirectory(repoStorePath); + } + + @Override + public String getRepoStorePath() { + return repoStorePath; + } + + @Override + public File getRootDirectory() { + return rootDirectory; + } + + /* TODO: Perhaps we should just delete bad directories on the fly. */ + @Override + public void purgeNonexistentProjects( + Collection existingProjectNames + ) { + List excludedFromDeletion = + new ArrayList<>(existingProjectNames); + excludedFromDeletion.add(".wlgb"); + deleteInDirectoryApartFrom( + rootDirectory, + excludedFromDeletion.toArray(new String[] {}) + ); + } + + @Override + public long totalSize() { + return FileUtils.sizeOfDirectory(rootDirectory); + } + + @Override + public InputStream bzip2Project( + String projectName, + long[] sizePtr + ) throws IOException { + Preconditions.checkArgument(Project.isValidProjectName(projectName)); + return Tar.bz2.zip(getDotGitForProject(projectName), sizePtr); + } + + @Override + public void remove(String projectName) throws IOException { + Preconditions.checkArgument(Project.isValidProjectName(projectName)); + FileUtils.deleteDirectory(new File(rootDirectory, projectName)); + } + + @Override + public void unbzip2Project( + String projectName, + InputStream dataStream + ) throws IOException { + Preconditions.checkArgument(Project.isValidProjectName(projectName)); + Preconditions.checkState(getDirForProject(projectName).mkdirs()); + Tar.bz2.unzip(dataStream, getDirForProject(projectName)); + } + + private File getDirForProject(String projectName) { + Preconditions.checkArgument(Project.isValidProjectName(projectName)); + return Paths.get( + rootDirectory.getAbsolutePath() + ).resolve( + projectName + ).toFile(); + } + + private File getDotGitForProject(String projectName) { + Preconditions.checkArgument(Project.isValidProjectName(projectName)); + return Paths.get( + rootDirectory.getAbsolutePath() + ).resolve( + projectName + ).resolve( + ".git" + ).toFile(); + } + + private File initRootGitDirectory(String rootGitDirectoryPath) { + File rootGitDirectory = new File(rootGitDirectoryPath); + rootGitDirectory.mkdirs(); + Preconditions.checkArgument(rootGitDirectory.isDirectory()); + return rootGitDirectory; + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java deleted file mode 100644 index 8f431fce07..0000000000 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java +++ /dev/null @@ -1,52 +0,0 @@ -package uk.ac.ic.wlgitbridge.bridge.repo; - -import uk.ac.ic.wlgitbridge.util.Util; - -import java.io.File; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -/** - * Created by winston on 20/08/2016. - */ -public class FSRepoStore implements RepoStore { - - private final String repoStorePath; - private final File rootDirectory; - - public FSRepoStore(String repoStorePath) { - this.repoStorePath = repoStorePath; - rootDirectory = initRootGitDirectory(repoStorePath); - } - - @Override - public String getRepoStorePath() { - return repoStorePath; - } - - @Override - public File getRootDirectory() { - return rootDirectory; - } - - @Override - public void purgeNonexistentProjects( - Collection existingProjectNames - ) { - List excludedFromDeletion = - new ArrayList<>(existingProjectNames); - excludedFromDeletion.add(".wlgb"); - Util.deleteInDirectoryApartFrom( - rootDirectory, - excludedFromDeletion.toArray(new String[] {}) - ); - } - - private File initRootGitDirectory(String rootGitDirectoryPath) { - File rootGitDirectory = new File(rootGitDirectoryPath); - rootGitDirectory.mkdirs(); - return rootGitDirectory; - } - -} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java index 9d9d6482d7..bdb95ac828 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java @@ -1,8 +1,8 @@ package uk.ac.ic.wlgitbridge.bridge.repo; -import uk.ac.ic.wlgitbridge.bridge.db.DBStore; - import java.io.File; +import java.io.IOException; +import java.io.InputStream; import java.util.Collection; /** @@ -10,13 +10,47 @@ import java.util.Collection; */ public interface RepoStore { + /* Still need to get rid of these two methods. + Main dependency: GitRepoStore needs a Repository which needs a directory. + Instead, use a visitor or something. */ String getRepoStorePath(); File getRootDirectory(); - void purgeNonexistentProjects( Collection existingProjectNames ); + long totalSize(); + + /** + * Tars and bzip2s the .git directory of the given project. Throws an + * IOException if the project doesn't exist. The returned stream is a copy + * of the original .git directory, which must be deleted using remove(). + */ + InputStream bzip2Project( + String projectName, + long[] sizePtr + ) throws IOException; + + default InputStream bzip2Project( + String projectName + ) throws IOException { + return bzip2Project(projectName, null); + } + + void remove(String projectName) throws IOException; + + /** + * Unbzip2s the given data stream into a .git directory for projectName. + * Creates the project directory. + * If projectName already exists, throws an IOException. + * @param projectName the name of the project, e.g. abc123 + * @param dataStream the data stream containing the bzipped contents. + */ + void unbzip2Project( + String projectName, + InputStream dataStream + ) throws IOException; + } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java deleted file mode 100644 index 04a52e4108..0000000000 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java +++ /dev/null @@ -1,12 +0,0 @@ -package uk.ac.ic.wlgitbridge.bridge.swap; - -/** - * Created by winston on 20/08/2016. - */ -public interface SwapJob { - - void start(int intervalMillis); - - void stop(); - -} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java deleted file mode 100644 index ea26962dd5..0000000000 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java +++ /dev/null @@ -1,53 +0,0 @@ -package uk.ac.ic.wlgitbridge.bridge.swap; - -import uk.ac.ic.wlgitbridge.bridge.db.DBStore; -import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock; -import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; -import uk.ac.ic.wlgitbridge.util.Util; - -import java.util.Timer; - -/** - * Created by winston on 20/08/2016. - */ -public class SwapJobImpl implements SwapJob { - - private final ProjectLock lock; - private final RepoStore repoStore; - private final SwapStore swapStore; - private final DBStore dbStore; - - private final Timer timer; - - public SwapJobImpl( - ProjectLock lock, - RepoStore repoStore, - DBStore dbStore, SwapStore swapStore - ) { - - this.lock = lock; - this.repoStore = repoStore; - this.swapStore = swapStore; - this.dbStore = dbStore; - timer = new Timer(); - } - - @Override - public void start(int intervalMillis) { - timer.scheduleAtFixedRate( - Util.makeTimerTask(this::doSwap), - 0, - intervalMillis - ); - } - - @Override - public void stop() { - timer.cancel(); - } - - private void doSwap() { - throw new UnsupportedOperationException(); - } - -} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java deleted file mode 100644 index d230de00ad..0000000000 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java +++ /dev/null @@ -1,7 +0,0 @@ -package uk.ac.ic.wlgitbridge.bridge.swap; - -/** - * Created by winston on 20/08/2016. - */ -public interface SwapStore { -} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/NoopSwapJob.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/NoopSwapJob.java new file mode 100644 index 0000000000..0825e80dc4 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/NoopSwapJob.java @@ -0,0 +1,30 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.job; + +import java.io.IOException; + +/** + * Created by winston on 24/08/2016. + */ +public class NoopSwapJob implements SwapJob { + + @Override + public void start() { + + } + + @Override + public void stop() { + + } + + @Override + public void evict(String projName) throws IOException { + + } + + @Override + public void restore(String projName) throws IOException { + + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java new file mode 100644 index 0000000000..1147488ff7 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java @@ -0,0 +1,42 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.job; + +import uk.ac.ic.wlgitbridge.bridge.db.DBStore; +import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock; +import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; +import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore; + +import java.io.IOException; +import java.util.Optional; + +/** + * Created by winston on 20/08/2016. + */ +public interface SwapJob { + + static SwapJob fromConfig( + Optional cfg, + ProjectLock lock, + RepoStore repoStore, + DBStore dbStore, + SwapStore swapStore + ) { + if (cfg.isPresent()) { + return new SwapJobImpl( + cfg.get(), + lock, + repoStore, + dbStore, + swapStore + ); + } + return new NoopSwapJob(); + } + + void start(); + + void stop(); + + void evict(String projName) throws IOException; + + void restore(String projName) throws IOException; +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java new file mode 100644 index 0000000000..057cb8148e --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java @@ -0,0 +1,41 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.job; + +/** + * Created by winston on 23/08/2016. + */ +public class SwapJobConfig { + + private final int minProjects; + private final int lowGiB; + private final int highGiB; + private final long intervalMillis; + + public SwapJobConfig( + int minProjects, + int lowGiB, + int highGiB, + long intervalMillis + ) { + this.minProjects = minProjects; + this.lowGiB = lowGiB; + this.highGiB = highGiB; + this.intervalMillis = intervalMillis; + } + + public int getMinProjects() { + return minProjects; + } + + public int getLowGiB() { + return lowGiB; + } + + public int getHighGiB() { + return highGiB; + } + + public long getIntervalMillis() { + return intervalMillis; + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java new file mode 100644 index 0000000000..fa919f7d5a --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java @@ -0,0 +1,175 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.job; + +import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import uk.ac.ic.wlgitbridge.bridge.db.DBStore; +import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard; +import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock; +import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; +import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore; +import uk.ac.ic.wlgitbridge.util.Log; + +import java.io.IOException; +import java.io.InputStream; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.Timer; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Created by winston on 20/08/2016. + */ +public class SwapJobImpl implements SwapJob { + + private static final long GiB = (1l << 30); + + int minProjects; + long lowWatermarkBytes; + long highWatermarkBytes; + Duration interval; + + private final ProjectLock lock; + private final RepoStore repoStore; + private final DBStore dbStore; + private final SwapStore swapStore; + + private final Timer timer; + + final AtomicInteger swaps; + + public SwapJobImpl( + SwapJobConfig cfg, + ProjectLock lock, + RepoStore repoStore, + DBStore dbStore, + SwapStore swapStore + ) { + this( + cfg.getMinProjects(), + GiB * cfg.getLowGiB(), + GiB * cfg.getHighGiB(), + Duration.ofMillis(cfg.getIntervalMillis()), + lock, + repoStore, + dbStore, + swapStore + ); + } + + SwapJobImpl( + int minProjects, + long lowWatermarkBytes, + long highWatermarkBytes, + Duration interval, + ProjectLock lock, + RepoStore repoStore, + DBStore dbStore, + SwapStore swapStore + ) { + this.minProjects = minProjects; + this.lowWatermarkBytes = lowWatermarkBytes; + this.highWatermarkBytes = highWatermarkBytes; + this.interval = interval; + this.lock = lock; + this.repoStore = repoStore; + this.dbStore = dbStore; + this.swapStore = swapStore; + timer = new Timer(); + swaps = new AtomicInteger(0); + } + + @Override + public void start() { + timer.schedule( + uk.ac.ic.wlgitbridge.util.Timer.makeTimerTask(this::doSwap), + 0 + ); + } + + @Override + public void stop() { + timer.cancel(); + } + + private void doSwap() { + try { + doSwap_(); + } catch (Throwable t) { + Log.warn("Exception thrown during swap job", t); + } + timer.schedule( + uk.ac.ic.wlgitbridge.util.Timer.makeTimerTask(this::doSwap), + interval.toMillis() + ); + } + + private void doSwap_() { + Log.info("Running swap number {}", swaps.get() + 1); + long totalSize = repoStore.totalSize(); + Log.info("Size is {}/{} (high)", totalSize, highWatermarkBytes); + if (totalSize < highWatermarkBytes) { + Log.info("No need to swap."); + swaps.incrementAndGet(); + return; + } + int numProjects = dbStore.getNumProjects(); + while ( + (totalSize = repoStore.totalSize()) > lowWatermarkBytes && + (numProjects = dbStore.getNumUnswappedProjects()) > minProjects + ) { + try { + evict(dbStore.getOldestUnswappedProject()); + } catch (IOException e) { + Log.warn("Exception while swapping, giving up", e); + } + } + if (totalSize > lowWatermarkBytes) { + Log.warn( + "Finished swapping, but total size is still too high." + ); + } + Log.info( + "Size: {}/{} (low), " + + "{} (high), " + + "projects on disk: {}/{}, " + + "min projects on disk: {}", + totalSize, + lowWatermarkBytes, + highWatermarkBytes, + numProjects, + dbStore.getNumProjects(), + minProjects + ); + swaps.incrementAndGet(); + } + + @Override + public void evict(String projName) throws IOException { + Preconditions.checkNotNull(projName); + Log.info("Evicting project: {}", projName); + try (LockGuard __ = lock.lockGuard(projName)) { + long[] sizePtr = new long[1]; + InputStream bzipped = repoStore.bzip2Project(projName, sizePtr); + swapStore.upload(projName, bzipped, sizePtr[0]); + dbStore.setLastAccessedTime(projName, null); + repoStore.remove(projName); + } + Log.info("Evicted project: {}", projName); + } + + @Override + public void restore(String projName) throws IOException { + try (LockGuard __ = lock.lockGuard(projName)) { + repoStore.unbzip2Project( + projName, + swapStore.openDownloadStream(projName) + ); + swapStore.remove(projName); + dbStore.setLastAccessedTime( + projName, + Timestamp.valueOf(LocalDateTime.now()) + ); + } + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStore.java new file mode 100644 index 0000000000..a507e7e829 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStore.java @@ -0,0 +1,54 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.store; + +import org.apache.commons.io.IOUtils; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +/** + * Created by winston on 23/08/2016. + */ +public class InMemorySwapStore implements SwapStore { + + private final Map store; + + public InMemorySwapStore() { + store = new HashMap<>(); + } + + public InMemorySwapStore(SwapStoreConfig __) { + this(); + } + + @Override + public void upload( + String projectName, + InputStream uploadStream, + long contentLength + ) throws IOException { + store.put( + projectName, + IOUtils.toByteArray(uploadStream, contentLength) + ); + } + + @Override + public InputStream openDownloadStream(String projectName) { + byte[] buf = store.get(projectName); + if (buf == null) { + throw new IllegalArgumentException( + "no such project in swap store: " + projectName + ); + } + return new ByteArrayInputStream(buf); + } + + @Override + public void remove(String projectName) { + store.remove(projectName); + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/NoopSwapStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/NoopSwapStore.java new file mode 100644 index 0000000000..72d3c38e12 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/NoopSwapStore.java @@ -0,0 +1,35 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.store; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * Created by winston on 24/08/2016. + */ +public class NoopSwapStore implements SwapStore { + + public NoopSwapStore(SwapStoreConfig config) { + + } + + @Override + public void upload( + String projectName, + InputStream uploadStream, + long contentLength + ) throws IOException { + + } + + @Override + public InputStream openDownloadStream(String projectName) { + return new ByteArrayInputStream(new byte[0]); + } + + @Override + public void remove(String projectName) { + + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStore.java new file mode 100644 index 0000000000..ed1e379a7b --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStore.java @@ -0,0 +1,72 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.store; + +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.*; + +import java.io.InputStream; + +/** + * Created by winston on 21/08/2016. + */ +public class S3SwapStore implements SwapStore { + + private final AmazonS3 s3; + + private final String bucketName; + + public S3SwapStore(SwapStoreConfig cfg) { + this( + cfg.getAwsAccessKey(), + cfg.getAwsSecret(), + cfg.getS3BucketName() + ); + } + + S3SwapStore( + String accessKey, + String secret, + String bucketName + ) { + s3 = new AmazonS3Client(new BasicAWSCredentials(accessKey, secret)); + this.bucketName = bucketName; + } + + @Override + public void upload( + String projectName, + InputStream uploadStream, + long contentLength + ) { + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentLength(contentLength); + PutObjectRequest put = new PutObjectRequest( + bucketName, + projectName, + uploadStream, + metadata + ); + PutObjectResult res = s3.putObject(put); + } + + @Override + public InputStream openDownloadStream(String projectName) { + GetObjectRequest get = new GetObjectRequest( + bucketName, + projectName + ); + S3Object res = s3.getObject(get); + return res.getObjectContent(); + } + + @Override + public void remove(String projectName) { + DeleteObjectRequest del = new DeleteObjectRequest( + bucketName, + projectName + ); + s3.deleteObject(del); + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStore.java new file mode 100644 index 0000000000..168e959313 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStore.java @@ -0,0 +1,44 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.store; + +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +/** + * Created by winston on 20/08/2016. + */ +public interface SwapStore { + + Map> swapStores = + new HashMap>() { + + { + put("noop", NoopSwapStore::new); + put("memory", InMemorySwapStore::new); + put("s3", S3SwapStore::new); + } + + }; + + static SwapStore fromConfig( + Optional cfg + ) { + SwapStoreConfig cfg_ = cfg.orElse(SwapStoreConfig.NOOP); + String type = cfg_.getType(); + return swapStores.get(type).apply(cfg_); + } + + void upload( + String projectName, + InputStream uploadStream, + long contentLength + ) throws IOException; + + InputStream openDownloadStream(String projectName); + + void remove(String projectName); + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStoreConfig.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStoreConfig.java new file mode 100644 index 0000000000..61219d02ee --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStoreConfig.java @@ -0,0 +1,76 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.store; + +/** + * Created by winston on 24/08/2016. + */ +public class SwapStoreConfig { + + public static final SwapStoreConfig NOOP = new SwapStoreConfig( + "noop", + null, + null, + null + ); + + private String type; + private String awsAccessKey; + private String awsSecret; + private String s3BucketName; + + public SwapStoreConfig() {} + + public SwapStoreConfig( + String awsAccessKey, + String awsSecret, + String s3BucketName + ) { + this( + "s3", + awsAccessKey, + awsSecret, + s3BucketName + ); + } + + SwapStoreConfig( + String type, + String awsAccessKey, + String awsSecret, + String s3BucketName + ) { + this.type = type; + this.awsAccessKey = awsAccessKey; + this.awsSecret = awsSecret; + this.s3BucketName = s3BucketName; + } + + public String getType() { + return type; + } + + public String getAwsAccessKey() { + return awsAccessKey; + } + + public String getAwsSecret() { + return awsSecret; + } + + public String getS3BucketName() { + return s3BucketName; + } + + public SwapStoreConfig sanitisedCopy() { + return new SwapStoreConfig( + type, + awsAccessKey == null ? null : "", + awsSecret == null ? null : "", + s3BucketName + ); + } + + public static SwapStoreConfig sanitisedCopy(SwapStoreConfig swapStore) { + return swapStore == null ? null : swapStore.sanitisedCopy(); + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/CandidateSnapshot.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/CandidateSnapshot.java index f3d99163ed..f27601227b 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/CandidateSnapshot.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/CandidateSnapshot.java @@ -17,7 +17,7 @@ import java.util.Map.Entry; /** * Created by Winston on 16/11/14. */ -public class CandidateSnapshot { +public class CandidateSnapshot implements AutoCloseable { private final String projectName; private final int currentVersion; @@ -25,14 +25,22 @@ public class CandidateSnapshot { private final List deleted; private File attsDirectory; - public CandidateSnapshot(String projectName, int currentVersion, RawDirectory directoryContents, RawDirectory oldDirectoryContents) { + public CandidateSnapshot( + String projectName, + int currentVersion, + RawDirectory directoryContents, + RawDirectory oldDirectoryContents + ) { this.projectName = projectName; this.currentVersion = currentVersion; files = diff(directoryContents, oldDirectoryContents); deleted = deleted(directoryContents, oldDirectoryContents); } - private List diff(RawDirectory directoryContents, RawDirectory oldDirectoryContents) { + private List diff( + RawDirectory directoryContents, + RawDirectory oldDirectoryContents + ) { List files = new LinkedList(); Map fileTable = directoryContents.getFileTable(); Map oldFileTable = oldDirectoryContents.getFileTable(); @@ -43,10 +51,16 @@ public class CandidateSnapshot { return files; } - private List deleted(RawDirectory directoryContents, RawDirectory oldDirectoryContents) { + private List deleted( + RawDirectory directoryContents, + RawDirectory oldDirectoryContents + ) { List deleted = new LinkedList(); Map fileTable = directoryContents.getFileTable(); - for (Entry entry : oldDirectoryContents.getFileTable().entrySet()) { + for ( + Entry entry : + oldDirectoryContents.getFileTable().entrySet() + ) { String path = entry.getKey(); RawFile newFile = fileTable.get(path); if (newFile == null) { @@ -57,7 +71,10 @@ public class CandidateSnapshot { } public void writeServletFiles(File rootGitDirectory) throws IOException { - attsDirectory = new File(rootGitDirectory, ".wlgb/atts/" + projectName); + attsDirectory = new File( + rootGitDirectory, + ".wlgb/atts/" + projectName + ); for (ServletFile file : files) { if (file.isChanged()) { file.writeToDisk(attsDirectory); @@ -90,11 +107,18 @@ public class CandidateSnapshot { return filesArray; } - private JsonObject getFileAsJson(ServletFile file, String projectURL, String postbackKey) { + private JsonObject getFileAsJson( + ServletFile file, + String projectURL, + String postbackKey + ) { JsonObject jsonFile = new JsonObject(); jsonFile.addProperty("name", file.getPath()); if (file.isChanged()) { - jsonFile.addProperty("url", projectURL + "/" + file.getPath() + "?key=" + postbackKey); + jsonFile.addProperty( + "url", + projectURL + "/" + file.getPath() + "?key=" + postbackKey + ); } return jsonFile; } @@ -119,4 +143,9 @@ public class CandidateSnapshot { return sb.toString(); } + @Override + public void close() throws IOException { + deleteServletFiles(); + } + } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ProjectLockImpl.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ProjectLockImpl.java index 803ba89e9f..34342ae1bf 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ProjectLockImpl.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ProjectLockImpl.java @@ -33,11 +33,13 @@ public class ProjectLockImpl implements ProjectLock { setWaiter(waiter); } + @Override public void lockForProject(String projectName) { getLockForProjectName(projectName).lock(); rlock.lock(); } + @Override public void unlockForProject(String projectName) { getLockForProjectName(projectName).unlock(); rlock.unlock(); diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/SnapshotRepositoryBuilder.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/SnapshotRepositoryBuilder.java deleted file mode 100644 index 5b0818cecb..0000000000 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/SnapshotRepositoryBuilder.java +++ /dev/null @@ -1,53 +0,0 @@ -package uk.ac.ic.wlgitbridge.data; - -import com.google.api.client.auth.oauth2.Credential; -import org.eclipse.jgit.errors.RepositoryNotFoundException; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.storage.file.FileRepositoryBuilder; -import org.eclipse.jgit.transport.ServiceMayNotContinueException; -import uk.ac.ic.wlgitbridge.bridge.Bridge; -import uk.ac.ic.wlgitbridge.bridge.WLBridgedProject; -import uk.ac.ic.wlgitbridge.git.exception.GitUserException; -import uk.ac.ic.wlgitbridge.snapshot.push.exception.InternalErrorException; -import uk.ac.ic.wlgitbridge.util.Log; - -import java.io.File; -import java.io.IOException; - -/** - * Created by Winston on 03/11/14. - */ -public class SnapshotRepositoryBuilder { - - private final Bridge bridgeAPI; - - public SnapshotRepositoryBuilder(Bridge bridgeAPI) { - this.bridgeAPI = bridgeAPI; - } - - public Repository getRepositoryWithNameAtRootDirectory(String name, File rootDirectory, Credential oauth2) throws RepositoryNotFoundException, ServiceMayNotContinueException, GitUserException { - if (!bridgeAPI.repositoryExists(oauth2, name)) { - throw new RepositoryNotFoundException(name); - } - File repositoryDirectory = new File(rootDirectory, name); - - Repository repository = null; - try { - repository = new FileRepositoryBuilder().setWorkTree(repositoryDirectory).build(); - new WLBridgedProject(repository, name, bridgeAPI).buildRepository(oauth2); - } catch (IOException e) { - Log.warn( - "IOException when trying to get repo: " + - name + - ", at: " - + rootDirectory.getAbsolutePath(), - e - ); - throw new ServiceMayNotContinueException( - new InternalErrorException().getDescriptionLines().get(0) - ); - } - return repository; - } - -} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/PersistentStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/PersistentStore.java deleted file mode 100644 index 59cf8103c2..0000000000 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/PersistentStore.java +++ /dev/null @@ -1,20 +0,0 @@ -package uk.ac.ic.wlgitbridge.data.model.db; - -import java.util.List; - -/** - * Created by m on 20/11/15. - */ -public interface PersistentStore { - List getProjectNames(); - - void setLatestVersionForProject(String project, int versionID); - - int getLatestVersionForProject(String project); - - void addURLIndexForProject(String projectName, String url, String path); - - void deleteFilesForProject(String project, String... files); - - String getPathForURLInProject(String projectName, String url); -} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/SQLUpdate.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/SQLUpdate.java deleted file mode 100644 index 097e2b1188..0000000000 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/SQLUpdate.java +++ /dev/null @@ -1,14 +0,0 @@ -package uk.ac.ic.wlgitbridge.data.model.db.sql; - -import java.sql.PreparedStatement; -import java.sql.SQLException; - -/** - * Created by Winston on 20/11/14. - */ -public interface SQLUpdate { - - public String getSQL(); - public void addParametersToStatement(PreparedStatement statement) throws SQLException; - -} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/SQLiteWLDatabase.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/SQLiteWLDatabase.java deleted file mode 100644 index af6acee858..0000000000 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/SQLiteWLDatabase.java +++ /dev/null @@ -1,107 +0,0 @@ -package uk.ac.ic.wlgitbridge.data.model.db.sql; - -import uk.ac.ic.wlgitbridge.data.model.db.sql.query.GetLatestVersionForProjectSQLQuery; -import uk.ac.ic.wlgitbridge.data.model.db.sql.query.GetPathForURLInProjectSQLQuery; -import uk.ac.ic.wlgitbridge.data.model.db.sql.query.GetProjectNamesSQLQuery; -import uk.ac.ic.wlgitbridge.data.model.db.sql.update.create.CreateIndexURLIndexStore; -import uk.ac.ic.wlgitbridge.data.model.db.sql.update.create.CreateProjectsTableSQLUpdate; -import uk.ac.ic.wlgitbridge.data.model.db.sql.update.create.CreateURLIndexStoreSQLUpdate; -import uk.ac.ic.wlgitbridge.data.model.db.sql.update.delete.DeleteFilesForProjectSQLUpdate; -import uk.ac.ic.wlgitbridge.data.model.db.sql.update.insert.AddURLIndexSQLUpdate; -import uk.ac.ic.wlgitbridge.data.model.db.sql.update.insert.SetProjectSQLUpdate; -import uk.ac.ic.wlgitbridge.util.Log; - -import java.io.File; -import java.sql.*; -import java.util.List; - -/** - * Created by Winston on 17/11/14. - */ -public class SQLiteWLDatabase { - - private final Connection connection; - - public SQLiteWLDatabase(File rootGitDirectory) throws SQLException, ClassNotFoundException { - File databaseFile = new File(rootGitDirectory, "/.wlgb/wlgb.db"); - File dotWlgbDir = databaseFile.getParentFile(); - if (!dotWlgbDir.exists()) { - if (!dotWlgbDir.mkdirs()) { - Log.error("{} directory didn't exist, and unable to create. Check your permissions", dotWlgbDir.getAbsolutePath()); - } - } - Class.forName("org.sqlite.JDBC"); - connection = DriverManager.getConnection("jdbc:sqlite:" + databaseFile.getAbsolutePath()); - createTables(); - } - - public void setVersionIDForProject(String projectName, int versionID) throws SQLException { - update(new SetProjectSQLUpdate(projectName, versionID)); - } - - public void addURLIndex(String projectName, String url, String path) throws SQLException { - update(new AddURLIndexSQLUpdate(projectName, url, path)); - } - - public void deleteFilesForProject(String projectName, String... paths) throws SQLException { - update(new DeleteFilesForProjectSQLUpdate(projectName, paths)); - } - - public int getVersionIDForProjectName(String projectName) throws SQLException { - return query(new GetLatestVersionForProjectSQLQuery(projectName)); - } - - public String getPathForURLInProject(String projectName, String url) throws SQLException { - return query(new GetPathForURLInProjectSQLQuery(projectName, url)); - } - - public List getProjectNames() throws SQLException { - return query(new GetProjectNamesSQLQuery()); - } - - private void createTables() throws SQLException { - final SQLUpdate[] createTableUpdates = { - new CreateProjectsTableSQLUpdate(), - new CreateURLIndexStoreSQLUpdate(), - new CreateIndexURLIndexStore() - }; - - for (SQLUpdate update : createTableUpdates) { - update(update); - } - } - - private void update(SQLUpdate update) throws SQLException { - PreparedStatement statement = null; - try { - statement = connection.prepareStatement(update.getSQL()); - update.addParametersToStatement(statement); - statement.executeUpdate(); - } catch (SQLException e) { - throw e; - } finally { - statement.close(); - } - } - - private T query(SQLQuery query) throws SQLException { - PreparedStatement statement = null; - ResultSet results = null; - try { - statement = connection.prepareStatement(query.getSQL()); - query.addParametersToStatement(statement); - results = statement.executeQuery(); - return query.processResultSet(results); - } catch (SQLException e) { - throw e; - } finally { - if (statement != null) { - statement.close(); - } - if (results != null) { - results.close(); - } - } - } - -} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/create/CreateIndexURLIndexStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/create/CreateIndexURLIndexStore.java deleted file mode 100644 index 6666722ff6..0000000000 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/create/CreateIndexURLIndexStore.java +++ /dev/null @@ -1,26 +0,0 @@ -package uk.ac.ic.wlgitbridge.data.model.db.sql.update.create; - -import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLUpdate; - -import java.sql.PreparedStatement; -import java.sql.SQLException; - -/** - * Created by Winston on 21/02/15. - */ -public class CreateIndexURLIndexStore implements SQLUpdate { - - public static final String CREATE_INDEX_URL_INDEX_STORE = - "CREATE UNIQUE INDEX IF NOT EXISTS `project_path_index` ON `url_index_store`(`project_name`, `path`);\n"; - - @Override - public String getSQL() { - return CREATE_INDEX_URL_INDEX_STORE; - } - - @Override - public void addParametersToStatement(PreparedStatement statement) throws SQLException { - - } - -} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/create/CreateProjectsTableSQLUpdate.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/create/CreateProjectsTableSQLUpdate.java deleted file mode 100644 index 6ac597148a..0000000000 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/create/CreateProjectsTableSQLUpdate.java +++ /dev/null @@ -1,29 +0,0 @@ -package uk.ac.ic.wlgitbridge.data.model.db.sql.update.create; - -import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLUpdate; - -import java.sql.PreparedStatement; -import java.sql.SQLException; - -/** - * Created by Winston on 20/11/14. - */ -public class CreateProjectsTableSQLUpdate implements SQLUpdate { - - private static final String CREATE_PROJECTS_TABLE = - "CREATE TABLE IF NOT EXISTS `projects` (\n" + - " `name` varchar(10) NOT NULL DEFAULT '',\n" + - " `version_id` int(11) NOT NULL DEFAULT 0,\n" + - " PRIMARY KEY (`name`)\n" + - ")"; - @Override - public String getSQL() { - return CREATE_PROJECTS_TABLE; - } - - @Override - public void addParametersToStatement(PreparedStatement statement) throws SQLException { - - } - -} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java index 542963767d..5ca19ba7d5 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java @@ -19,10 +19,10 @@ import javax.servlet.http.HttpServletRequest; /* */ public class WLReceivePackFactory implements ReceivePackFactory { - private final Bridge bridgeAPI; + private final Bridge bridge; - public WLReceivePackFactory(Bridge bridgeAPI) { - this.bridgeAPI = bridgeAPI; + public WLReceivePackFactory(Bridge bridge) { + this.bridge = bridge; } @Override @@ -33,7 +33,7 @@ public class WLReceivePackFactory implements ReceivePackFactory { +public class WLRepositoryResolver + implements RepositoryResolver { - private File rootGitDirectory; - private SnapshotRepositoryBuilder snapshotRepositoryBuilder; + private final Bridge bridge; - public WLRepositoryResolver(String rootGitDirectoryPath, SnapshotRepositoryBuilder repositorySource) throws InvalidRootDirectoryPathException { - this.snapshotRepositoryBuilder = repositorySource; - initRootGitDirectory(rootGitDirectoryPath); + public WLRepositoryResolver(Bridge bridge) { + this.bridge = bridge; } @Override - public Repository open(HttpServletRequest httpServletRequest, String name) throws RepositoryNotFoundException, ServiceNotAuthorizedException, ServiceNotEnabledException, ServiceMayNotContinueException { - Credential oauth2 = (Credential) httpServletRequest.getAttribute(Oauth2Filter.ATTRIBUTE_KEY); + public Repository open( + HttpServletRequest httpServletRequest, + String name + ) throws RepositoryNotFoundException, + ServiceNotAuthorizedException, + ServiceNotEnabledException, + ServiceMayNotContinueException { + Credential oauth2 = (Credential) httpServletRequest.getAttribute( + Oauth2Filter.ATTRIBUTE_KEY + ); + String projName = Util.removeAllSuffixes(name, "/", ".git"); try { - return snapshotRepositoryBuilder.getRepositoryWithNameAtRootDirectory(Util.removeAllSuffixes(name, "/", ".git"), rootGitDirectory, oauth2); + if (!bridge.projectExists(oauth2, projName)) { + throw new RepositoryNotFoundException(projName); + } + GitProjectRepo repo = new GitProjectRepo(projName); + bridge.updateRepository(oauth2, repo); + return repo.getJGitRepository(); } catch (RepositoryNotFoundException e) { Log.info("Repository not found: " + name); throw e; @@ -45,25 +58,25 @@ public class WLRepositoryResolver implements RepositoryResolver children0 = getChildren(f0, f0Base); + Set children1 = getChildren(f1, f1Base); + if (children0.size() != children1.size()) { + return false; + } + return children0.stream( + ).allMatch(c0 -> + children1.contains(c0) && childEquals(c0, f0Base, f1Base) + ); + } + + private static Set getChildren(File f0, Path f0Base) { + return FileUtils.listFilesAndDirs( + f0, + TrueFileFilter.TRUE, + TrueFileFilter.TRUE + ).stream( + ).map( + File::getAbsolutePath + ).map( + Paths::get + ).map(p -> + f0Base.relativize(p) + ).filter(p -> + !p.toString().isEmpty() + ).collect( + Collectors.toSet() + ); + } + + private static boolean childEquals( + Path child, + Path f0Base, + Path f1Base + ) throws UncheckedIOException { + File c0 = f0Base.resolve(child).toFile(); + File c1 = f1Base.resolve(child).toFile(); + boolean c0IsDir = c0.isDirectory(); + boolean c1IsDir = c1.isDirectory(); + if (c0IsDir || c1IsDir) { + return c0IsDir && c1IsDir; + } + try { + return c0.isFile() && c1.isFile() && Arrays.equals( + FileUtils.readFileToByteArray(c0), + FileUtils.readFileToByteArray(c1) + ); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Project.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Project.java new file mode 100644 index 0000000000..3c3b4239a1 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Project.java @@ -0,0 +1,13 @@ +package uk.ac.ic.wlgitbridge.util; + +/** + * Created by winston on 23/08/2016. + */ +public class Project { + + public static boolean isValidProjectName(String projectName) { + return projectName != null && !projectName.isEmpty() + && !projectName.startsWith("."); + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java new file mode 100644 index 0000000000..7ed57368de --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java @@ -0,0 +1,151 @@ +package uk.ac.ic.wlgitbridge.util; + +import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; +import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.output.ByteArrayOutputStream; + +import java.io.*; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Created by winston on 23/08/2016. + */ +public class Tar { + + public static class bz2 { + + public static InputStream zip( + File fileOrDir + ) throws IOException { + return zip(fileOrDir, null); + } + + public static InputStream zip( + File fileOrDir, + long[] sizePtr + ) throws IOException { + ByteArrayOutputStream target = new ByteArrayOutputStream(); + try (OutputStream bzip2 = new BZip2CompressorOutputStream(target)) { + tarTo(fileOrDir, bzip2); + } + if (sizePtr != null) { + sizePtr[0] = target.size(); + } + return target.toInputStream(); + } + + public static void unzip( + InputStream tarbz2, + File parentDir + ) throws IOException { + try (InputStream tar = new BZip2CompressorInputStream(tarbz2)) { + untar(tar, parentDir); + } + } + + } + + private Tar() {} + + public static InputStream tar(File fileOrDir) throws IOException { + ByteArrayOutputStream target = new ByteArrayOutputStream(); + tarTo(fileOrDir, target); + return target.toInputStream(); + } + + public static void tarTo( + File fileOrDir, + OutputStream target + ) throws IOException { + try (TarArchiveOutputStream tout = new TarArchiveOutputStream(target)) { + addTarEntry( + tout, + Paths.get(fileOrDir.getParentFile().getAbsolutePath()), + fileOrDir + ); + } + } + + public static void untar( + InputStream tar, + File parentDir + ) throws IOException { + TarArchiveInputStream tin = new TarArchiveInputStream(tar); + ArchiveEntry e; + while ((e = tin.getNextEntry()) != null) { + File f = new File(parentDir, e.getName()); + f.setLastModified(e.getLastModifiedDate().getTime()); + f.getParentFile().mkdirs(); + if (e.isDirectory()) { + f.mkdir(); + continue; + } + long size = e.getSize(); + Preconditions.checkArgument( + size > 0 && size < Integer.MAX_VALUE, + "file too big: tarTo should have thrown an IOException" + ); + try (OutputStream out = new FileOutputStream(f)) { + /* TarInputStream pretends each + entry's EOF is the stream's EOF */ + IOUtils.copy(tin, out); + } + } + } + + private static void addTarEntry( + TarArchiveOutputStream tout, + Path base, + File fileOrDir + ) throws IOException { + if (fileOrDir.isDirectory()) { + addTarDir(tout, base, fileOrDir); + } else if (fileOrDir.isFile()) { + addTarFile(tout, base, fileOrDir); + } else { + throw new IllegalArgumentException( + "invalid file or dir: " + fileOrDir + ); + } + } + + private static void addTarDir( + TarArchiveOutputStream tout, + Path base, + File dir + ) throws IOException { + Preconditions.checkArgument(dir.isDirectory()); + String name = base.relativize( + Paths.get(dir.getAbsolutePath()) + ).toString(); + ArchiveEntry entry = tout.createArchiveEntry(dir, name); + tout.putArchiveEntry(entry); + tout.closeArchiveEntry(); + for (File f : dir.listFiles()) { + addTarEntry(tout, base, f); + } + } + + private static void addTarFile( + TarArchiveOutputStream tout, + Path base, + File file + ) throws IOException { + Preconditions.checkArgument(file.isFile()); + String name = base.relativize( + Paths.get(file.getAbsolutePath()) + ).toString(); + ArchiveEntry entry = tout.createArchiveEntry(file, name); + tout.putArchiveEntry(entry); + tout.write(FileUtils.readFileToByteArray(file)); + tout.closeArchiveEntry(); + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Timer.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Timer.java new file mode 100644 index 0000000000..b2dd212707 --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Timer.java @@ -0,0 +1,19 @@ +package uk.ac.ic.wlgitbridge.util; + +import java.util.TimerTask; + +/** + * Created by winston on 23/08/2016. + */ +public class Timer { + + public static TimerTask makeTimerTask(Runnable lamb) { + return new TimerTask() { + @Override + public void run() { + lamb.run(); + } + }; + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Util.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Util.java index 3654561f9f..97440a9ff5 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Util.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Util.java @@ -19,15 +19,6 @@ public class Util { private static String POSTBACK_URL; private static final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSS"); - public static TimerTask makeTimerTask(Runnable lamb) { - return new TimerTask() { - @Override - public void run() { - lamb.run(); - } - }; - } - public static String entries(int entries) { if (entries == 1) { return "entry"; diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java similarity index 88% rename from services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest.java rename to services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java index b253a7b6df..516f99b1a2 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java @@ -1,13 +1,14 @@ -package uk.ac.ic.wlgitbridge; +package uk.ac.ic.wlgitbridge.application; import com.ning.http.client.AsyncHttpClient; import com.ning.http.client.Response; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.eclipse.jgit.api.errors.GitAPIException; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import uk.ac.ic.wlgitbridge.application.GitBridgeApp; +import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig; import uk.ac.ic.wlgitbridge.snapshot.servermock.server.MockSnapshotServer; import uk.ac.ic.wlgitbridge.snapshot.servermock.state.SnapshotAPIState; import uk.ac.ic.wlgitbridge.snapshot.servermock.state.SnapshotAPIStateBuilder; @@ -15,6 +16,7 @@ import uk.ac.ic.wlgitbridge.snapshot.servermock.util.FileUtil; import uk.ac.ic.wlgitbridge.util.Util; import java.io.*; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; @@ -102,6 +104,9 @@ public class WLGitBridgeIntegrationTest { put("canServePushedFiles", new HashMap() {{ put("state", new SnapshotAPIStateBuilder(getResourceAsStream("/canServePushedFiles/state/state.json")).build()); }}); + put("wlgbCanSwapProjects", new HashMap() {{ + put("state", new SnapshotAPIStateBuilder(getResourceAsStream("/wlgbCanSwapProjects/state/state.json")).build()); + }}); }}; @Rule @@ -587,14 +592,56 @@ public class WLGitBridgeIntegrationTest { wlgb.stop(); } + @Test + public void wlgbCanSwapProjects( + ) throws IOException, GitAPIException, InterruptedException { + MockSnapshotServer server = new MockSnapshotServer( + 3874, + getResource("/wlgbCanSwapProjects").toFile() + ); + server.start(); + server.setState(states.get("wlgbCanSwapProjects").get("state")); + GitBridgeApp wlgb = new GitBridgeApp(new String[] { + makeConfigFile(33874, 3874, new SwapJobConfig(1, 0, 0, 250)) + }); + wlgb.run(); + File dir = folder.newFolder(); + File rootGitDir = new File(wlgb.config.getRootGitDirectory()); + File testProj1ServerDir = new File(rootGitDir, "testproj1"); + File testProj2ServerDir = new File(rootGitDir, "testproj2"); + File testProj1Dir = cloneRepository("testproj1", 33874, dir); + assertTrue(testProj1ServerDir.exists()); + assertFalse(testProj2ServerDir.exists()); + cloneRepository("testproj2", 33874, dir); + while (testProj1ServerDir.exists()); + assertFalse(testProj1ServerDir.exists()); + assertTrue(testProj2ServerDir.exists()); + FileUtils.deleteDirectory(testProj1Dir); + cloneRepository("testproj1", 33874, dir); + while (testProj2ServerDir.exists()); + assertTrue(testProj1ServerDir.exists()); + assertFalse(testProj2ServerDir.exists()); + wlgb.stop(); + } + private File cloneRepository(String repositoryName, int port, File dir) throws IOException, InterruptedException { String repo = "git clone http://127.0.0.1:" + port + "/" + repositoryName + ".git"; Process gitProcess = runtime.exec(repo, null, dir); int exitCode = gitProcess.waitFor(); if (exitCode != 0) { System.err.println("git clone failed. Dumping stderr and stdout."); - System.err.println(IOUtils.toString(gitProcess.getErrorStream())); - System.err.println(IOUtils.toString(gitProcess.getInputStream())); + System.err.println( + IOUtils.toString( + gitProcess.getErrorStream(), + StandardCharsets.UTF_8 + ) + ); + System.err.println( + IOUtils.toString( + gitProcess.getInputStream(), + StandardCharsets.UTF_8 + ) + ); fail("git clone failed"); } File repositoryDir = new File(dir, repositoryName); @@ -606,30 +653,75 @@ public class WLGitBridgeIntegrationTest { return repositoryDir; } - private String makeConfigFile(int port, int apiPort) throws IOException { + private String makeConfigFile( + int port, + int apiPort + ) throws IOException { + return makeConfigFile(port, apiPort, null); + } + + private String makeConfigFile( + int port, + int apiPort, + SwapJobConfig swapCfg + ) throws IOException { File wlgb = folder.newFolder(); File config = folder.newFile(); PrintWriter writer = new PrintWriter(config); - writer.println("{\n" + - "\t\"port\": " + port + ",\n" + - "\t\"rootGitDirectory\": \"" + wlgb.getAbsolutePath() + "\",\n" + - "\t\"apiBaseUrl\": \"http://127.0.0.1:" + apiPort + "/api/v0\",\n" + - "\t\"username\": \"\",\n" + - "\t\"password\": \"\",\n" + - "\t\"postbackBaseUrl\": \"http://127.0.0.1:" + port + "\",\n" + - "\t\"serviceName\": \"Overleaf\"\n," + - " \"oauth2\": {\n" + - " \"oauth2ClientID\": \"clientID\",\n" + - " \"oauth2ClientSecret\": \"oauth2 client secret\",\n" + - " \"oauth2Server\": \"https://www.overleaf.com\"\n" + - " }\n" + - "}\n"); + String cfgStr = + "{\n" + + " \"port\": " + port + ",\n" + + " \"rootGitDirectory\": \"" + + wlgb.getAbsolutePath() + + "\",\n" + + " \"apiBaseUrl\": \"http://127.0.0.1:" + + apiPort + + "/api/v0\",\n" + + " \"username\": \"\",\n" + + " \"password\": \"\",\n" + + " \"postbackBaseUrl\": \"http://127.0.0.1:" + + port + + "\",\n" + + " \"serviceName\": \"Overleaf\",\n" + + " \"oauth2\": {\n" + + " \"oauth2ClientID\": \"clientID\",\n" + + " \"oauth2ClientSecret\": \"oauth2 client secret\",\n" + + " \"oauth2Server\": \"https://www.overleaf.com\"\n" + + " }"; + if (swapCfg != null) { + cfgStr += ",\n" + + " \"swapStore\": {\n" + + " \"type\": \"memory\",\n" + + " \"awsAccessKey\": null,\n" + + " \"awsSecret\": null,\n" + + " \"s3BucketName\": \"com.overleaf.testbucket\"\n" + + " },\n" + + " \"swapJob\": {\n" + + " \"minProjects\": " + + swapCfg.getMinProjects() + + ",\n" + + " \"lowGiB\": " + + swapCfg.getLowGiB() + + ",\n" + + " \"highGiB\": " + + swapCfg.getHighGiB() + + ",\n" + + " \"intervalMillis\": " + + swapCfg.getIntervalMillis() + + "\n" + + " }\n"; + } + cfgStr += "}\n"; + writer.print(cfgStr); writer.close(); return config.getAbsolutePath(); } private Path getResource(String path) { - return Paths.get("src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest" + path); + return Paths.get( + "src/test/resources/" + + "uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest" + path + ); } private InputStream getResourceAsStream(String path) { diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java index 68236e3735..a2ecda2850 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java @@ -3,15 +3,25 @@ package uk.ac.ic.wlgitbridge.bridge; import org.junit.Before; import org.junit.Test; import uk.ac.ic.wlgitbridge.bridge.db.DBStore; +import uk.ac.ic.wlgitbridge.bridge.db.ProjectState; import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock; import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache; import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI; -import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob; -import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore; +import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob; +import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore; +import uk.ac.ic.wlgitbridge.data.model.Snapshot; +import uk.ac.ic.wlgitbridge.git.exception.GitUserException; +import java.io.IOException; +import java.util.ArrayDeque; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** * Created by winston on 20/08/2016. @@ -35,24 +45,42 @@ public class BridgeTest { dbStore = mock(DBStore.class); swapStore = mock(SwapStore.class); snapshotAPI = mock(SnapshotAPI.class); + resourceCache = mock(ResourceCache.class); swapJob = mock(SwapJob.class); bridge = new Bridge( lock, repoStore, dbStore, swapStore, + swapJob, snapshotAPI, - resourceCache, - swapJob + resourceCache ); } @Test public void shutdownStopsSwapJob() { - bridge.startSwapJob(1000); + bridge.startSwapJob(); bridge.doShutdown(); - verify(swapJob).start(1000); + verify(swapJob).start(); verify(swapJob).stop(); } + @Test + public void updatingRepositorySetsLastAccessedTime( + ) throws IOException, GitUserException { + ProjectRepo repo = mock(ProjectRepo.class); + when(repo.getProjectName()).thenReturn("asdf"); + when(dbStore.getProjectState("asdf")).thenReturn(ProjectState.PRESENT); + when( + snapshotAPI.getSnapshotsForProjectAfterVersion( + any(), + any(), + anyInt() + ) + ).thenReturn(new ArrayDeque()); + bridge.updateRepository(null, repo); + verify(dbStore).setLastAccessedTime(eq("asdf"), any()); + } + } \ No newline at end of file diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java new file mode 100644 index 0000000000..02065c7b0c --- /dev/null +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStoreTest.java @@ -0,0 +1,150 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite; + +import org.junit.Before; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import uk.ac.ic.wlgitbridge.bridge.db.ProjectState; + +import java.io.IOException; +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Created by winston on 23/08/2016. + */ +public class SqliteDBStoreTest { + + private SqliteDBStore dbStore; + + @Before + public void setup() throws IOException { + TemporaryFolder tmpFolder = new TemporaryFolder(); + tmpFolder.create(); + dbStore = new SqliteDBStore(tmpFolder.newFile("dbStore.db")); + } + + @Test + public void testGetNumProjects() { + assertEquals(0, dbStore.getNumProjects()); + dbStore.setLatestVersionForProject("asdf", 1); + assertEquals(1, dbStore.getNumProjects()); + dbStore.setLatestVersionForProject("asdf1", 2); + assertEquals(2, dbStore.getNumProjects()); + dbStore.setLatestVersionForProject("asdf1", 3); + assertEquals(2, dbStore.getNumProjects()); + } + + @Test + public void swapTableStartsOutEmpty() { + assertNull(dbStore.getOldestUnswappedProject()); + } + + @Test + public void testGetOldestUnswappedProject() { + dbStore.setLatestVersionForProject("older", 3); + dbStore.setLastAccessedTime( + "older", + Timestamp.valueOf( + LocalDateTime.now().minus(5, ChronoUnit.SECONDS) + ) + ); + dbStore.setLatestVersionForProject("asdf", 1); + dbStore.setLastAccessedTime( + "asdf", + Timestamp.valueOf( + LocalDateTime.now().minus(1, ChronoUnit.SECONDS) + ) + ); + assertEquals("older", dbStore.getOldestUnswappedProject()); + dbStore.setLastAccessedTime( + "older", + Timestamp.valueOf( + LocalDateTime.now() + ) + ); + assertEquals("asdf", dbStore.getOldestUnswappedProject()); + } + + @Test + public void noOldestProjectIfAllEvicted() { + dbStore.setLatestVersionForProject("older", 3); + dbStore.setLastAccessedTime("older", null); + assertNull(dbStore.getOldestUnswappedProject()); + } + + @Test + public void nullLastAccessedTimesDoNotCount() { + dbStore.setLatestVersionForProject("older", 2); + dbStore.setLastAccessedTime( + "older", + Timestamp.valueOf( + LocalDateTime.now().minus(5, ChronoUnit.SECONDS) + ) + ); + dbStore.setLatestVersionForProject("newer", 3); + dbStore.setLastAccessedTime( + "newer", + Timestamp.valueOf( + LocalDateTime.now() + ) + ); + assertEquals("older", dbStore.getOldestUnswappedProject()); + dbStore.setLastAccessedTime("older", null); + assertEquals("newer", dbStore.getOldestUnswappedProject()); + } + + @Test + public void missingProjectLastAccessedTimeCanBeSet() { + dbStore.setLatestVersionForProject("asdf", 1); + dbStore.setLastAccessedTime( + "asdf", + Timestamp.valueOf(LocalDateTime.now()) + ); + assertEquals("asdf", dbStore.getOldestUnswappedProject()); + } + + @Test + public void testGetNumUnswappedProjects() { + dbStore.setLatestVersionForProject("asdf", 1); + dbStore.setLastAccessedTime( + "asdf", + Timestamp.valueOf(LocalDateTime.now()) + ); + assertEquals(1, dbStore.getNumUnswappedProjects()); + dbStore.setLastAccessedTime( + "asdf", + null + ); + assertEquals(0, dbStore.getNumUnswappedProjects()); + } + + @Test + public void projectStateIsNotPresentIfNotInDBAtAll() { + assertEquals( + ProjectState.NOT_PRESENT, + dbStore.getProjectState("asdf") + ); + } + + @Test + public void projectStateIsPresentIfProjectHasLastAccessed() { + dbStore.setLatestVersionForProject("asdf", 1); + dbStore.setLastAccessedTime( + "asdf", + Timestamp.valueOf(LocalDateTime.now()) + ); + assertEquals(ProjectState.PRESENT, dbStore.getProjectState("asdf")); + } + + @Test + public void projectStateIsSwappedIfLastAccessedIsNull() { + dbStore.setLatestVersionForProject("asdf", 1); + dbStore.setLastAccessedTime("asdf", null); + assertEquals(ProjectState.SWAPPED, dbStore.getProjectState("asdf")); + } + +} \ No newline at end of file diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteFilesForProjectSQLUpdateTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteFilesForProjectSQLUpdateTest.java new file mode 100644 index 0000000000..4cf2c21d57 --- /dev/null +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteFilesForProjectSQLUpdateTest.java @@ -0,0 +1,24 @@ +package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class DeleteFilesForProjectSQLUpdateTest { + + @Test + public void testGetSQL() { + DeleteFilesForProjectSQLUpdate update = + new DeleteFilesForProjectSQLUpdate( + "projname", + "path1", + "path2" + ); + assertEquals( + "DELETE FROM `url_index_store` " + + "WHERE `project_name` = ? " + + "AND path IN (?, ?);\n", + update.getSQL() + ); + } + +} \ No newline at end of file diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest.java new file mode 100644 index 0000000000..4924ce6da5 --- /dev/null +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest.java @@ -0,0 +1,83 @@ +package uk.ac.ic.wlgitbridge.bridge.repo; + +import org.apache.commons.io.FileUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import uk.ac.ic.wlgitbridge.util.Files; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; + +/** + * Created by winston on 23/08/2016. + */ +public class FSGitRepoStoreTest { + + public static File makeTempRepoDir( + TemporaryFolder tmpFolder, + String name + ) throws IOException { + File tmp = tmpFolder.newFolder(name); + Path rootdir = Paths.get( + "src/test/resources/uk/ac/ic/wlgitbridge/" + + "bridge/repo/FSGitRepoStoreTest/rootdir" + ); + FileUtils.copyDirectory(rootdir.toFile(), tmp); + Files.renameAll(tmp, "DOTgit", ".git"); + return tmp; + } + + private FSGitRepoStore repoStore; + private File original; + + @Before + public void setup() throws IOException { + TemporaryFolder tmpFolder = new TemporaryFolder(); + tmpFolder.create(); + File tmp = makeTempRepoDir(tmpFolder, "rootdir"); + original = tmpFolder.newFolder("original"); + FileUtils.copyDirectory(tmp, original); + repoStore = new FSGitRepoStore(tmp.getAbsolutePath()); + } + + @Test + public void testPurgeNonexistentProjects() { + File toDelete = new File(repoStore.getRootDirectory(), "idontexist"); + File wlgb = new File(repoStore.getRootDirectory(), ".wlgb"); + Assert.assertTrue(toDelete.exists()); + Assert.assertTrue(wlgb.exists()); + repoStore.purgeNonexistentProjects(Arrays.asList("proj1", "proj2")); + Assert.assertFalse(toDelete.exists()); + Assert.assertTrue(wlgb.exists()); + } + + @Test + public void testTotalSize() { + assertEquals(31860, repoStore.totalSize()); + } + + @Test + public void zipAndUnzipShouldBeTheSame() throws IOException { + long beforeSize = repoStore.totalSize(); + InputStream zipped = repoStore.bzip2Project("proj1"); + repoStore.remove("proj1"); + Assert.assertTrue(beforeSize > repoStore.totalSize()); + repoStore.unbzip2Project("proj1", zipped); + Assert.assertEquals(beforeSize, repoStore.totalSize()); + Assert.assertTrue( + Files.contentsAreEqual( + original, + repoStore.getRootDirectory() + ) + ); + } + +} \ No newline at end of file diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java new file mode 100644 index 0000000000..c2359766bb --- /dev/null +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java @@ -0,0 +1,119 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.job; + +import org.junit.Before; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import uk.ac.ic.wlgitbridge.bridge.db.DBStore; +import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore; +import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock; +import uk.ac.ic.wlgitbridge.bridge.repo.FSGitRepoStore; +import uk.ac.ic.wlgitbridge.bridge.repo.FSGitRepoStoreTest; +import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; +import uk.ac.ic.wlgitbridge.bridge.swap.store.InMemorySwapStore; +import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore; +import uk.ac.ic.wlgitbridge.data.ProjectLockImpl; + +import java.io.IOException; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Created by winston on 20/08/2016. + */ +public class SwapJobImplTest { + + private SwapJobImpl swapJob; + + private ProjectLock lock; + private RepoStore repoStore; + private DBStore dbStore; + private SwapStore swapStore; + + @Before + public void setup() throws IOException { + TemporaryFolder tmpFolder = new TemporaryFolder(); + tmpFolder.create(); + lock = new ProjectLockImpl(); + repoStore = new FSGitRepoStore( + FSGitRepoStoreTest.makeTempRepoDir( + tmpFolder, + "repostore" + ).getAbsolutePath() + ); + dbStore = new SqliteDBStore(tmpFolder.newFile()); + dbStore.setLatestVersionForProject("proj1", 0); + dbStore.setLatestVersionForProject("proj2", 0); + dbStore.setLastAccessedTime( + "proj1", + Timestamp.valueOf(LocalDateTime.now()) + ); + dbStore.setLastAccessedTime( + "proj2", + Timestamp.valueOf( + LocalDateTime.now().minus(1, ChronoUnit.SECONDS) + ) + ); + swapStore = new InMemorySwapStore(); + swapJob = new SwapJobImpl( + 1, + 15000, + 30000, + Duration.ofMillis(100), + lock, + repoStore, + dbStore, + swapStore + ); + } + + @Test + public void startingTimerAlwaysCausesASwap() { + swapJob.lowWatermarkBytes = 16384; + swapJob.interval = Duration.ofHours(1); + assertEquals(0, swapJob.swaps.get()); + swapJob.start(); + while (swapJob.swaps.get() <= 0); + assertTrue(swapJob.swaps.get() > 0); + } + + @Test + public void swapsHappenEveryInterval() { + swapJob.lowWatermarkBytes = 16384; + assertEquals(0, swapJob.swaps.get()); + swapJob.start(); + while (swapJob.swaps.get() <= 1); + assertTrue(swapJob.swaps.get() > 1); + } + + @Test + public void noProjectsGetSwappedWhenUnderHighWatermark() { + swapJob.highWatermarkBytes = 65536; + assertEquals(2, dbStore.getNumUnswappedProjects()); + swapJob.start(); + while (swapJob.swaps.get() < 1); + assertEquals(2, dbStore.getNumUnswappedProjects()); + } + + @Test + public void correctProjGetSwappedWhenOverHighWatermark( + ) throws IOException { + swapJob.lowWatermarkBytes = 16384; + assertEquals(2, dbStore.getNumUnswappedProjects()); + assertEquals("proj2", dbStore.getOldestUnswappedProject()); + swapJob.start(); + while (swapJob.swaps.get() < 1); + assertEquals(1, dbStore.getNumUnswappedProjects()); + assertEquals("proj1", dbStore.getOldestUnswappedProject()); + swapJob.restore("proj2"); + int numSwaps = swapJob.swaps.get(); + while (swapJob.swaps.get() <= numSwaps); + assertEquals(1, dbStore.getNumUnswappedProjects()); + assertEquals("proj2", dbStore.getOldestUnswappedProject()); + } + +} \ No newline at end of file diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStoreTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStoreTest.java new file mode 100644 index 0000000000..7e257f2305 --- /dev/null +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStoreTest.java @@ -0,0 +1,96 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.store; + +import org.apache.commons.io.IOUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import uk.ac.ic.wlgitbridge.bridge.swap.store.InMemorySwapStore; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +import static org.junit.Assert.assertArrayEquals; + +/** + * Created by winston on 23/08/2016. + */ +public class InMemorySwapStoreTest { + + private final InMemorySwapStore swapStore = new InMemorySwapStore(); + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + @Test + public void downloadingNonExistentFileThrows() { + exception.expect(IllegalArgumentException.class); + swapStore.openDownloadStream("asdf"); + } + + @Test + public void canDownloadUploadedFiles() throws IOException { + byte[] proj1Contents = "helloproj1".getBytes(); + byte[] proj2Contents = "asdfproj2".getBytes(); + swapStore.upload( + "proj1", + new ByteArrayInputStream(proj1Contents), + proj1Contents.length + ); + swapStore.upload( + "proj2", + new ByteArrayInputStream(proj2Contents), + proj2Contents.length + ); + assertArrayEquals( + proj1Contents, + IOUtils.toByteArray(swapStore.openDownloadStream("proj1")) + ); + assertArrayEquals( + proj2Contents, + IOUtils.toByteArray(swapStore.openDownloadStream("proj2")) + ); + } + + @Test + public void uploadingForTheSameProjectOverwritesTheFile( + ) throws IOException { + byte[] proj1Contents = "helloproj1".getBytes(); + byte[] proj1NewContents = "goodbyeproj1".getBytes(); + swapStore.upload( + "proj1", + new ByteArrayInputStream(proj1Contents), + proj1Contents.length + ); + assertArrayEquals( + proj1Contents, + IOUtils.toByteArray(swapStore.openDownloadStream("proj1")) + ); + swapStore.upload( + "proj1", + new ByteArrayInputStream(proj1NewContents), + proj1NewContents.length + ); + assertArrayEquals( + proj1NewContents, + IOUtils.toByteArray(swapStore.openDownloadStream("proj1")) + ); + } + + @Test + public void canRemoveFiles() throws IOException { + byte[] projContents = "total garbage".getBytes(); + swapStore.upload( + "proj", + new ByteArrayInputStream(projContents), + projContents.length + ); + assertArrayEquals( + projContents, + IOUtils.toByteArray(swapStore.openDownloadStream("proj")) + ); + swapStore.remove("proj"); + exception.expect(IllegalArgumentException.class); + swapStore.openDownloadStream("proj"); + } + +} \ No newline at end of file diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStoreTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStoreTest.java new file mode 100644 index 0000000000..f8a7ec59e1 --- /dev/null +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStoreTest.java @@ -0,0 +1,41 @@ +package uk.ac.ic.wlgitbridge.bridge.swap.store; + +import org.junit.Before; + +/** + * Created by winston on 21/08/2016. + */ +public class S3SwapStoreTest { + + private static final String accessKey = null; + private static final String secret = null; + private static final String bucketName = "com.overleaf.testbucket"; + + private S3SwapStore s3; + + @Before + public void setup() { + if (accessKey == null || secret == null) { + s3 = null; + return; + } + s3 = new S3SwapStore(accessKey, secret, bucketName); + } + +// @Ignore +// @Test +// public void testUploadDownloadDelete() throws Exception { +// assumeNotNull(s3); +// String projName = "abc123"; +// byte[] contents = "hello".getBytes(); +// s3.upload( +// projName, +// new ByteArrayInputStream(contents), +// contents.length +// ); +// InputStream down = s3.openDownloadStream(projName); +// s3.remove(projName); +// assertArrayEquals(contents, IOUtils.toByteArray(down)); +// } + +} \ No newline at end of file diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/delete/DeleteFilesForProjectSQLUpdateTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/delete/DeleteFilesForProjectSQLUpdateTest.java deleted file mode 100644 index 8904e60877..0000000000 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/db/sql/update/delete/DeleteFilesForProjectSQLUpdateTest.java +++ /dev/null @@ -1,14 +0,0 @@ -package uk.ac.ic.wlgitbridge.data.model.db.sql.update.delete; - -import org.junit.Test; -import static org.junit.Assert.*; - -public class DeleteFilesForProjectSQLUpdateTest { - - @Test - public void testGetSQL() { - DeleteFilesForProjectSQLUpdate update = new DeleteFilesForProjectSQLUpdate("projname", "path1", "path2"); - assertEquals("DELETE FROM `url_index_store` WHERE `project_name` = ? AND path IN (?, ?);\n", update.getSQL()); - } - -} \ No newline at end of file diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/ProjectTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/ProjectTest.java new file mode 100644 index 0000000000..23d664e4ed --- /dev/null +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/ProjectTest.java @@ -0,0 +1,18 @@ +package uk.ac.ic.wlgitbridge.util; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Created by winston on 23/08/2016. + */ +public class ProjectTest { + + @Test + public void testValidProjectNames() { + Assert.assertFalse(Project.isValidProjectName(null)); + Assert.assertFalse(Project.isValidProjectName("")); + Assert.assertFalse(Project.isValidProjectName(".wlgb")); + } + +} diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TarTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TarTest.java new file mode 100644 index 0000000000..1ce147e5d8 --- /dev/null +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TarTest.java @@ -0,0 +1,52 @@ +package uk.ac.ic.wlgitbridge.util; + +import org.apache.commons.io.FileUtils; +import org.junit.Before; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.Assert.assertTrue; + +/** + * Created by winston on 23/08/2016. + */ +public class TarTest { + + private File testDir; + private File tmpDir; + + @Before + public void setup() throws IOException { + TemporaryFolder tmpFolder = new TemporaryFolder(); + tmpFolder.create(); + testDir = tmpFolder.newFolder("testdir"); + Path resdir = Paths.get( + "src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir" + ); + FileUtils.copyDirectory(resdir.toFile(), testDir); + tmpDir = tmpFolder.newFolder(); + } + + @Test + public void tarAndUntarProducesTheSameResult() throws IOException { + InputStream tar = Tar.tar(testDir); + Tar.untar(tar, tmpDir); + File untarred = new File(tmpDir, "testdir"); + assertTrue(Files.contentsAreEqual(testDir, untarred)); + } + + @Test + public void tarbz2AndUntarbz2ProducesTheSameResult() throws IOException { + InputStream tarbz2 = Tar.bz2.zip(testDir); + Tar.bz2.unzip(tarbz2, tmpDir); + File unzipped = new File(tmpDir, "testdir"); + assertTrue(Files.contentsAreEqual(testDir, unzipped)); + } + +} \ No newline at end of file diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TimerTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TimerTest.java new file mode 100644 index 0000000000..487c7d8bf5 --- /dev/null +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TimerTest.java @@ -0,0 +1,19 @@ +package uk.ac.ic.wlgitbridge.util; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Created by winston on 23/08/2016. + */ +public class TimerTest { + + @Test + public void testMakeTimerTask() { + int[] iPtr = new int[] { 3 }; + Timer.makeTimerTask(() -> iPtr[0] = 5).run(); + assertEquals(5, iPtr[0]); + } + +} diff --git a/services/git-bridge/src/test/resources/logback-test.xml b/services/git-bridge/src/test/resources/logback-test.xml index 4e1539cf06..455a379541 100644 --- a/services/git-bridge/src/test/resources/logback-test.xml +++ b/services/git-bridge/src/test/resources/logback-test.xml @@ -1,7 +1,7 @@ - - ${java.io.tmpdir}/git-bridge-test.log + + System.err %d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%thread] %logger{0}: %msg%n @@ -12,6 +12,6 @@ - + diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/state.json b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/state.json new file mode 100644 index 0000000000..77d610d756 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/state.json @@ -0,0 +1,90 @@ +[ + { + "project": "testproj1", + "getDoc": { + "versionID": 1, + "createdAt": "2014-11-30T18:40:58.123Z", + "email": "jdleesmiller+1@gmail.com", + "name": "John+1" + }, + "getSavedVers": [ + { + "versionID": 1, + "comment": "added more info on doc GET and error details", + "email": "jdleesmiller+1@gmail.com", + "name": "John+1", + "createdAt": "2014-11-30T18:47:01.456Z" + } + ], + "getForVers": [ + { + "versionID": 1, + "srcs": [ + { + "content": "content\n", + "path": "main.tex" + }, + { + "content": "This text is from another file.", + "path": "foo/bar/test.tex" + } + ], + "atts": [ + { + "url": "http://127.0.0.1:3874/state/testproj1/overleaf-white-410.png", + "path": "overleaf-white-410.png" + } + ] + } + ], + "push": "success", + "postback": { + "type": "success", + "versionID": 2 + } + }, + { + "project": "testproj2", + "getDoc": { + "versionID": 1, + "createdAt": "2014-11-30T18:40:58.123Z", + "email": "jdleesmiller+1@gmail.com", + "name": "John+1" + }, + "getSavedVers": [ + { + "versionID": 1, + "comment": "added more info on doc GET and error details", + "email": "jdleesmiller+1@gmail.com", + "name": "John+1", + "createdAt": "2014-11-30T18:47:01.456Z" + } + ], + "getForVers": [ + { + "versionID": 1, + "srcs": [ + { + "content": "different content\n", + "path": "main.tex" + }, + { + "content": "a different one", + "path": "foo/bar/test.tex" + } + ], + "atts": [ + { + "url": "http://127.0.0.1:3874/state/testproj2/editor-versions-a7e4de19d015c3e7477e3f7eaa6c418e.png", + "path": "editor-versions-a7e4de19d015c3e7477e3f7eaa6c418e.png" + } + ] + } + ], + "push": "success", + "postback": { + "type": "success", + "versionID": 2 + } + } +] \ No newline at end of file diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj1/foo/bar/test.tex b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj1/foo/bar/test.tex new file mode 100644 index 0000000000..046794f19a --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj1/foo/bar/test.tex @@ -0,0 +1 @@ +This text is from another file. \ No newline at end of file diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj1/main.tex b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj1/main.tex new file mode 100644 index 0000000000..d95f3ad14d --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj1/main.tex @@ -0,0 +1 @@ +content diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj1/overleaf-white-410.png b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj1/overleaf-white-410.png new file mode 100644 index 0000000000..6a23d10c15 Binary files /dev/null and b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj1/overleaf-white-410.png differ diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj2/editor-versions-a7e4de19d015c3e7477e3f7eaa6c418e.png b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj2/editor-versions-a7e4de19d015c3e7477e3f7eaa6c418e.png new file mode 100644 index 0000000000..7fa339be7a Binary files /dev/null and b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj2/editor-versions-a7e4de19d015c3e7477e3f7eaa6c418e.png differ diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj2/foo/bar/test.tex b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj2/foo/bar/test.tex new file mode 100644 index 0000000000..1c05c0183f --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj2/foo/bar/test.tex @@ -0,0 +1 @@ +a different one \ No newline at end of file diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj2/main.tex b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj2/main.tex new file mode 100644 index 0000000000..802d4bbd2e --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/wlgbCanSwapProjects/state/testproj2/main.tex @@ -0,0 +1 @@ +different content diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/.wlgb/wlgb.db b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/.wlgb/wlgb.db new file mode 100644 index 0000000000..e69de29bb2 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/idontexist/idontexist.txt b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/idontexist/idontexist.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/COMMIT_EDITMSG b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/COMMIT_EDITMSG new file mode 100644 index 0000000000..c098216e78 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/COMMIT_EDITMSG @@ -0,0 +1 @@ +Main diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/HEAD b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/HEAD new file mode 100644 index 0000000000..cb089cd89a --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/config b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/config new file mode 100644 index 0000000000..6c9406b7d9 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/config @@ -0,0 +1,7 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = false + logallrefupdates = true + ignorecase = true + precomposeunicode = true diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/description b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/description new file mode 100644 index 0000000000..498b267a8c --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/description @@ -0,0 +1 @@ +Unnamed repository; edit this file 'description' to name the repository. diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/applypatch-msg.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/applypatch-msg.sample new file mode 100755 index 0000000000..a5d7b84a67 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/applypatch-msg.sample @@ -0,0 +1,15 @@ +#!/bin/sh +# +# An example hook script to check the commit log message taken by +# applypatch from an e-mail message. +# +# The hook should exit with non-zero status after issuing an +# appropriate message if it wants to stop the commit. The hook is +# allowed to edit the commit message file. +# +# To enable this hook, rename this file to "applypatch-msg". + +. git-sh-setup +commitmsg="$(git rev-parse --git-path hooks/commit-msg)" +test -x "$commitmsg" && exec "$commitmsg" ${1+"$@"} +: diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/commit-msg.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/commit-msg.sample new file mode 100755 index 0000000000..b58d1184a9 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/commit-msg.sample @@ -0,0 +1,24 @@ +#!/bin/sh +# +# An example hook script to check the commit log message. +# Called by "git commit" with one argument, the name of the file +# that has the commit message. The hook should exit with non-zero +# status after issuing an appropriate message if it wants to stop the +# commit. The hook is allowed to edit the commit message file. +# +# To enable this hook, rename this file to "commit-msg". + +# Uncomment the below to add a Signed-off-by line to the message. +# Doing this in a hook is a bad idea in general, but the prepare-commit-msg +# hook is more suited to it. +# +# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') +# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1" + +# This example catches duplicate Signed-off-by lines. + +test "" = "$(grep '^Signed-off-by: ' "$1" | + sort | uniq -c | sed -e '/^[ ]*1[ ]/d')" || { + echo >&2 Duplicate Signed-off-by lines. + exit 1 +} diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/post-update.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/post-update.sample new file mode 100755 index 0000000000..ec17ec1939 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/post-update.sample @@ -0,0 +1,8 @@ +#!/bin/sh +# +# An example hook script to prepare a packed repository for use over +# dumb transports. +# +# To enable this hook, rename this file to "post-update". + +exec git update-server-info diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/pre-applypatch.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/pre-applypatch.sample new file mode 100755 index 0000000000..4142082bcb --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/pre-applypatch.sample @@ -0,0 +1,14 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed +# by applypatch from an e-mail message. +# +# The hook should exit with non-zero status after issuing an +# appropriate message if it wants to stop the commit. +# +# To enable this hook, rename this file to "pre-applypatch". + +. git-sh-setup +precommit="$(git rev-parse --git-path hooks/pre-commit)" +test -x "$precommit" && exec "$precommit" ${1+"$@"} +: diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/pre-commit.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/pre-commit.sample new file mode 100755 index 0000000000..68d62d5446 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/pre-commit.sample @@ -0,0 +1,49 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed. +# Called by "git commit" with no arguments. The hook should +# exit with non-zero status after issuing an appropriate message if +# it wants to stop the commit. +# +# To enable this hook, rename this file to "pre-commit". + +if git rev-parse --verify HEAD >/dev/null 2>&1 +then + against=HEAD +else + # Initial commit: diff against an empty tree object + against=4b825dc642cb6eb9a060e54bf8d69288fbee4904 +fi + +# If you want to allow non-ASCII filenames set this variable to true. +allownonascii=$(git config --bool hooks.allownonascii) + +# Redirect output to stderr. +exec 1>&2 + +# Cross platform projects tend to avoid non-ASCII filenames; prevent +# them from being added to the repository. We exploit the fact that the +# printable range starts at the space character and ends with tilde. +if [ "$allownonascii" != "true" ] && + # Note that the use of brackets around a tr range is ok here, (it's + # even required, for portability to Solaris 10's /usr/bin/tr), since + # the square bracket bytes happen to fall in the designated range. + test $(git diff --cached --name-only --diff-filter=A -z $against | + LC_ALL=C tr -d '[ -~]\0' | wc -c) != 0 +then + cat <<\EOF +Error: Attempt to add a non-ASCII file name. + +This can cause problems if you want to work with people on other platforms. + +To be portable it is advisable to rename the file. + +If you know what you are doing you can disable this check using: + + git config hooks.allownonascii true +EOF + exit 1 +fi + +# If there are whitespace errors, print the offending file names and fail. +exec git diff-index --check --cached $against -- diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/pre-push.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/pre-push.sample new file mode 100755 index 0000000000..6187dbf439 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/pre-push.sample @@ -0,0 +1,53 @@ +#!/bin/sh + +# An example hook script to verify what is about to be pushed. Called by "git +# push" after it has checked the remote status, but before anything has been +# pushed. If this script exits with a non-zero status nothing will be pushed. +# +# This hook is called with the following parameters: +# +# $1 -- Name of the remote to which the push is being done +# $2 -- URL to which the push is being done +# +# If pushing without using a named remote those arguments will be equal. +# +# Information about the commits which are being pushed is supplied as lines to +# the standard input in the form: +# +# +# +# This sample shows how to prevent push of commits where the log message starts +# with "WIP" (work in progress). + +remote="$1" +url="$2" + +z40=0000000000000000000000000000000000000000 + +while read local_ref local_sha remote_ref remote_sha +do + if [ "$local_sha" = $z40 ] + then + # Handle delete + : + else + if [ "$remote_sha" = $z40 ] + then + # New branch, examine all commits + range="$local_sha" + else + # Update to existing branch, examine new commits + range="$remote_sha..$local_sha" + fi + + # Check for WIP commit + commit=`git rev-list -n 1 --grep '^WIP' "$range"` + if [ -n "$commit" ] + then + echo >&2 "Found WIP commit in $local_ref, not pushing" + exit 1 + fi + fi +done + +exit 0 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/pre-rebase.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/pre-rebase.sample new file mode 100755 index 0000000000..9773ed4cb2 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/pre-rebase.sample @@ -0,0 +1,169 @@ +#!/bin/sh +# +# Copyright (c) 2006, 2008 Junio C Hamano +# +# The "pre-rebase" hook is run just before "git rebase" starts doing +# its job, and can prevent the command from running by exiting with +# non-zero status. +# +# The hook is called with the following parameters: +# +# $1 -- the upstream the series was forked from. +# $2 -- the branch being rebased (or empty when rebasing the current branch). +# +# This sample shows how to prevent topic branches that are already +# merged to 'next' branch from getting rebased, because allowing it +# would result in rebasing already published history. + +publish=next +basebranch="$1" +if test "$#" = 2 +then + topic="refs/heads/$2" +else + topic=`git symbolic-ref HEAD` || + exit 0 ;# we do not interrupt rebasing detached HEAD +fi + +case "$topic" in +refs/heads/??/*) + ;; +*) + exit 0 ;# we do not interrupt others. + ;; +esac + +# Now we are dealing with a topic branch being rebased +# on top of master. Is it OK to rebase it? + +# Does the topic really exist? +git show-ref -q "$topic" || { + echo >&2 "No such branch $topic" + exit 1 +} + +# Is topic fully merged to master? +not_in_master=`git rev-list --pretty=oneline ^master "$topic"` +if test -z "$not_in_master" +then + echo >&2 "$topic is fully merged to master; better remove it." + exit 1 ;# we could allow it, but there is no point. +fi + +# Is topic ever merged to next? If so you should not be rebasing it. +only_next_1=`git rev-list ^master "^$topic" ${publish} | sort` +only_next_2=`git rev-list ^master ${publish} | sort` +if test "$only_next_1" = "$only_next_2" +then + not_in_topic=`git rev-list "^$topic" master` + if test -z "$not_in_topic" + then + echo >&2 "$topic is already up-to-date with master" + exit 1 ;# we could allow it, but there is no point. + else + exit 0 + fi +else + not_in_next=`git rev-list --pretty=oneline ^${publish} "$topic"` + /usr/bin/perl -e ' + my $topic = $ARGV[0]; + my $msg = "* $topic has commits already merged to public branch:\n"; + my (%not_in_next) = map { + /^([0-9a-f]+) /; + ($1 => 1); + } split(/\n/, $ARGV[1]); + for my $elem (map { + /^([0-9a-f]+) (.*)$/; + [$1 => $2]; + } split(/\n/, $ARGV[2])) { + if (!exists $not_in_next{$elem->[0]}) { + if ($msg) { + print STDERR $msg; + undef $msg; + } + print STDERR " $elem->[1]\n"; + } + } + ' "$topic" "$not_in_next" "$not_in_master" + exit 1 +fi + +exit 0 + +################################################################ + +This sample hook safeguards topic branches that have been +published from being rewound. + +The workflow assumed here is: + + * Once a topic branch forks from "master", "master" is never + merged into it again (either directly or indirectly). + + * Once a topic branch is fully cooked and merged into "master", + it is deleted. If you need to build on top of it to correct + earlier mistakes, a new topic branch is created by forking at + the tip of the "master". This is not strictly necessary, but + it makes it easier to keep your history simple. + + * Whenever you need to test or publish your changes to topic + branches, merge them into "next" branch. + +The script, being an example, hardcodes the publish branch name +to be "next", but it is trivial to make it configurable via +$GIT_DIR/config mechanism. + +With this workflow, you would want to know: + +(1) ... if a topic branch has ever been merged to "next". Young + topic branches can have stupid mistakes you would rather + clean up before publishing, and things that have not been + merged into other branches can be easily rebased without + affecting other people. But once it is published, you would + not want to rewind it. + +(2) ... if a topic branch has been fully merged to "master". + Then you can delete it. More importantly, you should not + build on top of it -- other people may already want to + change things related to the topic as patches against your + "master", so if you need further changes, it is better to + fork the topic (perhaps with the same name) afresh from the + tip of "master". + +Let's look at this example: + + o---o---o---o---o---o---o---o---o---o "next" + / / / / + / a---a---b A / / + / / / / + / / c---c---c---c B / + / / / \ / + / / / b---b C \ / + / / / / \ / + ---o---o---o---o---o---o---o---o---o---o---o "master" + + +A, B and C are topic branches. + + * A has one fix since it was merged up to "next". + + * B has finished. It has been fully merged up to "master" and "next", + and is ready to be deleted. + + * C has not merged to "next" at all. + +We would want to allow C to be rebased, refuse A, and encourage +B to be deleted. + +To compute (1): + + git rev-list ^master ^topic next + git rev-list ^master next + + if these match, topic has not merged in next at all. + +To compute (2): + + git rev-list master..topic + + if this is empty, it is fully merged to "master". diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/prepare-commit-msg.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/prepare-commit-msg.sample new file mode 100755 index 0000000000..f093a02ec4 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/prepare-commit-msg.sample @@ -0,0 +1,36 @@ +#!/bin/sh +# +# An example hook script to prepare the commit log message. +# Called by "git commit" with the name of the file that has the +# commit message, followed by the description of the commit +# message's source. The hook's purpose is to edit the commit +# message file. If the hook fails with a non-zero status, +# the commit is aborted. +# +# To enable this hook, rename this file to "prepare-commit-msg". + +# This hook includes three examples. The first comments out the +# "Conflicts:" part of a merge commit. +# +# The second includes the output of "git diff --name-status -r" +# into the message, just before the "git status" output. It is +# commented because it doesn't cope with --amend or with squashed +# commits. +# +# The third example adds a Signed-off-by line to the message, that can +# still be edited. This is rarely a good idea. + +case "$2,$3" in + merge,) + /usr/bin/perl -i.bak -ne 's/^/# /, s/^# #/#/ if /^Conflicts/ .. /#/; print' "$1" ;; + +# ,|template,) +# /usr/bin/perl -i.bak -pe ' +# print "\n" . `git diff --cached --name-status -r` +# if /^#/ && $first++ == 0' "$1" ;; + + *) ;; +esac + +# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') +# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1" diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/update.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/update.sample new file mode 100755 index 0000000000..80ba94135c --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/hooks/update.sample @@ -0,0 +1,128 @@ +#!/bin/sh +# +# An example hook script to block unannotated tags from entering. +# Called by "git receive-pack" with arguments: refname sha1-old sha1-new +# +# To enable this hook, rename this file to "update". +# +# Config +# ------ +# hooks.allowunannotated +# This boolean sets whether unannotated tags will be allowed into the +# repository. By default they won't be. +# hooks.allowdeletetag +# This boolean sets whether deleting tags will be allowed in the +# repository. By default they won't be. +# hooks.allowmodifytag +# This boolean sets whether a tag may be modified after creation. By default +# it won't be. +# hooks.allowdeletebranch +# This boolean sets whether deleting branches will be allowed in the +# repository. By default they won't be. +# hooks.denycreatebranch +# This boolean sets whether remotely creating branches will be denied +# in the repository. By default this is allowed. +# + +# --- Command line +refname="$1" +oldrev="$2" +newrev="$3" + +# --- Safety check +if [ -z "$GIT_DIR" ]; then + echo "Don't run this script from the command line." >&2 + echo " (if you want, you could supply GIT_DIR then run" >&2 + echo " $0 )" >&2 + exit 1 +fi + +if [ -z "$refname" -o -z "$oldrev" -o -z "$newrev" ]; then + echo "usage: $0 " >&2 + exit 1 +fi + +# --- Config +allowunannotated=$(git config --bool hooks.allowunannotated) +allowdeletebranch=$(git config --bool hooks.allowdeletebranch) +denycreatebranch=$(git config --bool hooks.denycreatebranch) +allowdeletetag=$(git config --bool hooks.allowdeletetag) +allowmodifytag=$(git config --bool hooks.allowmodifytag) + +# check for no description +projectdesc=$(sed -e '1q' "$GIT_DIR/description") +case "$projectdesc" in +"Unnamed repository"* | "") + echo "*** Project description file hasn't been set" >&2 + exit 1 + ;; +esac + +# --- Check types +# if $newrev is 0000...0000, it's a commit to delete a ref. +zero="0000000000000000000000000000000000000000" +if [ "$newrev" = "$zero" ]; then + newrev_type=delete +else + newrev_type=$(git cat-file -t $newrev) +fi + +case "$refname","$newrev_type" in + refs/tags/*,commit) + # un-annotated tag + short_refname=${refname##refs/tags/} + if [ "$allowunannotated" != "true" ]; then + echo "*** The un-annotated tag, $short_refname, is not allowed in this repository" >&2 + echo "*** Use 'git tag [ -a | -s ]' for tags you want to propagate." >&2 + exit 1 + fi + ;; + refs/tags/*,delete) + # delete tag + if [ "$allowdeletetag" != "true" ]; then + echo "*** Deleting a tag is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/tags/*,tag) + # annotated tag + if [ "$allowmodifytag" != "true" ] && git rev-parse $refname > /dev/null 2>&1 + then + echo "*** Tag '$refname' already exists." >&2 + echo "*** Modifying a tag is not allowed in this repository." >&2 + exit 1 + fi + ;; + refs/heads/*,commit) + # branch + if [ "$oldrev" = "$zero" -a "$denycreatebranch" = "true" ]; then + echo "*** Creating a branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/heads/*,delete) + # delete branch + if [ "$allowdeletebranch" != "true" ]; then + echo "*** Deleting a branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/remotes/*,commit) + # tracking branch + ;; + refs/remotes/*,delete) + # delete tracking branch + if [ "$allowdeletebranch" != "true" ]; then + echo "*** Deleting a tracking branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + *) + # Anything else (is there anything else?) + echo "*** Update hook: unknown type of update to ref $refname of type $newrev_type" >&2 + exit 1 + ;; +esac + +# --- Finished +exit 0 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/index b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/index new file mode 100644 index 0000000000..c9459c6fa5 Binary files /dev/null and b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/index differ diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/info/exclude b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/info/exclude new file mode 100644 index 0000000000..a5196d1be8 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/info/exclude @@ -0,0 +1,6 @@ +# git ls-files --others --exclude-from=.git/info/exclude +# Lines that start with '#' are comments. +# For a project mostly in C, the following would be a good set of +# exclude patterns (uncomment them if you want to use them): +# *.[oa] +# *~ diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/logs/HEAD b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/logs/HEAD new file mode 100644 index 0000000000..87a3c02d99 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/logs/HEAD @@ -0,0 +1 @@ +0000000000000000000000000000000000000000 e5fc0d2678ec7b9bacf0bf514bac035fa371cb6e Winston Li 1471957665 +0100 commit (initial): Main diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/logs/refs/heads/master b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/logs/refs/heads/master new file mode 100644 index 0000000000..87a3c02d99 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/logs/refs/heads/master @@ -0,0 +1 @@ +0000000000000000000000000000000000000000 e5fc0d2678ec7b9bacf0bf514bac035fa371cb6e Winston Li 1471957665 +0100 commit (initial): Main diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/objects/8b/6f970d184c1e097e6e6bae9b0eb03fec7796bf b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/objects/8b/6f970d184c1e097e6e6bae9b0eb03fec7796bf new file mode 100644 index 0000000000..63b4550453 Binary files /dev/null and b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/objects/8b/6f970d184c1e097e6e6bae9b0eb03fec7796bf differ diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/objects/e5/fc0d2678ec7b9bacf0bf514bac035fa371cb6e b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/objects/e5/fc0d2678ec7b9bacf0bf514bac035fa371cb6e new file mode 100644 index 0000000000..13bccfaac3 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/objects/e5/fc0d2678ec7b9bacf0bf514bac035fa371cb6e @@ -0,0 +1,2 @@ +xA +1 @Q=E4Zu$pZ#s}AO/yVec 3Hpx@BYx"fJ}g]m\{ pĜRG]ΌdB4& \ No newline at end of file diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/objects/f2/7f21327e2f0f53e9d8afab217fedaeea6a1cee b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/objects/f2/7f21327e2f0f53e9d8afab217fedaeea6a1cee new file mode 100644 index 0000000000..d0cc2ae76e Binary files /dev/null and b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/objects/f2/7f21327e2f0f53e9d8afab217fedaeea6a1cee differ diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/refs/heads/master b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/refs/heads/master new file mode 100644 index 0000000000..d9abf312d9 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj1/DOTgit/refs/heads/master @@ -0,0 +1 @@ +e5fc0d2678ec7b9bacf0bf514bac035fa371cb6e diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/COMMIT_EDITMSG b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/COMMIT_EDITMSG new file mode 100644 index 0000000000..c098216e78 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/COMMIT_EDITMSG @@ -0,0 +1 @@ +Main diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/HEAD b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/HEAD new file mode 100644 index 0000000000..cb089cd89a --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/config b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/config new file mode 100644 index 0000000000..6c9406b7d9 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/config @@ -0,0 +1,7 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = false + logallrefupdates = true + ignorecase = true + precomposeunicode = true diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/description b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/description new file mode 100644 index 0000000000..498b267a8c --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/description @@ -0,0 +1 @@ +Unnamed repository; edit this file 'description' to name the repository. diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/applypatch-msg.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/applypatch-msg.sample new file mode 100755 index 0000000000..a5d7b84a67 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/applypatch-msg.sample @@ -0,0 +1,15 @@ +#!/bin/sh +# +# An example hook script to check the commit log message taken by +# applypatch from an e-mail message. +# +# The hook should exit with non-zero status after issuing an +# appropriate message if it wants to stop the commit. The hook is +# allowed to edit the commit message file. +# +# To enable this hook, rename this file to "applypatch-msg". + +. git-sh-setup +commitmsg="$(git rev-parse --git-path hooks/commit-msg)" +test -x "$commitmsg" && exec "$commitmsg" ${1+"$@"} +: diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/commit-msg.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/commit-msg.sample new file mode 100755 index 0000000000..b58d1184a9 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/commit-msg.sample @@ -0,0 +1,24 @@ +#!/bin/sh +# +# An example hook script to check the commit log message. +# Called by "git commit" with one argument, the name of the file +# that has the commit message. The hook should exit with non-zero +# status after issuing an appropriate message if it wants to stop the +# commit. The hook is allowed to edit the commit message file. +# +# To enable this hook, rename this file to "commit-msg". + +# Uncomment the below to add a Signed-off-by line to the message. +# Doing this in a hook is a bad idea in general, but the prepare-commit-msg +# hook is more suited to it. +# +# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') +# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1" + +# This example catches duplicate Signed-off-by lines. + +test "" = "$(grep '^Signed-off-by: ' "$1" | + sort | uniq -c | sed -e '/^[ ]*1[ ]/d')" || { + echo >&2 Duplicate Signed-off-by lines. + exit 1 +} diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/post-update.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/post-update.sample new file mode 100755 index 0000000000..ec17ec1939 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/post-update.sample @@ -0,0 +1,8 @@ +#!/bin/sh +# +# An example hook script to prepare a packed repository for use over +# dumb transports. +# +# To enable this hook, rename this file to "post-update". + +exec git update-server-info diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/pre-applypatch.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/pre-applypatch.sample new file mode 100755 index 0000000000..4142082bcb --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/pre-applypatch.sample @@ -0,0 +1,14 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed +# by applypatch from an e-mail message. +# +# The hook should exit with non-zero status after issuing an +# appropriate message if it wants to stop the commit. +# +# To enable this hook, rename this file to "pre-applypatch". + +. git-sh-setup +precommit="$(git rev-parse --git-path hooks/pre-commit)" +test -x "$precommit" && exec "$precommit" ${1+"$@"} +: diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/pre-commit.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/pre-commit.sample new file mode 100755 index 0000000000..68d62d5446 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/pre-commit.sample @@ -0,0 +1,49 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed. +# Called by "git commit" with no arguments. The hook should +# exit with non-zero status after issuing an appropriate message if +# it wants to stop the commit. +# +# To enable this hook, rename this file to "pre-commit". + +if git rev-parse --verify HEAD >/dev/null 2>&1 +then + against=HEAD +else + # Initial commit: diff against an empty tree object + against=4b825dc642cb6eb9a060e54bf8d69288fbee4904 +fi + +# If you want to allow non-ASCII filenames set this variable to true. +allownonascii=$(git config --bool hooks.allownonascii) + +# Redirect output to stderr. +exec 1>&2 + +# Cross platform projects tend to avoid non-ASCII filenames; prevent +# them from being added to the repository. We exploit the fact that the +# printable range starts at the space character and ends with tilde. +if [ "$allownonascii" != "true" ] && + # Note that the use of brackets around a tr range is ok here, (it's + # even required, for portability to Solaris 10's /usr/bin/tr), since + # the square bracket bytes happen to fall in the designated range. + test $(git diff --cached --name-only --diff-filter=A -z $against | + LC_ALL=C tr -d '[ -~]\0' | wc -c) != 0 +then + cat <<\EOF +Error: Attempt to add a non-ASCII file name. + +This can cause problems if you want to work with people on other platforms. + +To be portable it is advisable to rename the file. + +If you know what you are doing you can disable this check using: + + git config hooks.allownonascii true +EOF + exit 1 +fi + +# If there are whitespace errors, print the offending file names and fail. +exec git diff-index --check --cached $against -- diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/pre-push.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/pre-push.sample new file mode 100755 index 0000000000..6187dbf439 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/pre-push.sample @@ -0,0 +1,53 @@ +#!/bin/sh + +# An example hook script to verify what is about to be pushed. Called by "git +# push" after it has checked the remote status, but before anything has been +# pushed. If this script exits with a non-zero status nothing will be pushed. +# +# This hook is called with the following parameters: +# +# $1 -- Name of the remote to which the push is being done +# $2 -- URL to which the push is being done +# +# If pushing without using a named remote those arguments will be equal. +# +# Information about the commits which are being pushed is supplied as lines to +# the standard input in the form: +# +# +# +# This sample shows how to prevent push of commits where the log message starts +# with "WIP" (work in progress). + +remote="$1" +url="$2" + +z40=0000000000000000000000000000000000000000 + +while read local_ref local_sha remote_ref remote_sha +do + if [ "$local_sha" = $z40 ] + then + # Handle delete + : + else + if [ "$remote_sha" = $z40 ] + then + # New branch, examine all commits + range="$local_sha" + else + # Update to existing branch, examine new commits + range="$remote_sha..$local_sha" + fi + + # Check for WIP commit + commit=`git rev-list -n 1 --grep '^WIP' "$range"` + if [ -n "$commit" ] + then + echo >&2 "Found WIP commit in $local_ref, not pushing" + exit 1 + fi + fi +done + +exit 0 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/pre-rebase.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/pre-rebase.sample new file mode 100755 index 0000000000..9773ed4cb2 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/pre-rebase.sample @@ -0,0 +1,169 @@ +#!/bin/sh +# +# Copyright (c) 2006, 2008 Junio C Hamano +# +# The "pre-rebase" hook is run just before "git rebase" starts doing +# its job, and can prevent the command from running by exiting with +# non-zero status. +# +# The hook is called with the following parameters: +# +# $1 -- the upstream the series was forked from. +# $2 -- the branch being rebased (or empty when rebasing the current branch). +# +# This sample shows how to prevent topic branches that are already +# merged to 'next' branch from getting rebased, because allowing it +# would result in rebasing already published history. + +publish=next +basebranch="$1" +if test "$#" = 2 +then + topic="refs/heads/$2" +else + topic=`git symbolic-ref HEAD` || + exit 0 ;# we do not interrupt rebasing detached HEAD +fi + +case "$topic" in +refs/heads/??/*) + ;; +*) + exit 0 ;# we do not interrupt others. + ;; +esac + +# Now we are dealing with a topic branch being rebased +# on top of master. Is it OK to rebase it? + +# Does the topic really exist? +git show-ref -q "$topic" || { + echo >&2 "No such branch $topic" + exit 1 +} + +# Is topic fully merged to master? +not_in_master=`git rev-list --pretty=oneline ^master "$topic"` +if test -z "$not_in_master" +then + echo >&2 "$topic is fully merged to master; better remove it." + exit 1 ;# we could allow it, but there is no point. +fi + +# Is topic ever merged to next? If so you should not be rebasing it. +only_next_1=`git rev-list ^master "^$topic" ${publish} | sort` +only_next_2=`git rev-list ^master ${publish} | sort` +if test "$only_next_1" = "$only_next_2" +then + not_in_topic=`git rev-list "^$topic" master` + if test -z "$not_in_topic" + then + echo >&2 "$topic is already up-to-date with master" + exit 1 ;# we could allow it, but there is no point. + else + exit 0 + fi +else + not_in_next=`git rev-list --pretty=oneline ^${publish} "$topic"` + /usr/bin/perl -e ' + my $topic = $ARGV[0]; + my $msg = "* $topic has commits already merged to public branch:\n"; + my (%not_in_next) = map { + /^([0-9a-f]+) /; + ($1 => 1); + } split(/\n/, $ARGV[1]); + for my $elem (map { + /^([0-9a-f]+) (.*)$/; + [$1 => $2]; + } split(/\n/, $ARGV[2])) { + if (!exists $not_in_next{$elem->[0]}) { + if ($msg) { + print STDERR $msg; + undef $msg; + } + print STDERR " $elem->[1]\n"; + } + } + ' "$topic" "$not_in_next" "$not_in_master" + exit 1 +fi + +exit 0 + +################################################################ + +This sample hook safeguards topic branches that have been +published from being rewound. + +The workflow assumed here is: + + * Once a topic branch forks from "master", "master" is never + merged into it again (either directly or indirectly). + + * Once a topic branch is fully cooked and merged into "master", + it is deleted. If you need to build on top of it to correct + earlier mistakes, a new topic branch is created by forking at + the tip of the "master". This is not strictly necessary, but + it makes it easier to keep your history simple. + + * Whenever you need to test or publish your changes to topic + branches, merge them into "next" branch. + +The script, being an example, hardcodes the publish branch name +to be "next", but it is trivial to make it configurable via +$GIT_DIR/config mechanism. + +With this workflow, you would want to know: + +(1) ... if a topic branch has ever been merged to "next". Young + topic branches can have stupid mistakes you would rather + clean up before publishing, and things that have not been + merged into other branches can be easily rebased without + affecting other people. But once it is published, you would + not want to rewind it. + +(2) ... if a topic branch has been fully merged to "master". + Then you can delete it. More importantly, you should not + build on top of it -- other people may already want to + change things related to the topic as patches against your + "master", so if you need further changes, it is better to + fork the topic (perhaps with the same name) afresh from the + tip of "master". + +Let's look at this example: + + o---o---o---o---o---o---o---o---o---o "next" + / / / / + / a---a---b A / / + / / / / + / / c---c---c---c B / + / / / \ / + / / / b---b C \ / + / / / / \ / + ---o---o---o---o---o---o---o---o---o---o---o "master" + + +A, B and C are topic branches. + + * A has one fix since it was merged up to "next". + + * B has finished. It has been fully merged up to "master" and "next", + and is ready to be deleted. + + * C has not merged to "next" at all. + +We would want to allow C to be rebased, refuse A, and encourage +B to be deleted. + +To compute (1): + + git rev-list ^master ^topic next + git rev-list ^master next + + if these match, topic has not merged in next at all. + +To compute (2): + + git rev-list master..topic + + if this is empty, it is fully merged to "master". diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/prepare-commit-msg.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/prepare-commit-msg.sample new file mode 100755 index 0000000000..f093a02ec4 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/prepare-commit-msg.sample @@ -0,0 +1,36 @@ +#!/bin/sh +# +# An example hook script to prepare the commit log message. +# Called by "git commit" with the name of the file that has the +# commit message, followed by the description of the commit +# message's source. The hook's purpose is to edit the commit +# message file. If the hook fails with a non-zero status, +# the commit is aborted. +# +# To enable this hook, rename this file to "prepare-commit-msg". + +# This hook includes three examples. The first comments out the +# "Conflicts:" part of a merge commit. +# +# The second includes the output of "git diff --name-status -r" +# into the message, just before the "git status" output. It is +# commented because it doesn't cope with --amend or with squashed +# commits. +# +# The third example adds a Signed-off-by line to the message, that can +# still be edited. This is rarely a good idea. + +case "$2,$3" in + merge,) + /usr/bin/perl -i.bak -ne 's/^/# /, s/^# #/#/ if /^Conflicts/ .. /#/; print' "$1" ;; + +# ,|template,) +# /usr/bin/perl -i.bak -pe ' +# print "\n" . `git diff --cached --name-status -r` +# if /^#/ && $first++ == 0' "$1" ;; + + *) ;; +esac + +# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') +# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1" diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/update.sample b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/update.sample new file mode 100755 index 0000000000..80ba94135c --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/hooks/update.sample @@ -0,0 +1,128 @@ +#!/bin/sh +# +# An example hook script to block unannotated tags from entering. +# Called by "git receive-pack" with arguments: refname sha1-old sha1-new +# +# To enable this hook, rename this file to "update". +# +# Config +# ------ +# hooks.allowunannotated +# This boolean sets whether unannotated tags will be allowed into the +# repository. By default they won't be. +# hooks.allowdeletetag +# This boolean sets whether deleting tags will be allowed in the +# repository. By default they won't be. +# hooks.allowmodifytag +# This boolean sets whether a tag may be modified after creation. By default +# it won't be. +# hooks.allowdeletebranch +# This boolean sets whether deleting branches will be allowed in the +# repository. By default they won't be. +# hooks.denycreatebranch +# This boolean sets whether remotely creating branches will be denied +# in the repository. By default this is allowed. +# + +# --- Command line +refname="$1" +oldrev="$2" +newrev="$3" + +# --- Safety check +if [ -z "$GIT_DIR" ]; then + echo "Don't run this script from the command line." >&2 + echo " (if you want, you could supply GIT_DIR then run" >&2 + echo " $0 )" >&2 + exit 1 +fi + +if [ -z "$refname" -o -z "$oldrev" -o -z "$newrev" ]; then + echo "usage: $0 " >&2 + exit 1 +fi + +# --- Config +allowunannotated=$(git config --bool hooks.allowunannotated) +allowdeletebranch=$(git config --bool hooks.allowdeletebranch) +denycreatebranch=$(git config --bool hooks.denycreatebranch) +allowdeletetag=$(git config --bool hooks.allowdeletetag) +allowmodifytag=$(git config --bool hooks.allowmodifytag) + +# check for no description +projectdesc=$(sed -e '1q' "$GIT_DIR/description") +case "$projectdesc" in +"Unnamed repository"* | "") + echo "*** Project description file hasn't been set" >&2 + exit 1 + ;; +esac + +# --- Check types +# if $newrev is 0000...0000, it's a commit to delete a ref. +zero="0000000000000000000000000000000000000000" +if [ "$newrev" = "$zero" ]; then + newrev_type=delete +else + newrev_type=$(git cat-file -t $newrev) +fi + +case "$refname","$newrev_type" in + refs/tags/*,commit) + # un-annotated tag + short_refname=${refname##refs/tags/} + if [ "$allowunannotated" != "true" ]; then + echo "*** The un-annotated tag, $short_refname, is not allowed in this repository" >&2 + echo "*** Use 'git tag [ -a | -s ]' for tags you want to propagate." >&2 + exit 1 + fi + ;; + refs/tags/*,delete) + # delete tag + if [ "$allowdeletetag" != "true" ]; then + echo "*** Deleting a tag is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/tags/*,tag) + # annotated tag + if [ "$allowmodifytag" != "true" ] && git rev-parse $refname > /dev/null 2>&1 + then + echo "*** Tag '$refname' already exists." >&2 + echo "*** Modifying a tag is not allowed in this repository." >&2 + exit 1 + fi + ;; + refs/heads/*,commit) + # branch + if [ "$oldrev" = "$zero" -a "$denycreatebranch" = "true" ]; then + echo "*** Creating a branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/heads/*,delete) + # delete branch + if [ "$allowdeletebranch" != "true" ]; then + echo "*** Deleting a branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/remotes/*,commit) + # tracking branch + ;; + refs/remotes/*,delete) + # delete tracking branch + if [ "$allowdeletebranch" != "true" ]; then + echo "*** Deleting a tracking branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + *) + # Anything else (is there anything else?) + echo "*** Update hook: unknown type of update to ref $refname of type $newrev_type" >&2 + exit 1 + ;; +esac + +# --- Finished +exit 0 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/index b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/index new file mode 100644 index 0000000000..9756dde938 Binary files /dev/null and b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/index differ diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/info/exclude b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/info/exclude new file mode 100644 index 0000000000..a5196d1be8 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/info/exclude @@ -0,0 +1,6 @@ +# git ls-files --others --exclude-from=.git/info/exclude +# Lines that start with '#' are comments. +# For a project mostly in C, the following would be a good set of +# exclude patterns (uncomment them if you want to use them): +# *.[oa] +# *~ diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/logs/HEAD b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/logs/HEAD new file mode 100644 index 0000000000..2c05a99cac --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/logs/HEAD @@ -0,0 +1 @@ +0000000000000000000000000000000000000000 6c12c073e5702530a9d06b83840d62f8a6621764 Winston Li 1471957694 +0100 commit (initial): Main diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/logs/refs/heads/master b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/logs/refs/heads/master new file mode 100644 index 0000000000..2c05a99cac --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/logs/refs/heads/master @@ -0,0 +1 @@ +0000000000000000000000000000000000000000 6c12c073e5702530a9d06b83840d62f8a6621764 Winston Li 1471957694 +0100 commit (initial): Main diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/objects/6c/12c073e5702530a9d06b83840d62f8a6621764 b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/objects/6c/12c073e5702530a9d06b83840d62f8a6621764 new file mode 100644 index 0000000000..00a6b21dae --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/objects/6c/12c073e5702530a9d06b83840d62f8a6621764 @@ -0,0 +1,2 @@ +x= +1@abzAfH&! ::&x}AO`W溊qngrEϘr.km&FJ٤WOǛ!' (r єL۟\ _2 \ No newline at end of file diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/objects/7f/37654ebf6d0a19650abbcf5db3953b15001d1b b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/objects/7f/37654ebf6d0a19650abbcf5db3953b15001d1b new file mode 100644 index 0000000000..f77bc41b90 Binary files /dev/null and b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/objects/7f/37654ebf6d0a19650abbcf5db3953b15001d1b differ diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/objects/d8/5308af36ff394df8bf063719b2aea26077aaea b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/objects/d8/5308af36ff394df8bf063719b2aea26077aaea new file mode 100644 index 0000000000..433ce0d1fc Binary files /dev/null and b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/objects/d8/5308af36ff394df8bf063719b2aea26077aaea differ diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/refs/heads/master b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/refs/heads/master new file mode 100644 index 0000000000..4b8bababa2 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest/rootdir/proj2/DOTgit/refs/heads/master @@ -0,0 +1 @@ +6c12c073e5702530a9d06b83840d62f8a6621764 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/file1 b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/file1 new file mode 100644 index 0000000000..e2129701f1 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/file1 @@ -0,0 +1 @@ +file1 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/file2 b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/file2 new file mode 100644 index 0000000000..6c493ff740 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/file2 @@ -0,0 +1 @@ +file2 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file1 b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file1 new file mode 100644 index 0000000000..5c441e6377 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file1 @@ -0,0 +1 @@ +nest1/file1 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file2 b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file2 new file mode 100644 index 0000000000..2b7361bb7e --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file2 @@ -0,0 +1 @@ +nest1file2 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file3 b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file3 new file mode 100644 index 0000000000..ac69189828 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file3 @@ -0,0 +1 @@ +nest1/file3 diff --git a/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/nest2/file1 b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/nest2/file1 new file mode 100644 index 0000000000..5e92e63c44 --- /dev/null +++ b/services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/nest2/file1 @@ -0,0 +1 @@ +nest1/nest2/file1 diff --git a/services/git-bridge/writelatex-git-bridge.iml b/services/git-bridge/writelatex-git-bridge.iml deleted file mode 100644 index 87a4714796..0000000000 --- a/services/git-bridge/writelatex-git-bridge.iml +++ /dev/null @@ -1,113 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file