agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-06-01 13:21:37 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add implementations, implement S3SwapStore (with only tars), FSRepoStore, Tar and File utils, add tests

Browse Source
This commit is contained in:
Winston Li
2016-08-23 17:51:56 +01:00
committed by Michael Mazour
parent 1850689a63
commit 8c0937511e
32 changed files with 848 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java
View File

@@ -30,6 +30,7 @@ import uk.ac.ic.wlgitbridge.snapshot.push.exception.*;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.IOException;
import java.time.Duration;
import java.util.*;
/**
@@ -104,8 +105,8 @@ public class Bridge {
Log.info("Bye");
}
public void startSwapJob(int intervalMillis) {
swapJob.start(intervalMillis);
public void startSwapJob(Duration interval) {
swapJob.start(interval);
}
/* TODO: Remove these when WLBridged is moved into RepoStore */

10
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java
View File

@@ -1,5 +1,6 @@
package uk.ac.ic.wlgitbridge.bridge.db;
import java.sql.Timestamp;
import java.util.List;
/**
@@ -19,4 +20,13 @@ public interface DBStore {
String getPathForURLInProject(String projectName, String url);
String getOldestUnswappedProject();
/**
* Sets the last accessed time for the given project name.
* @param projectName the project's name
* @param time the time, or null if the project is to be swapped
*/
void setLastAccessedTime(String projectName, Timestamp time);
}

11
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/SqliteDBStore.java
View File

@@ -5,6 +5,7 @@ import uk.ac.ic.wlgitbridge.util.Log;
import java.io.File;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.Arrays;
import java.util.List;
@@ -84,4 +85,14 @@ public class SqliteDBStore implements DBStore {
}
}
@Override
public String getOldestUnswappedProject() {
throw new UnsupportedOperationException();
}
@Override
public void setLastAccessedTime(String projectName, Timestamp time) {
throw new UnsupportedOperationException();
}
}

61
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java
View File

@@ -1,12 +1,20 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import uk.ac.ic.wlgitbridge.util.Util;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import uk.ac.ic.wlgitbridge.util.Project;
import uk.ac.ic.wlgitbridge.util.Tar;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import static uk.ac.ic.wlgitbridge.util.Util.deleteInDirectoryApartFrom;
/**
* Created by winston on 20/08/2016.
*/
@@ -30,6 +38,7 @@ public class FSRepoStore implements RepoStore {
return rootDirectory;
}
/* TODO: Perhaps we should just delete bad directories on the fly. */
@Override
public void purgeNonexistentProjects(
Collection<String> existingProjectNames
@@ -37,15 +46,63 @@ public class FSRepoStore implements RepoStore {
List<String> excludedFromDeletion =
new ArrayList<>(existingProjectNames);
excludedFromDeletion.add(".wlgb");
Util.deleteInDirectoryApartFrom(
deleteInDirectoryApartFrom(
rootDirectory,
excludedFromDeletion.toArray(new String[] {})
);
}
@Override
public long totalSize() {
return FileUtils.sizeOfDirectory(rootDirectory);
}
@Override
public InputStream bzip2Project(String projectName) throws IOException {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
return Tar.tar(getDotGitForProject(projectName));
}
@Override
public void remove(String projectName) throws IOException {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
FileUtils.deleteDirectory(new File(rootDirectory, projectName));
}
@Override
public void unbzip2Project(
String projectName,
InputStream dataStream
) throws IOException {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
Preconditions.checkState(getDirForProject(projectName).mkdirs());
Tar.untar(dataStream, getDirForProject(projectName));
}
private File getDirForProject(String projectName) {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
return Paths.get(
rootDirectory.getAbsolutePath()
).resolve(
projectName
).toFile();
}
private File getDotGitForProject(String projectName) {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
return Paths.get(
rootDirectory.getAbsolutePath()
).resolve(
projectName
).resolve(
".git"
).toFile();
}
private File initRootGitDirectory(String rootGitDirectoryPath) {
File rootGitDirectory = new File(rootGitDirectoryPath);
rootGitDirectory.mkdirs();
Preconditions.checkArgument(rootGitDirectory.isDirectory());
return rootGitDirectory;
}

25
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java
View File

@@ -1,8 +1,8 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
/**
@@ -14,9 +14,28 @@ public interface RepoStore {
File getRootDirectory();
void purgeNonexistentProjects(
Collection<String> existingProjectNames
);
long totalSize();
/*
* Tars and bzip2s the .git directory of the given project. Throws an
* IOException if the project doesn't exist. The returned stream is a copy
* of the original .git directory, which must be deleted using remove().
*/
InputStream bzip2Project(String projectName) throws IOException;
void remove(String projectName) throws IOException;
/**
* Unbzip2s the given data stream into a .git directory for projectName.
* Creates the project directory.
* If projectName already exists, throws an IOException.
* @param projectName the name of the project, e.g. abc123
* @param dataStream the data stream containing the bzipped contents.
*/
void unbzip2Project(String projectName, InputStream dataStream) throws IOException;
}

64
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStore.java Normal file
View File

@@ -0,0 +1,64 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.*;
import java.io.InputStream;
/**
* Created by winston on 21/08/2016.
*/
public class S3SwapStore implements SwapStore {
private final AmazonS3 s3;
private final String bucketName;
public S3SwapStore(
String accessKey,
String secret,
String bucketName
) {
s3 = new AmazonS3Client(new BasicAWSCredentials(accessKey, secret));
this.bucketName = bucketName;
}
@Override
public void upload(
String projectName,
InputStream uploadStream,
long contentLength
) {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(contentLength);
PutObjectRequest put = new PutObjectRequest(
bucketName,
projectName,
uploadStream,
metadata
);
PutObjectResult res = s3.putObject(put);
}
@Override
public InputStream openDownloadStream(String projectName) {
GetObjectRequest get = new GetObjectRequest(
bucketName,
projectName
);
S3Object res = s3.getObject(get);
return res.getObjectContent();
}
@Override
public void remove(String projectName) {
DeleteObjectRequest del = new DeleteObjectRequest(
bucketName,
projectName
);
s3.deleteObject(del);
}
}

4
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java
View File

@@ -1,11 +1,13 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
import java.time.Duration;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapJob {
void start(int intervalMillis);
void start(Duration interval);
void stop();

17
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java
View File

@@ -3,9 +3,10 @@ package uk.ac.ic.wlgitbridge.bridge.swap;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.util.Util;
import java.time.Duration;
import java.util.Timer;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Created by winston on 20/08/2016.
@@ -19,10 +20,13 @@ public class SwapJobImpl implements SwapJob {
private final Timer timer;
final AtomicInteger swaps;
public SwapJobImpl(
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore, SwapStore swapStore
DBStore dbStore,
SwapStore swapStore
) {
this.lock = lock;
@@ -30,14 +34,15 @@ public class SwapJobImpl implements SwapJob {
this.swapStore = swapStore;
this.dbStore = dbStore;
timer = new Timer();
swaps = new AtomicInteger(0);
}
@Override
public void start(int intervalMillis) {
public void start(Duration interval) {
timer.scheduleAtFixedRate(
Util.makeTimerTask(this::doSwap),
uk.ac.ic.wlgitbridge.util.Timer.makeTimerTask(this::doSwap),
0,
intervalMillis
interval.toMillis()
);
}
@@ -47,7 +52,7 @@ public class SwapJobImpl implements SwapJob {
}
private void doSwap() {
throw new UnsupportedOperationException();
swaps.incrementAndGet();
}
}

9
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java
View File

@@ -1,7 +1,16 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
import java.io.InputStream;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapStore {
void upload(String projectName, InputStream uploadStream, long contentLength);
InputStream openDownloadStream(String projectName);
void remove(String projectName);
}

18
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java
View File

@@ -24,6 +24,7 @@ import javax.servlet.DispatcherType;
import javax.servlet.Filter;
import javax.servlet.ServletException;
import java.io.File;
import java.io.InputStream;
import java.net.BindException;
import java.util.EnumSet;
@@ -50,7 +51,22 @@ public class GitBridgeServer {
this.rootGitDirectoryPath = config.getRootGitDirectory();
RepoStore repoStore = new FSRepoStore(rootGitDirectoryPath);
DBStore dbStore = new SqliteDBStore(repoStore.getRootDirectory());
SwapStore swapStore = new SwapStore() {};
SwapStore swapStore = new SwapStore() {
@Override
public void upload(String projectName, InputStream uploadStream, long contentLength) {
}
@Override
public InputStream openDownloadStream(String projectName) {
return null;
}
@Override
public void remove(String projectName) {
}
};
bridgeAPI = Bridge.make(
repoStore,
dbStore,

127
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Files.java Normal file
View File

@@ -0,0 +1,127 @@
package uk.ac.ic.wlgitbridge.util;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.TrueFileFilter;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Created by winston on 23/08/2016.
*/
public class Files {
private Files() {}
public static boolean contentsAreEqual(
File f0,
File f1
) throws IOException {
try {
return uncheckedContentsAreEqual(f0, f1);
} catch (UncheckedIOException e) {
throw e.getCause();
}
}
public static void renameAll(
File fileOrDir,
String from,
String to
) {
if (fileOrDir.isDirectory()) {
File f = doRename(fileOrDir, from, to);
for (File c : f.listFiles()) {
renameAll(c, from, to);
}
} else if (fileOrDir.isFile()) {
doRename(fileOrDir, from, to);
} else {
throw new IllegalArgumentException(
"not a file or dir: " + fileOrDir
);
}
}
private static File doRename(File fileOrDir, String from, String to) {
if (!fileOrDir.getName().equals(from)) {
return fileOrDir;
}
File renamed = new File(fileOrDir.getParent(), to);
Preconditions.checkState(fileOrDir.renameTo(renamed));
return renamed;
}
private static boolean uncheckedContentsAreEqual(File f0, File f1) throws IOException {
if (f0.equals(f1)) {
return true;
}
if (!f0.isDirectory() || !f1.isDirectory()) {
return !f0.isDirectory() && !f1.isDirectory() &&
Arrays.equals(
FileUtils.readFileToByteArray(f0),
FileUtils.readFileToByteArray(f1)
);
}
Path f0Base = Paths.get(f0.getAbsolutePath());
Path f1Base = Paths.get(f1.getAbsolutePath());
Set<Path> children0 = getChildren(f0, f0Base);
Set<Path> children1 = getChildren(f1, f1Base);
if (children0.size() != children1.size()) {
return false;
}
return children0.stream(
).allMatch(c0 ->
children1.contains(c0) && childEquals(c0, f0Base, f1Base)
);
}
private static Set<Path> getChildren(File f0, Path f0Base) {
return FileUtils.listFilesAndDirs(
f0,
TrueFileFilter.TRUE,
TrueFileFilter.TRUE
).stream(
).map(
File::getAbsolutePath
).map(
Paths::get
).map(p ->
f0Base.relativize(p)
).filter(p ->
!p.toString().isEmpty()
).collect(
Collectors.toSet()
);
}
private static boolean childEquals(
Path child,
Path f0Base,
Path f1Base
) throws UncheckedIOException {
File c0 = f0Base.resolve(child).toFile();
File c1 = f1Base.resolve(child).toFile();
boolean c0IsDir = c0.isDirectory();
boolean c1IsDir = c1.isDirectory();
if (c0IsDir || c1IsDir) {
return c0IsDir && c1IsDir;
}
try {
return c0.isFile() && c1.isFile() && Arrays.equals(
FileUtils.readFileToByteArray(c0),
FileUtils.readFileToByteArray(c1)
);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}

13
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Project.java Normal file
View File

@@ -0,0 +1,13 @@
package uk.ac.ic.wlgitbridge.util;
/**
* Created by winston on 23/08/2016.
*/
public class Project {
public static boolean isValidProjectName(String projectName) {
return projectName != null && !projectName.isEmpty()
&& !projectName.startsWith(".");
}
}

106
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java Normal file
View File

@@ -0,0 +1,106 @@
package uk.ac.ic.wlgitbridge.util;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.utils.IOUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.output.ByteArrayOutputStream;
import java.io.*;
import java.nio.file.Path;
import java.nio.file.Paths;
/**
* Created by winston on 23/08/2016.
*/
public class Tar {
private Tar() {}
public static InputStream tar(File fileOrDir) throws IOException {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
TarArchiveOutputStream tout = new TarArchiveOutputStream(bout);
addTarEntry(
tout,
Paths.get(fileOrDir.getParentFile().getAbsolutePath()),
fileOrDir
);
tout.close();
return new ByteArrayInputStream(bout.toByteArray());
}
private static void addTarEntry(
TarArchiveOutputStream tout,
Path base,
File fileOrDir
) throws IOException {
if (fileOrDir.isDirectory()) {
addTarDir(tout, base, fileOrDir);
} else if (fileOrDir.isFile()) {
addTarFile(tout, base, fileOrDir);
} else {
throw new IllegalArgumentException(
"invalid file or dir: " + fileOrDir
);
}
}
private static void addTarDir(
TarArchiveOutputStream tout,
Path base,
File dir
) throws IOException {
Preconditions.checkArgument(dir.isDirectory());
String name = base.relativize(
Paths.get(dir.getAbsolutePath())
).toString();
ArchiveEntry entry = tout.createArchiveEntry(dir, name);
tout.putArchiveEntry(entry);
tout.closeArchiveEntry();
for (File f : dir.listFiles()) {
addTarEntry(tout, base, f);
}
}
private static void addTarFile(
TarArchiveOutputStream tout,
Path base,
File file
) throws IOException {
Preconditions.checkArgument(file.isFile());
String name = base.relativize(
Paths.get(file.getAbsolutePath())
).toString();
ArchiveEntry entry = tout.createArchiveEntry(file, name);
tout.putArchiveEntry(entry);
tout.write(FileUtils.readFileToByteArray(file));
tout.closeArchiveEntry();
}
public static void untar(InputStream tar, File parentDir) throws IOException {
TarArchiveInputStream tin = new TarArchiveInputStream(tar);
ArchiveEntry e;
while ((e = tin.getNextEntry()) != null) {
File f = new File(parentDir, e.getName());
f.setLastModified(e.getLastModifiedDate().getTime());
f.getParentFile().mkdirs();
if (e.isDirectory()) {
f.mkdir();
continue;
}
long size = e.getSize();
Preconditions.checkArgument(
size > 0 && size < Integer.MAX_VALUE,
"file too big: tar should have thrown an IOException"
);
try (OutputStream out = new FileOutputStream(f)) {
/* TarInputStream pretends each
entry's EOF is the stream's EOF */
IOUtils.copy(tin, out);
}
}
}
}

19
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Timer.java Normal file
View File

@@ -0,0 +1,19 @@
package uk.ac.ic.wlgitbridge.util;
import java.util.TimerTask;
/**
* Created by winston on 23/08/2016.
*/
public class Timer {
public static TimerTask makeTimerTask(Runnable lamb) {
return new TimerTask() {
@Override
public void run() {
lamb.run();
}
};
}
}

9
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Util.java
View File

@@ -19,15 +19,6 @@ public class Util {
private static String POSTBACK_URL;
private static final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSS");
public static TimerTask makeTimerTask(Runnable lamb) {
return new TimerTask() {
@Override
public void run() {
lamb.run();
}
};
}
public static String entries(int entries) {
if (entries == 1) {
return "entry";