agw/overleaf-cep
1
0
Fork 0
mirror of https://github.com/yu-i-i/overleaf-cep.git synced 2026-11-13 21:53:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #67 from overleaf/jpa-offline-s3

Browse Source 
Add support for offline s3 tests
This commit is contained in:
Jakob Ackermann
2020-01-31 15:25:53 +01:00
committed by GitHub
parent 9f2e36d3d5 ca668ad657
commit e0d3b3fe17
12 changed files with 116 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/track-changes/Dockerfile
+5
View File
@@ -1,3 +1,8 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
# Version: 1.3.4
FROM node:6.11.2 as app
WORKDIR /app
services/track-changes/Jenkinsfile
Vendored
+2 -3
View File
@@ -16,6 +16,7 @@ pipeline {
}
stages {
stage('Install') {
steps {
withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
@@ -44,9 +45,7 @@ pipeline {
stage('Acceptance Tests') {
steps {
withCredentials([usernamePassword(credentialsId: 'S3_DOCSTORE_TEST_AWS_KEYS', passwordVariable: 'AWS_SECRET_ACCESS_KEY', usernameVariable: 'AWS_ACCESS_KEY_ID')]) {
sh 'AWS_BUCKET="sl-acceptance-tests" AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance'
}
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance'
}
}
services/track-changes/Makefile
+8 -4
View File
@@ -1,7 +1,7 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
# Version: 1.1.24
# Version: 1.3.4
BUILD_NUMBER ?= local
BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
@@ -11,9 +11,6 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
BRANCH_NAME=$(BRANCH_NAME) \
PROJECT_NAME=$(PROJECT_NAME) \
MOCHA_GREP=${MOCHA_GREP} \
AWS_BUCKET=${AWS_BUCKET} \
AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
docker-compose ${DOCKER_COMPOSE_FLAGS}
clean:
@@ -31,14 +28,20 @@ test_unit:
test_acceptance: test_clean test_acceptance_pre_run test_acceptance_run
test_acceptance_debug: test_clean test_acceptance_pre_run test_acceptance_run_debug
test_acceptance_run:
@[ ! -d test/acceptance ] && echo "track-changes has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance
test_acceptance_run_debug:
@[ ! -d test/acceptance ] && echo "track-changes has no acceptance tests" || $(DOCKER_COMPOSE) run -p 127.0.0.9:19999:19999 --rm test_acceptance npm run test:acceptance -- --inspect=0.0.0.0:19999 --inspect-brk
test_clean:
$(DOCKER_COMPOSE) down -v -t 0
test_acceptance_pre_run:
@[ ! -f test/acceptance/js/scripts/pre-run ] && echo "track-changes has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
build:
docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
--tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
@@ -51,4 +54,5 @@ publish:
docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
.PHONY: clean test test_unit test_acceptance test_clean build publish
services/track-changes/app/coffee/MongoAWS.coffee
+2
View File
@@ -14,6 +14,8 @@ createStream = (streamConstructor, project_id, doc_id, pack_id) ->
AWS_CONFIG =
accessKeyId: settings.trackchanges.s3.key
secretAccessKey: settings.trackchanges.s3.secret
endpoint: settings.trackchanges.s3.endpoint
s3ForcePathStyle: settings.trackchanges.s3.pathStyle
return streamConstructor new AWS.S3(AWS_CONFIG), {
"Bucket": settings.trackchanges.stores.doc_history,
services/track-changes/buildscript.txt
+8 -8
View File
@@ -1,10 +1,10 @@
track-changes
--language=coffeescript
--node-version=6.11.2
--acceptance-creds=aws
--dependencies=mongo,redis
--docker-repos=gcr.io/overleaf-ops
--build-target=docker
--script-version=1.1.24
--env-pass-through=
--public-repo=True
--language=coffeescript
--env-add=AWS_BUCKET=bucket
--node-version=6.11.2
--acceptance-creds=None
--dependencies=mongo,redis,s3
--docker-repos=gcr.io/overleaf-ops
--env-pass-through=
--script-version=1.3.4
services/track-changes/config/settings.defaults.coffee
+2
View File
@@ -38,6 +38,8 @@ module.exports =
s3:
key: process.env['AWS_ACCESS_KEY_ID']
secret: process.env['AWS_SECRET_ACCESS_KEY']
endpoint: process.env['AWS_S3_ENDPOINT']
pathStyle: process.env['AWS_S3_PATH_STYLE'] == 'true'
stores:
doc_history: process.env['AWS_BUCKET']
continueOnError: process.env['TRACK_CHANGES_CONTINUE_ON_ERROR'] or false
services/track-changes/docker-compose.ci.yml
+20 -10
View File
@@ -1,9 +1,9 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
# Version: 1.1.24
# Version: 1.3.4
version: "2"
version: "2.1"
services:
test_unit:
@@ -22,19 +22,24 @@ services:
REDIS_HOST: redis
MONGO_HOST: mongo
POSTGRES_HOST: postgres
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
AWS_BUCKET: ${AWS_BUCKET}
AWS_S3_ENDPOINT: http://s3:9090
AWS_S3_PATH_STYLE: 'true'
AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake
MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: test
AWS_BUCKET: bucket
depends_on:
- mongo
- redis
mongo:
condition: service_healthy
redis:
condition: service_healthy
s3:
condition: service_healthy
user: node
command: npm run test:acceptance:_run
tar:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
@@ -42,9 +47,14 @@ services:
- ./:/tmp/build/
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
redis:
image: redis
mongo:
image: mongo:3.4
image: mongo:3.6
s3:
image: adobe/s3mock
environment:
- initialBuckets=fake_user_files,fake_template_files,fake_public_files,bucket
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9090"]
services/track-changes/docker-compose.yml
+21 -20
View File
@@ -1,9 +1,9 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
# Version: 1.1.24
# Version: 1.3.4
version: "2"
version: "2.1"
services:
test_unit:
@@ -18,7 +18,7 @@ services:
user: node
test_acceptance:
build: .
image: node:6.11.2
volumes:
- .:/app
working_dir: /app
@@ -27,32 +27,33 @@ services:
REDIS_HOST: redis
MONGO_HOST: mongo
POSTGRES_HOST: postgres
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
AWS_BUCKET: ${AWS_BUCKET}
AWS_S3_ENDPOINT: http://s3:9090
AWS_S3_PATH_STYLE: 'true'
AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake
MOCHA_GREP: ${MOCHA_GREP}
LOG_LEVEL: ERROR
NODE_ENV: test
AWS_BUCKET: bucket
user: node
depends_on:
- mongo
- redis
mongo:
condition: service_healthy
redis:
condition: service_healthy
s3:
condition: service_healthy
command: npm run test:acceptance
tar:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
volumes:
- ./:/tmp/build/
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
redis:
image: redis
mongo:
image: mongo:3.4
image: mongo:3.6
s3:
image: adobe/s3mock
environment:
- initialBuckets=fake_user_files,fake_template_files,fake_public_files,bucket
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9090"]
services/track-changes/package.json
+1 -1
View File
@@ -9,7 +9,7 @@
"scripts": {
"compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
"start": "npm run compile:app && node $NODE_APP_OPTIONS app.js",
"test:acceptance:_run": "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY AWS_BUCKET=$AWS_BUCKET AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js",
"test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
"test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP",
services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee
+4 -1
View File
@@ -17,9 +17,12 @@ MockWebApi = require "./helpers/MockWebApi"
describe "Archiving updates", ->
before (done) ->
if Settings?.trackchanges?.s3?.key.length < 1
message = "s3 keys not setup, this test setup will fail"
message = new Error("s3 keys not setup, this test setup will fail")
return done(message)
TrackChangesClient.waitForS3 done
before (done) ->
@now = Date.now()
@to = @now
@user_id = ObjectId().toString()
services/track-changes/test/acceptance/coffee/helpers/TrackChangesApp.coffee
+1 -1
View File
@@ -1,5 +1,5 @@
app = require('../../../../app')
require("logger-sharelatex").logger.level("info")
require("logger-sharelatex")
logger = require("logger-sharelatex")
Settings = require("settings-sharelatex")
services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee
+42 -20
View File
@@ -6,6 +6,15 @@ rclient = require("redis-sharelatex").createClient(Settings.redis.history) # Onl
Keys = Settings.redis.history.key_schema
{db, ObjectId} = require "../../../../app/js/mongojs"
aws = require "aws-sdk"
s3 = new aws.S3(
accessKeyId: Settings.trackchanges.s3.key
secretAccessKey: Settings.trackchanges.s3.secret
endpoint: Settings.trackchanges.s3.endpoint
s3ForcePathStyle: Settings.trackchanges.s3.pathStyle
)
S3_BUCKET = Settings.trackchanges.stores.doc_history
module.exports = TrackChangesClient =
flushAndGetCompressedUpdates: (project_id, doc_id, callback = (error, updates) ->) ->
TrackChangesClient.flushDoc project_id, doc_id, (error) ->
@@ -91,32 +100,45 @@ module.exports = TrackChangesClient =
response.statusCode.should.equal 204
callback(error)
buildS3Options: (content, key)->
return {
aws:
key: Settings.trackchanges.s3.key
secret: Settings.trackchanges.s3.secret
bucket: Settings.trackchanges.stores.doc_history
timeout: 30 * 1000
json: content
uri:"https://#{Settings.trackchanges.stores.doc_history}.s3.amazonaws.com/#{key}"
}
waitForS3: (done, retries=42) ->
if !Settings.trackchanges.s3.endpoint
return done()
request.get "#{Settings.trackchanges.s3.endpoint}/", (err, res) ->
if res && res.statusCode < 500
return done()
if retries == 0
return done(err or new Error("s3 returned #{res.statusCode}"))
setTimeout () ->
TrackChangesClient.waitForS3(done, --retries)
, 1000
getS3Doc: (project_id, doc_id, pack_id, callback = (error, body) ->) ->
options = TrackChangesClient.buildS3Options(true, project_id+"/changes-"+doc_id+"/pack-"+pack_id)
options.encoding = null
request.get options, (err, res, body) ->
params =
Bucket: S3_BUCKET
Key: "#{project_id}/changes-#{doc_id}/pack-#{pack_id}"
s3.getObject params, (error, data) ->
return callback(error) if error?
body = data.Body
return callback(new Error("empty response from s3")) if not body?
zlib.gunzip body, (err, result) ->
return callback(err) if err?
callback(null, JSON.parse(result.toString()))
removeS3Doc: (project_id, doc_id, callback = (error, res, body) ->) ->
options = TrackChangesClient.buildS3Options(true, "?prefix=" + project_id + "/changes-" +doc_id)
request.get options, (error, res, body) ->
keys = body.match /[0-9a-f]{24}\/changes-[0-9a-f]{24}\/pack-[0-9a-f]{24}/g
async.eachSeries keys, (key, cb) ->
options = TrackChangesClient.buildS3Options(true, key)
request.del options, cb
, callback
params =
Bucket: S3_BUCKET
Prefix: "#{project_id}/changes-#{doc_id}"
s3.listObjects params, (error, data) ->
return callback(error) if error?
params =
Bucket: S3_BUCKET
Delete:
Objects: data.Contents.map((s3object) -> {Key: s3object.Key})
s3.deleteObjects params, callback