From 868bed0ed933a2726a090c92e0add54b9226db8a Mon Sep 17 00:00:00 2001 From: David Rotermund Date: Sun, 25 Oct 2026 16:37:55 +0000 Subject: [PATCH] Out of the box S3 server (since minio is dead, I use seaweedfs) --- tools/s3_server_compose/.gitignore | 2 + .../0_pull_other_containers.sh | 1 + tools/s3_server_compose/1_install_aws.sh | 2 + .../s3_server_compose/2_create_s3_buckets.sh | 41 +++++ tools/s3_server_compose/3_online_backup_s3.sh | 36 ++++ tools/s3_server_compose/README.md | 169 ++++++++++++++++++ .../autoupdate_seaweedfs-filer.sh | 44 +++++ .../autoupdate_seaweedfs-master.sh | 44 +++++ .../autoupdate_seaweedfs-s3.sh | 44 +++++ .../autoupdate_seaweedfs-volume.sh | 44 +++++ tools/s3_server_compose/compose.yaml | 32 ++++ tools/s3_server_compose/down.sh | 2 + .../s3_server_compose/down_seaweedfs-filer.sh | 2 + .../down_seaweedfs-master.sh | 2 + tools/s3_server_compose/down_seaweedfs-s3.sh | 2 + .../down_seaweedfs-volume.sh | 2 + tools/s3_server_compose/env.sample | 7 + .../s3_server_compose/logs_seaweedfs-filer.sh | 2 + .../logs_seaweedfs-master.sh | 2 + tools/s3_server_compose/logs_seaweedfs-s3.sh | 2 + .../logs_seaweedfs-volume.sh | 2 + .../seaweedfs-filer/.gitignore | 1 + .../seaweedfs-filer/compose.yaml | 24 +++ .../seaweedfs-master/.gitignore | 1 + .../seaweedfs-master/compose.yaml | 26 +++ .../seaweedfs-s3/compose.yaml | 26 +++ .../seaweedfs-volume/.gitignore | 1 + .../seaweedfs-volume/compose.yaml | 24 +++ tools/s3_server_compose/up.sh | 2 + tools/s3_server_compose/up_seaweedfs-filer.sh | 2 + .../s3_server_compose/up_seaweedfs-master.sh | 2 + tools/s3_server_compose/up_seaweedfs-s3.sh | 2 + .../s3_server_compose/up_seaweedfs-volume.sh | 2 + 33 files changed, 597 insertions(+) create mode 100644 tools/s3_server_compose/.gitignore create mode 100755 tools/s3_server_compose/0_pull_other_containers.sh create mode 100644 tools/s3_server_compose/1_install_aws.sh create mode 100644 tools/s3_server_compose/2_create_s3_buckets.sh create mode 100755 tools/s3_server_compose/3_online_backup_s3.sh create mode 100644 tools/s3_server_compose/README.md create mode 100755 tools/s3_server_compose/autoupdate_seaweedfs-filer.sh create mode 100755 tools/s3_server_compose/autoupdate_seaweedfs-master.sh create mode 100755 tools/s3_server_compose/autoupdate_seaweedfs-s3.sh create mode 100755 tools/s3_server_compose/autoupdate_seaweedfs-volume.sh create mode 100644 tools/s3_server_compose/compose.yaml create mode 100644 tools/s3_server_compose/down.sh create mode 100644 tools/s3_server_compose/down_seaweedfs-filer.sh create mode 100644 tools/s3_server_compose/down_seaweedfs-master.sh create mode 100644 tools/s3_server_compose/down_seaweedfs-s3.sh create mode 100644 tools/s3_server_compose/down_seaweedfs-volume.sh create mode 100644 tools/s3_server_compose/env.sample create mode 100755 tools/s3_server_compose/logs_seaweedfs-filer.sh create mode 100755 tools/s3_server_compose/logs_seaweedfs-master.sh create mode 100755 tools/s3_server_compose/logs_seaweedfs-s3.sh create mode 100755 tools/s3_server_compose/logs_seaweedfs-volume.sh create mode 100644 tools/s3_server_compose/seaweedfs-filer/.gitignore create mode 100644 tools/s3_server_compose/seaweedfs-filer/compose.yaml create mode 100644 tools/s3_server_compose/seaweedfs-master/.gitignore create mode 100644 tools/s3_server_compose/seaweedfs-master/compose.yaml create mode 100644 tools/s3_server_compose/seaweedfs-s3/compose.yaml create mode 100644 tools/s3_server_compose/seaweedfs-volume/.gitignore create mode 100644 tools/s3_server_compose/seaweedfs-volume/compose.yaml create mode 100644 tools/s3_server_compose/up.sh create mode 100644 tools/s3_server_compose/up_seaweedfs-filer.sh create mode 100644 tools/s3_server_compose/up_seaweedfs-master.sh create mode 100644 tools/s3_server_compose/up_seaweedfs-s3.sh create mode 100644 tools/s3_server_compose/up_seaweedfs-volume.sh diff --git a/tools/s3_server_compose/.gitignore b/tools/s3_server_compose/.gitignore new file mode 100644 index 0000000000..7397eb55b1 --- /dev/null +++ b/tools/s3_server_compose/.gitignore @@ -0,0 +1,2 @@ +.env +backup_seaweedfs_online \ No newline at end of file diff --git a/tools/s3_server_compose/0_pull_other_containers.sh b/tools/s3_server_compose/0_pull_other_containers.sh new file mode 100755 index 0000000000..a338459e50 --- /dev/null +++ b/tools/s3_server_compose/0_pull_other_containers.sh @@ -0,0 +1 @@ +docker pull chrislusf/seaweedfs:latest diff --git a/tools/s3_server_compose/1_install_aws.sh b/tools/s3_server_compose/1_install_aws.sh new file mode 100644 index 0000000000..a586829c65 --- /dev/null +++ b/tools/s3_server_compose/1_install_aws.sh @@ -0,0 +1,2 @@ +sudo apt update +sudo apt install -y awscli diff --git a/tools/s3_server_compose/2_create_s3_buckets.sh b/tools/s3_server_compose/2_create_s3_buckets.sh new file mode 100644 index 0000000000..8a9266d68d --- /dev/null +++ b/tools/s3_server_compose/2_create_s3_buckets.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Load environment variables from .env file +if [ -f .env ]; then + set -a + source .env + set +a +else + echo "Error: .env file not found!" >&2 + exit 1 +fi + +export AWS_ACCESS_KEY_ID +export AWS_SECRET_ACCESS_KEY + +ENDPOINT="http://localhost:8333" + +echo "Testing connection to SeaweedFS S3 gateway..." +aws --endpoint-url "$ENDPOINT" s3 ls + +# Array of required Overleaf buckets +BUCKETS=( + "overleaf-user-files" + "overleaf-template-files" + "overleaf-project-blobs" + "overleaf-chunks" +) + +echo "Checking and creating Overleaf buckets..." +for bucket in "${BUCKETS[@]}"; do + if ! aws --endpoint-url "$ENDPOINT" s3 ls "s3://$bucket" >/dev/null 2>&1; then + echo "Creating bucket: s3://$bucket" + aws --endpoint-url "$ENDPOINT" s3 mb "s3://$bucket" + else + echo "Bucket already exists: s3://$bucket" + fi +done + +echo "Bucket setup complete. Current buckets:" +aws --endpoint-url "$ENDPOINT" s3 ls diff --git a/tools/s3_server_compose/3_online_backup_s3.sh b/tools/s3_server_compose/3_online_backup_s3.sh new file mode 100755 index 0000000000..24d0a4b458 --- /dev/null +++ b/tools/s3_server_compose/3_online_backup_s3.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Path to your .env file (adjust if your script runs from a different directory) +ENV_FILE="./.env" + +if [ -f "$ENV_FILE" ]; then + set -a + source "$ENV_FILE" + set +a +else + echo "Error: .env file not found at $ENV_FILE" >&2 + exit 1 +fi + +# Ensure AWS variables are exported for the AWS CLI +export AWS_ACCESS_KEY_ID +export AWS_SECRET_ACCESS_KEY + +ENDPOINT="http://localhost:8333" +BACKUP_DIR="./backup_seaweedfs_online" +TIMESTAMP=$(date +%F_%H-%M-%S) + +mkdir -p "$BACKUP_DIR/$TIMESTAMP" + +echo "1. Capturing online metadata snapshot..." +docker exec seaweedfs-filer weed shell -master="master:9333" <<< "fs.meta.save /data/filer_meta_$TIMESTAMP.json" + +echo "2. Syncing S3 buckets live..." +for bucket in overleaf-user-files overleaf-template-files overleaf-project-blobs overleaf-chunks; do + echo "Syncing s3://$bucket..." + aws --endpoint-url "$ENDPOINT" s3 sync "s3://$bucket" "$BACKUP_DIR/$TIMESTAMP/$bucket" +done + +echo "Online backup completed successfully at $TIMESTAMP" + diff --git a/tools/s3_server_compose/README.md b/tools/s3_server_compose/README.md new file mode 100644 index 0000000000..bdfbc4cefc --- /dev/null +++ b/tools/s3_server_compose/README.md @@ -0,0 +1,169 @@ +# SeaweedFS S3 Server (Docker Compose) + +A small single-node SeaweedFS cluster with a hosted S3 API. It serves as the +S3 backend (filestore *and* history). + +All four services share one image (`chrislusf/seaweedfs:latest`); the root +`compose.yaml` merges them from the per-service files with `extends:`. + +## Layout + +``` +. +├── compose.yaml # root compose (merges the 4 services, depends_on order) +├── seaweedfs-master/compose.yaml # cluster metadata + data/ (persisted) +├── seaweedfs-volume/compose.yaml # data bricks + data/ (persisted) +├── seaweedfs-filer/compose.yaml # file namespace + data/ (persisted) +├── seaweedfs-s3/compose.yaml # S3 API gateway (no data dir) +├── .env / env.sample # S3 credentials (never commit .env) +├── 0_pull_other_containers.sh # setup: pull the shared image +├── 1_install_aws.sh # setup: awscli on the host +├── 2_create_s3_buckets.sh # setup: create the Overleaf buckets (idempotent) +├── 3_online_backup_s3.sh # backup: metadata snapshot + bucket sync +├── autoupdate_seaweedfs-*.sh # update: one service at a time, only if image is newer +├── up.sh / down.sh # whole stack up / down +└── up|down|logs_seaweedfs-*.sh # per-service shortcuts +``` + +## Services, ports, network aliases + +Start-up order is enforced with `depends_on ... condition: service_healthy`: +`master` → `volume` → `filer` → `s3`. + +| Service | Network alias | Host ports (host → container) | Role | Healthcheck | +|--------------------|---------------|------------------------------------------------|-------------------------------|---------------------------| +| `seaweedfs-master` | `master` | `9333` (master HTTP API), `19333` (master gRPC), `9327` (Prometheus metrics) | cluster metadata | `:9333/cluster/healthz` | +| `seaweedfs-volume` | `volume` | `18080` → `8080` (volume gRPC; host `8080` is taken by another service) | data bricks | `:8080/status` | +| `seaweedfs-filer` | `filer` | `8888` (filer HTTP), `18888` (filer gRPC) | file namespace / directories | `:8888/` | +| `seaweedfs-s3` | `s3` | `8333` (S3 API) | S3 gateway for AWS-SDK clients | `:8333/` answers HTTP | + +**Where the aliases resolve (common gotcha):** + +- The aliases (`s3`, `filer`, `master`, `volume`) are registered on the + compose network (default project network of this folder, i.e. + `s3_server_compose_default`). +- ✅ They resolve from **any container attached to that network** — that is + why `http://s3:8333` works in the Overleaf config below. Check with + `docker exec seaweedfs-filer ping -c1 s3` (must reply from the stack IP). +- ❌ They do **not** resolve from the host machine — Docker's embedded DNS + exists only inside the network. From the host, always use the published + ports, e.g. `http://localhost:8333` (your `2_`/`3_` scripts already do this). +- A container on a *different* network cannot reach the aliases at all; it + must either be attached to this network (`docker network connect + s3_server_compose_default `) or use `http://:8333`. + +## First-time setup + +```bash +cp env.sample .env # then fill in keys (generation one-liners are in the file) +./0_pull_other_containers.sh +./1_install_aws.sh # installs awscli via apt on the host +./up.sh # = docker compose up -d (wait until all 4 are healthy) +./2_create_s3_buckets.sh # idempotent: creates the 4 Overleaf buckets if missing +``` + +Verify: + +```bash +docker ps | grep seaweedfs # 4 × healthy +curl -s -o /dev/null -w '%{http_code}\n' http://localhost:8333/ # 403 = S3 API answers (expected) +``` + +The Overleaf buckets created by `2_create_s3_buckets.sh`: + +`overleaf-user-files`, `overleaf-template-files`, `overleaf-project-blobs`, `overleaf-chunks` + +## Overleaf integration + +For the Overleaf container compose.yaml file (it is assume that the overleaf compose contains AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY): + +``` + ######################################### + ## Overleaf Filestore S3 Configuration ## + ######################################### + + OVERLEAF_FILESTORE_BACKEND: s3 + OVERLEAF_FILESTORE_USER_FILES_BUCKET_NAME: overleaf-user-files + OVERLEAF_FILESTORE_TEMPLATE_FILES_BUCKET_NAME: overleaf-template-files + + OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + OVERLEAF_FILESTORE_S3_ENDPOINT: http://s3:8333 + OVERLEAF_FILESTORE_S3_PATH_STYLE: "true" + OVERLEAF_FILESTORE_S3_REGION: "us-east-1" + + ######################################### + ## Overleaf History S3 Configuration ## + ######################################### + + OVERLEAF_HISTORY_BACKEND: s3 + OVERLEAF_HISTORY_PROJECT_BLOBS_BUCKET: overleaf-project-blobs + OVERLEAF_HISTORY_CHUNKS_BUCKET: overleaf-chunks + + OVERLEAF_HISTORY_S3_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + OVERLEAF_HISTORY_S3_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + OVERLEAF_HISTORY_S3_ENDPOINT: http://s3:8333 + OVERLEAF_HISTORY_S3_PATH_STYLE: "true" + OVERLEAF_HISTORY_S3_REGION: "us-east-1" +``` + +Notes: + +- `http://s3:8333` relies on the `s3` network alias — it only works while the + Overleaf **server container is attached to this stack's network** + (same compose network as the `seaweedfs-*` containers). +- If the Overleaf server runs elsewhere (other host, other compose network), + replace the endpoint with `http://:8333` in both + `OVERLEAF_FILESTORE_S3_ENDPOINT` and `OVERLEAF_HISTORY_S3_ENDPOINT`. +- `OVERLEAF_*_S3_PATH_STYLE: "true"` is required (SeaweedFS is a virtual-host + style-incompatible gateway), and the region value is cosmetic for a local + gateway but must be set consistently. + +## Scripts + +### Setup (numbered) + +| Script | What it does | +|---------------------------|--------------| +| `0_pull_other_containers.sh` | `docker pull chrislusf/seaweedfs:latest` — all four services share this one image, so a single pull covers the whole stack. | +| `1_install_aws.sh` | Installs `awscli` on the host via `apt` (needed by the bucket/backup scripts, which use `aws s3` locally). | +| `2_create_s3_buckets.sh` | Loads `.env`, checks reachability of `http://localhost:8333`, then creates the four Overleaf buckets if they don't exist yet. **Idempotent** — safe to re-run. | +| `3_online_backup_s3.sh` | **Online** backup (no downtime): ① snapshots filer metadata into the running filer via `weed shell` (`fs.meta.save`, reached through the `master:9333` alias), ② `aws s3 sync`s all four buckets into `./backup_seaweedfs_online///`. | + +### Updates + +| Script | What it does | +|-------------------------------|--------------| +| `autoupdate_seaweedfs-master.sh` / `…-volume.sh` / `…-filer.sh` / `…-s3.sh` | Pulls the latest image, compares the short image ID of the running container with the local one; if unchanged → no-op, if newer → `docker compose down` + `up -d` **of that service only**; if the container is missing → just starts it. Sources `.env` first. | + +> The autoupdate scripts `cd "$COMPOSE_DIRECTORY"` before calling +> `docker compose`, which now defaults to the **current directory** +> (`${COMPOSE_DIRECTORY:-$(pwd)}`). Just run them from this folder, +> or point `COMPOSE_DIRECTORY` somewhere else to override. + +To update the whole stack, run the four in this order: `master` → `volume` → +`filer` → `s3`. + +### Everyday + +| Script | Equivalent | +|---------------------------------|------------| +| `up.sh` / `down.sh` | `docker compose up -d` / `docker compose down` | +| `up_seaweedfs-.sh` | `docker compose up -d ` | +| `down_seaweedfs-.sh` | `docker compose down ` | +| `logs_seaweedfs-.sh` | `docker compose logs -f ` | + +## Data & operations + +- Persistent data: `seaweedfs-master/data/`, `seaweedfs-volume/data/`, + `seaweedfs-filer/data/` (bind-mounted at `/data` in each container). + The S3 gateway needs no data dir. Back up the **volume** data dir (the + actual file bytes) — metadata + buckets are reconstructed/synced by the + other pieces. +- All services run `restart: unless-stopped`; healthchecks per the table + above gate the start-up order and `depends_on`. +- Metrics: Prometheus endpoint at `http://localhost:9327` (master). +- Security: the S3 gateway binds `0.0.0.0:8333` — the keypair in `.env` is + the only gate (plus a `403` on anonymous requests). Keep `.env` private + (already git-ignored) and firewall `8333` if this host is reachable from + the network. diff --git a/tools/s3_server_compose/autoupdate_seaweedfs-filer.sh b/tools/s3_server_compose/autoupdate_seaweedfs-filer.sh new file mode 100755 index 0000000000..4a8e204e40 --- /dev/null +++ b/tools/s3_server_compose/autoupdate_seaweedfs-filer.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Load environment variables from .env file +if [ -f .env ]; then + set -a + source .env + set +a +else + echo "Error: .env file not found!" >&2 + exit 1 +fi + +SERVICE_NAME="seaweedfs-filer" +IMAGE_NAME="chrislusf/seaweedfs:latest" +YAML_DIRECTORY_NAME="${COMPOSE_DIRECTORY:-$(pwd)}" + +# Get current image ID (short format) +CURRENT_IMAGE=$(docker inspect --format='{{.Image}}' "${SERVICE_NAME}" 2>/dev/null | cut -d: -f2 | cut -c1-12) || true + +# Pull the latest image +echo "Pulling latest image..." +docker pull "${IMAGE_NAME}" + +# Get new image ID (short format) +NEW_IMAGE=$(docker images "${IMAGE_NAME}" --format '{{.ID}}') + +echo "Current image: $CURRENT_IMAGE" +echo "Latest image: $NEW_IMAGE" + +# Compare and restart only if different +if [ -z "$CURRENT_IMAGE" ]; then + echo "Container not found, starting it up..." + cd "${YAML_DIRECTORY_NAME}" + docker compose up -d "${SERVICE_NAME}" +elif [ "$CURRENT_IMAGE" != "$NEW_IMAGE" ]; then + echo "New image detected, restarting container..." + cd "${YAML_DIRECTORY_NAME}" + docker compose down "${SERVICE_NAME}" + docker compose up -d "${SERVICE_NAME}" + echo "Update completed!" +else + echo "Already running the latest version, no restart needed." +fi diff --git a/tools/s3_server_compose/autoupdate_seaweedfs-master.sh b/tools/s3_server_compose/autoupdate_seaweedfs-master.sh new file mode 100755 index 0000000000..01212e641e --- /dev/null +++ b/tools/s3_server_compose/autoupdate_seaweedfs-master.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Load environment variables from .env file +if [ -f .env ]; then + set -a + source .env + set +a +else + echo "Error: .env file not found!" >&2 + exit 1 +fi + +SERVICE_NAME="seaweedfs-master" +IMAGE_NAME="chrislusf/seaweedfs:latest" +YAML_DIRECTORY_NAME="${COMPOSE_DIRECTORY:-$(pwd)}" + +# Get current image ID (short format) +CURRENT_IMAGE=$(docker inspect --format='{{.Image}}' "${SERVICE_NAME}" 2>/dev/null | cut -d: -f2 | cut -c1-12) || true + +# Pull the latest image +echo "Pulling latest image..." +docker pull "${IMAGE_NAME}" + +# Get new image ID (short format) +NEW_IMAGE=$(docker images "${IMAGE_NAME}" --format '{{.ID}}') + +echo "Current image: $CURRENT_IMAGE" +echo "Latest image: $NEW_IMAGE" + +# Compare and restart only if different +if [ -z "$CURRENT_IMAGE" ]; then + echo "Container not found, starting it up..." + cd "${YAML_DIRECTORY_NAME}" + docker compose up -d "${SERVICE_NAME}" +elif [ "$CURRENT_IMAGE" != "$NEW_IMAGE" ]; then + echo "New image detected, restarting container..." + cd "${YAML_DIRECTORY_NAME}" + docker compose down "${SERVICE_NAME}" + docker compose up -d "${SERVICE_NAME}" + echo "Update completed!" +else + echo "Already running the latest version, no restart needed." +fi diff --git a/tools/s3_server_compose/autoupdate_seaweedfs-s3.sh b/tools/s3_server_compose/autoupdate_seaweedfs-s3.sh new file mode 100755 index 0000000000..3e078f4a74 --- /dev/null +++ b/tools/s3_server_compose/autoupdate_seaweedfs-s3.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Load environment variables from .env file +if [ -f .env ]; then + set -a + source .env + set +a +else + echo "Error: .env file not found!" >&2 + exit 1 +fi + +SERVICE_NAME="seaweedfs-s3" +IMAGE_NAME="chrislusf/seaweedfs:latest" +YAML_DIRECTORY_NAME="${COMPOSE_DIRECTORY:-$(pwd)}" + +# Get current image ID (short format) +CURRENT_IMAGE=$(docker inspect --format='{{.Image}}' "${SERVICE_NAME}" 2>/dev/null | cut -d: -f2 | cut -c1-12) || true + +# Pull the latest image +echo "Pulling latest image..." +docker pull "${IMAGE_NAME}" + +# Get new image ID (short format) +NEW_IMAGE=$(docker images "${IMAGE_NAME}" --format '{{.ID}}') + +echo "Current image: $CURRENT_IMAGE" +echo "Latest image: $NEW_IMAGE" + +# Compare and restart only if different +if [ -z "$CURRENT_IMAGE" ]; then + echo "Container not found, starting it up..." + cd "${YAML_DIRECTORY_NAME}" + docker compose up -d "${SERVICE_NAME}" +elif [ "$CURRENT_IMAGE" != "$NEW_IMAGE" ]; then + echo "New image detected, restarting container..." + cd "${YAML_DIRECTORY_NAME}" + docker compose down "${SERVICE_NAME}" + docker compose up -d "${SERVICE_NAME}" + echo "Update completed!" +else + echo "Already running the latest version, no restart needed." +fi diff --git a/tools/s3_server_compose/autoupdate_seaweedfs-volume.sh b/tools/s3_server_compose/autoupdate_seaweedfs-volume.sh new file mode 100755 index 0000000000..b03e76b239 --- /dev/null +++ b/tools/s3_server_compose/autoupdate_seaweedfs-volume.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Load environment variables from .env file +if [ -f .env ]; then + set -a + source .env + set +a +else + echo "Error: .env file not found!" >&2 + exit 1 +fi + +SERVICE_NAME="seaweedfs-volume" +IMAGE_NAME="chrislusf/seaweedfs:latest" +YAML_DIRECTORY_NAME="${COMPOSE_DIRECTORY:-$(pwd)}" + +# Get current image ID (short format) +CURRENT_IMAGE=$(docker inspect --format='{{.Image}}' "${SERVICE_NAME}" 2>/dev/null | cut -d: -f2 | cut -c1-12) || true + +# Pull the latest image +echo "Pulling latest image..." +docker pull "${IMAGE_NAME}" + +# Get new image ID (short format) +NEW_IMAGE=$(docker images "${IMAGE_NAME}" --format '{{.ID}}') + +echo "Current image: $CURRENT_IMAGE" +echo "Latest image: $NEW_IMAGE" + +# Compare and restart only if different +if [ -z "$CURRENT_IMAGE" ]; then + echo "Container not found, starting it up..." + cd "${YAML_DIRECTORY_NAME}" + docker compose up -d "${SERVICE_NAME}" +elif [ "$CURRENT_IMAGE" != "$NEW_IMAGE" ]; then + echo "New image detected, restarting container..." + cd "${YAML_DIRECTORY_NAME}" + docker compose down "${SERVICE_NAME}" + docker compose up -d "${SERVICE_NAME}" + echo "Update completed!" +else + echo "Already running the latest version, no restart needed." +fi diff --git a/tools/s3_server_compose/compose.yaml b/tools/s3_server_compose/compose.yaml new file mode 100644 index 0000000000..b03548c301 --- /dev/null +++ b/tools/s3_server_compose/compose.yaml @@ -0,0 +1,32 @@ +services: + seaweedfs-master: + extends: + file: ./seaweedfs-master/compose.yaml + service: seaweedfs-master + + seaweedfs-volume: + extends: + file: ./seaweedfs-volume/compose.yaml + service: seaweedfs-volume + depends_on: + seaweedfs-master: + condition: service_healthy + + seaweedfs-filer: + extends: + file: ./seaweedfs-filer/compose.yaml + service: seaweedfs-filer + depends_on: + seaweedfs-master: + condition: service_healthy + seaweedfs-volume: + condition: service_healthy + + seaweedfs-s3: + extends: + file: ./seaweedfs-s3/compose.yaml + service: seaweedfs-s3 + depends_on: + seaweedfs-filer: + condition: service_healthy + diff --git a/tools/s3_server_compose/down.sh b/tools/s3_server_compose/down.sh new file mode 100644 index 0000000000..c8642093b6 --- /dev/null +++ b/tools/s3_server_compose/down.sh @@ -0,0 +1,2 @@ +docker compose down + diff --git a/tools/s3_server_compose/down_seaweedfs-filer.sh b/tools/s3_server_compose/down_seaweedfs-filer.sh new file mode 100644 index 0000000000..54d886b00e --- /dev/null +++ b/tools/s3_server_compose/down_seaweedfs-filer.sh @@ -0,0 +1,2 @@ +docker compose down seaweedfs-filer + diff --git a/tools/s3_server_compose/down_seaweedfs-master.sh b/tools/s3_server_compose/down_seaweedfs-master.sh new file mode 100644 index 0000000000..b57048304a --- /dev/null +++ b/tools/s3_server_compose/down_seaweedfs-master.sh @@ -0,0 +1,2 @@ +docker compose down seaweedfs-master + diff --git a/tools/s3_server_compose/down_seaweedfs-s3.sh b/tools/s3_server_compose/down_seaweedfs-s3.sh new file mode 100644 index 0000000000..557430e817 --- /dev/null +++ b/tools/s3_server_compose/down_seaweedfs-s3.sh @@ -0,0 +1,2 @@ +docker compose down seaweedfs-s3 + diff --git a/tools/s3_server_compose/down_seaweedfs-volume.sh b/tools/s3_server_compose/down_seaweedfs-volume.sh new file mode 100644 index 0000000000..f21d3d8b27 --- /dev/null +++ b/tools/s3_server_compose/down_seaweedfs-volume.sh @@ -0,0 +1,2 @@ +docker compose down seaweedfs-volume + diff --git a/tools/s3_server_compose/env.sample b/tools/s3_server_compose/env.sample new file mode 100644 index 0000000000..1481051163 --- /dev/null +++ b/tools/s3_server_compose/env.sample @@ -0,0 +1,7 @@ +# You need a .env based on this env.sample +#S3 Buckets +# openssl rand -base64 30 | tr -dc 'a-zA-Z0-9' | head -c 20; echo +AWS_ACCESS_KEY_ID=[MISSING] +# openssl rand -base64 60 | tr -dc 'a-zA-Z0-9' | head -c 40; echo +AWS_SECRET_ACCESS_KEY=[MISSING] + diff --git a/tools/s3_server_compose/logs_seaweedfs-filer.sh b/tools/s3_server_compose/logs_seaweedfs-filer.sh new file mode 100755 index 0000000000..c07b70197f --- /dev/null +++ b/tools/s3_server_compose/logs_seaweedfs-filer.sh @@ -0,0 +1,2 @@ +docker compose logs -f seaweedfs-filer + diff --git a/tools/s3_server_compose/logs_seaweedfs-master.sh b/tools/s3_server_compose/logs_seaweedfs-master.sh new file mode 100755 index 0000000000..92da34358b --- /dev/null +++ b/tools/s3_server_compose/logs_seaweedfs-master.sh @@ -0,0 +1,2 @@ +docker compose logs -f seaweedfs-master + diff --git a/tools/s3_server_compose/logs_seaweedfs-s3.sh b/tools/s3_server_compose/logs_seaweedfs-s3.sh new file mode 100755 index 0000000000..2c04282ef0 --- /dev/null +++ b/tools/s3_server_compose/logs_seaweedfs-s3.sh @@ -0,0 +1,2 @@ +docker compose logs -f seaweedfs-s3 + diff --git a/tools/s3_server_compose/logs_seaweedfs-volume.sh b/tools/s3_server_compose/logs_seaweedfs-volume.sh new file mode 100755 index 0000000000..a1a1a6c610 --- /dev/null +++ b/tools/s3_server_compose/logs_seaweedfs-volume.sh @@ -0,0 +1,2 @@ +docker compose logs -f seaweedfs-volume + diff --git a/tools/s3_server_compose/seaweedfs-filer/.gitignore b/tools/s3_server_compose/seaweedfs-filer/.gitignore new file mode 100644 index 0000000000..6320cd248d --- /dev/null +++ b/tools/s3_server_compose/seaweedfs-filer/.gitignore @@ -0,0 +1 @@ +data \ No newline at end of file diff --git a/tools/s3_server_compose/seaweedfs-filer/compose.yaml b/tools/s3_server_compose/seaweedfs-filer/compose.yaml new file mode 100644 index 0000000000..ece97c345f --- /dev/null +++ b/tools/s3_server_compose/seaweedfs-filer/compose.yaml @@ -0,0 +1,24 @@ +services: + seaweedfs-filer: + image: chrislusf/seaweedfs:latest + container_name: seaweedfs-filer + hostname: seaweedfs-filer + ports: + - "8888:8888" # filer HTTP + - "18888:18888" # filer gRPC + command: 'filer -master="master:9333" -ip=filer -ip.bind=0.0.0.0' + volumes: + - ./data:/data + networks: + default: + aliases: + - filer + restart: unless-stopped + + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:8888/"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 15s + diff --git a/tools/s3_server_compose/seaweedfs-master/.gitignore b/tools/s3_server_compose/seaweedfs-master/.gitignore new file mode 100644 index 0000000000..6320cd248d --- /dev/null +++ b/tools/s3_server_compose/seaweedfs-master/.gitignore @@ -0,0 +1 @@ +data \ No newline at end of file diff --git a/tools/s3_server_compose/seaweedfs-master/compose.yaml b/tools/s3_server_compose/seaweedfs-master/compose.yaml new file mode 100644 index 0000000000..c5e1758234 --- /dev/null +++ b/tools/s3_server_compose/seaweedfs-master/compose.yaml @@ -0,0 +1,26 @@ +services: + seaweedfs-master: + image: chrislusf/seaweedfs:latest + container_name: seaweedfs-master + hostname: seaweedfs-master + ports: + - "9333:9333" # master HTTP + - "19333:19333" # master gRPC + command: "master -ip=master -ip.bind=0.0.0.0 -metricsPort=9327" + volumes: + - ./data:/data + networks: + default: + # classic seaweedfs names (master:9333 etc.) must resolve on the shared + # network while the container keeps its seaweedfs-* name. + aliases: + - master + restart: unless-stopped + + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:9333/cluster/healthz"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 10s + diff --git a/tools/s3_server_compose/seaweedfs-s3/compose.yaml b/tools/s3_server_compose/seaweedfs-s3/compose.yaml new file mode 100644 index 0000000000..7fb54b798d --- /dev/null +++ b/tools/s3_server_compose/seaweedfs-s3/compose.yaml @@ -0,0 +1,26 @@ +services: + seaweedfs-s3: + image: chrislusf/seaweedfs:latest + container_name: seaweedfs-s3 + hostname: seaweedfs-s3 + ports: + - "8333:8333" # S3 API + command: 's3 -filer="filer:8888" -ip.bind=0.0.0.0' + + environment: + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + + networks: + default: + aliases: + - s3 + restart: unless-stopped + + healthcheck: + test: ["CMD-SHELL", "wget -S -o- http://localhost:8333/ 2>&1 | grep -q 'HTTP/'"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 15s + diff --git a/tools/s3_server_compose/seaweedfs-volume/.gitignore b/tools/s3_server_compose/seaweedfs-volume/.gitignore new file mode 100644 index 0000000000..6320cd248d --- /dev/null +++ b/tools/s3_server_compose/seaweedfs-volume/.gitignore @@ -0,0 +1 @@ +data \ No newline at end of file diff --git a/tools/s3_server_compose/seaweedfs-volume/compose.yaml b/tools/s3_server_compose/seaweedfs-volume/compose.yaml new file mode 100644 index 0000000000..5e23155650 --- /dev/null +++ b/tools/s3_server_compose/seaweedfs-volume/compose.yaml @@ -0,0 +1,24 @@ +services: + seaweedfs-volume: + image: chrislusf/seaweedfs:latest + container_name: seaweedfs-volume + # no host port mapping: volume is only reached inside the network as + # volume:8080 (host 8080 is taken by another service) — gRPC stays mapped. + ports: + - "18080:18080" # volume gRPC + command: 'volume -mserver="master:9333" -ip=volume -ip.bind=0.0.0.0 -port=8080 -dir=/data -max=0' + volumes: + - ./data:/data + networks: + default: + aliases: + - volume + restart: unless-stopped + + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/status"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 15s + diff --git a/tools/s3_server_compose/up.sh b/tools/s3_server_compose/up.sh new file mode 100644 index 0000000000..a4a5dbb01a --- /dev/null +++ b/tools/s3_server_compose/up.sh @@ -0,0 +1,2 @@ +docker compose up -d + diff --git a/tools/s3_server_compose/up_seaweedfs-filer.sh b/tools/s3_server_compose/up_seaweedfs-filer.sh new file mode 100644 index 0000000000..bdd33090b3 --- /dev/null +++ b/tools/s3_server_compose/up_seaweedfs-filer.sh @@ -0,0 +1,2 @@ +docker compose up -d seaweedfs-filer + diff --git a/tools/s3_server_compose/up_seaweedfs-master.sh b/tools/s3_server_compose/up_seaweedfs-master.sh new file mode 100644 index 0000000000..0a87f5423d --- /dev/null +++ b/tools/s3_server_compose/up_seaweedfs-master.sh @@ -0,0 +1,2 @@ +docker compose up -d seaweedfs-master + diff --git a/tools/s3_server_compose/up_seaweedfs-s3.sh b/tools/s3_server_compose/up_seaweedfs-s3.sh new file mode 100644 index 0000000000..7f11f8e1ad --- /dev/null +++ b/tools/s3_server_compose/up_seaweedfs-s3.sh @@ -0,0 +1,2 @@ +docker compose up -d seaweedfs-s3 + diff --git a/tools/s3_server_compose/up_seaweedfs-volume.sh b/tools/s3_server_compose/up_seaweedfs-volume.sh new file mode 100644 index 0000000000..b42cd1e322 --- /dev/null +++ b/tools/s3_server_compose/up_seaweedfs-volume.sh @@ -0,0 +1,2 @@ +docker compose up -d seaweedfs-volume +