mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-11-12 13:13:45 +01:00
76 lines
2.4 KiB
Bash
Executable File
76 lines
2.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Download & extract LanguageTool n-gram data (UNTESTED set) into ngrams/untested/
|
|
# next to this script.
|
|
#
|
|
# Sources: https://languagetool.org/download/ngram-data/untested/
|
|
# ngram-he-20150916.zip (~225 MB)
|
|
# ngram-it-20150915.zip (~1.1 GB)
|
|
# ngram-ru-20150914.zip (~2.3 GB)
|
|
# ngram-zh-20150916.zip (~1.7 GB)
|
|
#
|
|
# NOTE: "an" (Aragonese) was requested but does NOT exist on the server
|
|
# (checked ngram-data/, untested/, temp/, archive/ — only he/it/ru/zh are
|
|
# offered). Kept commented out so it can be enabled if it appears later.
|
|
#
|
|
# Usage: ./download_ngrams_untested.sh
|
|
# Re-running is safe: already-downloaded & valid zips are skipped (download is
|
|
# resumable via curl -C).
|
|
|
|
set -euo pipefail
|
|
|
|
BASE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
NGRAMS_DIR="$BASE/ngrams"
|
|
UNTESTED_DIR="$NGRAMS_DIR"
|
|
BASE_URL="https://languagetool.org/download/ngram-data/untested"
|
|
|
|
# zip filenames (verified against server directory listing)
|
|
declare -a FILES=(
|
|
"ngram-he-20150916.zip"
|
|
"ngram-it-20150915.zip"
|
|
"ngram-ru-20150914.zip"
|
|
"ngram-zh-20150916.zip"
|
|
# "ngram-an-*.zip (404 — not available on the server as of 2025/2026)"
|
|
)
|
|
|
|
command -v curl >/dev/null || { echo "ERROR: curl not found" >&2; exit 1; }
|
|
command -v unzip >/dev/null || { echo "ERROR: unzip not found" >&2; exit 1; }
|
|
|
|
mkdir -p "$UNTESTED_DIR"
|
|
echo "Target dir: $UNTESTED_DIR"
|
|
|
|
failed=0
|
|
for zip in "${FILES[@]}"; do
|
|
url="$BASE_URL/$zip"
|
|
dest="$UNTESTED_DIR/$zip"
|
|
|
|
echo "=================================================================="
|
|
echo "$zip"
|
|
|
|
if [ -f "$dest" ] && unzip -t "$dest" >/dev/null 2>&1; then
|
|
echo " [skip] already downloaded and valid: $zip"
|
|
else
|
|
echo " [dl] $url"
|
|
if ! curl --fail --location --retry 3 --retry-delay 5 -C - \
|
|
--connect-timeout 30 --progress-bar -o "$dest" "$url"; then
|
|
echo " [FAIL] download failed for $url (partial file kept for resume: $dest)" >&2
|
|
rm -f "$dest" # 404/error body would corrupt the name for future runs
|
|
failed=1
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
echo " [extract] -> $UNTESTED_DIR"
|
|
if ! unzip -o -q "$dest" -d "$UNTESTED_DIR"; then
|
|
echo " [FAIL] extraction failed for $zip" >&2
|
|
failed=1
|
|
fi
|
|
done
|
|
|
|
echo "=================================================================="
|
|
if [ "$failed" -ne 0 ]; then
|
|
echo "Finished with ERRORS — re-run to resume failed downloads." >&2
|
|
exit 1
|
|
fi
|
|
echo "All done. Contents:"
|
|
ls -lh "$UNTESTED_DIR"
|