mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-11-12 21:23:45 +01:00
72 lines
2.1 KiB
Bash
Executable File
72 lines
2.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Download & extract LanguageTool n-gram data (main set: de en es fr nl)
|
|
# into ngrams/ next to this script.
|
|
#
|
|
# Sources: https://languagetool.org/download/ngram-data/
|
|
# ngrams-de-20150819.zip (~1.7 GB)
|
|
# ngrams-en-20150817.zip (~9.0 GB)
|
|
# ngrams-es-20150915.zip (~1.7 GB)
|
|
# ngrams-fr-20150913.zip (~1.8 GB)
|
|
# ngrams-nl-20181229.zip (~1.2 GB)
|
|
#
|
|
# Usage: ./download_ngrams.sh
|
|
# Re-running is safe: already-downloaded & valid zips are skipped (download is
|
|
# resumable via curl -C).
|
|
|
|
set -euo pipefail
|
|
|
|
BASE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
NGRAMS_DIR="$BASE/ngrams"
|
|
BASE_URL="https://languagetool.org/download/ngram-data"
|
|
|
|
# lang -> zip filename (verified against server directory listing)
|
|
declare -a FILES=(
|
|
"ngrams-de-20150819.zip"
|
|
"ngrams-en-20150817.zip"
|
|
"ngrams-es-20150915.zip"
|
|
"ngrams-fr-20150913.zip"
|
|
"ngrams-nl-20181229.zip"
|
|
)
|
|
|
|
command -v curl >/dev/null || { echo "ERROR: curl not found" >&2; exit 1; }
|
|
command -v unzip >/dev/null || { echo "ERROR: unzip not found" >&2; exit 1; }
|
|
|
|
mkdir -p "$NGRAMS_DIR"
|
|
echo "Target dir: $NGRAMS_DIR"
|
|
|
|
failed=0
|
|
for zip in "${FILES[@]}"; do
|
|
url="$BASE_URL/$zip"
|
|
dest="$NGRAMS_DIR/$zip"
|
|
|
|
echo "=================================================================="
|
|
echo "$zip"
|
|
|
|
if [ -f "$dest" ] && unzip -t "$dest" >/dev/null 2>&1; then
|
|
echo " [skip] already downloaded and valid: $zip"
|
|
else
|
|
echo " [dl] $url"
|
|
if ! curl --fail --location --retry 3 --retry-delay 5 -C - \
|
|
--connect-timeout 30 --progress-bar -o "$dest" "$url"; then
|
|
echo " [FAIL] download failed for $url (partial file kept for resume: $dest)" >&2
|
|
rm -f "$dest" # 404/error body would corrupt the name for future runs
|
|
failed=1
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
echo " [extract] -> $NGRAMS_DIR"
|
|
if ! unzip -o -q "$dest" -d "$NGRAMS_DIR"; then
|
|
echo " [FAIL] extraction failed for $zip" >&2
|
|
failed=1
|
|
fi
|
|
done
|
|
|
|
echo "=================================================================="
|
|
if [ "$failed" -ne 0 ]; then
|
|
echo "Finished with ERRORS — re-run to resume failed downloads." >&2
|
|
exit 1
|
|
fi
|
|
echo "All done. Contents:"
|
|
ls -lh "$NGRAMS_DIR"
|