From 923b1f74077d5762f400454411db243fcaa00eb8 Mon Sep 17 00:00:00 2001 From: Thomas Dannenberg Date: Sat, 30 Aug 2025 19:37:32 +0000 Subject: [PATCH] Kapitel 13/Tutorial.md aktualisiert --- Kapitel 13/Tutorial.md | 134 +++++++++++++++++++++++------------------ 1 file changed, 74 insertions(+), 60 deletions(-) diff --git a/Kapitel 13/Tutorial.md b/Kapitel 13/Tutorial.md index b72397b..890f143 100644 --- a/Kapitel 13/Tutorial.md +++ b/Kapitel 13/Tutorial.md @@ -451,82 +451,96 @@ In diesem Schritt erstellen wir den eigentlichen Workflow in **n8n**. Er sorgt d ``` Nun befüllst du sie mit: ```bash - #!/usr/bin/env bash - set -euo pipefail - . /etc/clipper/clipper.env + #!/usr/bin/env bash + set -euo pipefail + . /etc/clipper/clipper.env - ID="${1:?need VOD id}" - URL="${2:-https://www.twitch.tv/videos/${ID}}" + ID="${1:?need VOD id}" + URL="${2:-https://www.twitch.tv/videos/${ID}}" - TMP="${CLIPPER_TMP}" - OUT_BASE="${CLIPPER_OUT}/${ID}" - LOGDIR="${CLIPPER_LOG}/${ID}" - CONF="/home/clipper/.config/rclone/rclone.conf" + TMP="${CLIPPER_TMP}" + OUT_BASE="${CLIPPER_OUT}/${ID}" + LOGDIR="${CLIPPER_LOG}/${ID}" + CONF="/home/clipper/.config/rclone/rclone.conf" - DST="${CLIPPER_NC_REMOTE}/VODs/${ID}/" + DST="${CLIPPER_NC_REMOTE}/VODs/${ID}/" - OUT="$TMP/${ID}.%(ext)s" - FILE="$TMP/${ID}.mp4" - TEMP="$TMP/${ID}.temp.mp4" - PART="$TMP/${ID}.mp4.part" - LOCK="$TMP/${ID}.lock" + OUT="$TMP/${ID}.%(ext)s" + FILE="$TMP/${ID}.mp4" + TEMP="$TMP/${ID}.temp.mp4" + PART="$TMP/${ID}.mp4.part" + LOCK="$TMP/${ID}.lock" - mkdir -p "$TMP" "$LOGDIR" - LOG="$LOGDIR/download.log" - log(){ echo "[$(date '+%F %T')] $*"; } - exec > >(tee -a "$LOG") 2>&1 + mkdir -p "$TMP" "$LOGDIR" "$OUT_BASE" + LOG="$LOGDIR/download.log" + log(){ echo "[$(date '+%F %T')] $*"; } + exec > >(tee -a "$LOG") 2>&1 - log "=== Start VOD $ID ===" - log "URL: $URL" - log "DST: $DST" + # ---- atomarer Lock (kein Race zwischen zwei Starts) + exec 9>"$LOCK" + if ! flock -n 9; then + log "LOCK: $ID wird bereits verarbeitet" + exit 0 + fi + trap 'flock -u 9; rm -f "$LOCK"' EXIT - if rclone lsf "$DST" --config "$CONF" >/dev/null 2>&1; then - log "SKIP: $ID bereits in Nextcloud" - exit 0 - fi + log "=== Start VOD $ID ===" + log "URL: $URL" + log "DST: $DST" - if [[ -e "$LOCK" ]]; then - log "LOCK: $ID wird bereits verarbeitet" - exit 0 - fi - trap 'rm -f "$LOCK"' EXIT - : > "$LOCK" + # ---- Prüfe, ob die ZIELDATEI bereits existiert (nicht nur der Ordner) + if rclone lsf "$DST" --config "$CONF" | grep -qx "${ID}.mp4"; then + log "SKIP: $ID.mp4 bereits in Nextcloud" + exit 0 + fi - # Resume - if [[ -s "$TEMP" && ! -s "$FILE" ]]; then - log "RESUME: $TEMP -> $FILE" - mv -f "$TEMP" "$FILE" - fi + # ---- Resume: unvollständigen Download sauber fortsetzen + if [[ -s "$TEMP" && ! -s "$FILE" ]]; then + log "RESUME: $TEMP -> $FILE" + mv -f "$TEMP" "$FILE" + fi - if [[ -s "$FILE" ]]; then - log "MOVE (resume): $FILE → $DST" - rclone move "$FILE" "$DST" --config "$CONF" --create-empty-src-dirs -v - rm -f "$PART" "$TEMP" || true - log "CLEANUP: $TMP" - rm -rf "${TMP:?}/"* - log "=== Done VOD $ID (resume path) ===" - exit 0 - fi + # ---- Gemeinsame rclone-Flags: Nextcloud-Chunking + robuste Timeouts + export RCLONE_WEBDAV_CHUNK_SIZE=100Mi # wirkt bei neueren rclone Versionen - # Download + Remux - yt-dlp -q --no-progress --retries 20 --fragment-retries 50 --retry-sleep 5 \ - --socket-timeout 30 --hls-prefer-ffmpeg --remux-video mp4 -o "$OUT" "$URL" + rclone move "$FILE" "$DST" \ + --config "$CONF" --create-empty-src-dirs \ + --transfers 1 --checkers 4 \ + --retries 10 --low-level-retries 50 --retries-sleep 30s \ + --timeout 1h --contimeout 1m --expect-continue-timeout 10m \ + -v - [[ -s "$FILE" ]] || { [[ -s "$TEMP" ]] && mv -f "$TEMP" "$FILE"; } + # ---- Wenn Datei schon da: nur noch hochschieben + if [[ -s "$FILE" ]]; then + log "MOVE (resume): $FILE → $DST" + rclone move "$FILE" "$DST" "${RCLONE_FLAGS[@]}" + rm -f "$PART" "$TEMP" || true + log "CLEANUP: $TMP" + rm -rf "${TMP:?}/"* + log "=== Done VOD $ID (resume path) ===" + exit 0 + fi - if [[ ! -s "$FILE" ]]; then - log "ERROR: Download fehlgeschlagen ($FILE fehlt/leer)" - exit 2 - fi + # ---- Download + Remux + yt-dlp -q --no-progress --retries 20 --fragment-retries 50 --retry-sleep 5 \ + --socket-timeout 30 --hls-prefer-ffmpeg --remux-video mp4 -o "$OUT" "$URL" - log "MOVE: $FILE → $DST" - rclone move "$FILE" "$DST" --config "$CONF" --create-empty-src-dirs -v - rm -f "$PART" "$TEMP" || true + [[ -s "$FILE" ]] || { [[ -s "$TEMP" ]] && mv -f "$TEMP" "$FILE"; } - log "CLEANUP: $TMP" - rm -rf "${TMP:?}/"* + if [[ ! -s "$FILE" ]]; then + log "ERROR: Download fehlgeschlagen ($FILE fehlt/leer)" + exit 2 + fi - log "=== Done VOD $ID ===" + log "MOVE: $FILE → $DST" + rclone move "$FILE" "$DST" "${RCLONE_FLAGS[@]}" + rm -f "$PART" "$TEMP" || true + + log "CLEANUP: $TMP" + rm -rf "${TMP:?}/"* + + log "=== Done VOD $ID ===" + " ``` Mit diesem Skript laden wir die aktuellen VODs herunter, laden sie in die Nextcloud für die weitere Verabeitung und räumen wieder auf. Zusätzlich erzeugen wir logs in `/logs/.log`.