Batch pdf to md.sh

#!/usr/bin/env bash
# Batch wrapper untuk pdf_to_rag_md.py
#
# Penggunaan:
#   ./batch_pdf_to_md.sh /path/folder_pdf
#   ./batch_pdf_to_md.sh /path/folder_pdf /path/folder_output
#   ./batch_pdf_to_md.sh /path/folder_pdf /path/folder_output /path/pdf_to_rag_md.py
#
# Hasil:
#   OUTPUT_DIR/<path-relatif>/<nama-pdf>/
#       ├── *.md
#       └── manifest.json

set -Eeuo pipefail
IFS=$'\n\t'

PROGRAM_NAME="$(basename "$0")"
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"

usage() {
    cat <<EOF
Penggunaan:
  $PROGRAM_NAME INPUT_PATH [OUTPUT_DIR] [PYTHON_SCRIPT]

Argumen:
  INPUT_PATH      Folder yang akan dicari secara rekursif, atau satu file PDF.
  OUTPUT_DIR      Folder induk hasil konversi.
                  Default: INPUT_PATH/output untuk input folder,
                  atau folder_file/output untuk input satu PDF.
  PYTHON_SCRIPT   Lokasi script Python konverter.
                  Default: dicari di folder yang sama dengan shell script.

Contoh:
  $PROGRAM_NAME /opt/ai-stack/documents
  $PROGRAM_NAME /opt/ai-stack/documents /opt/ai-stack/output
  $PROGRAM_NAME ./pdf ./output ./pdf_to_rag_md.py
EOF
}

log() {
    printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"
}

fail() {
    printf 'ERROR: %s\n' "$*" >&2
    exit 1
}

if [[ ${1:-} == "-h" || ${1:-} == "--help" ]]; then
    usage
    exit 0
fi

[[ $# -ge 1 && $# -le 3 ]] || {
    usage >&2
    exit 2
}

command -v python3 >/dev/null 2>&1 || fail "python3 tidak ditemukan."
command -v find >/dev/null 2>&1 || fail "Perintah find tidak ditemukan."

INPUT_PATH="$1"
[[ -e "$INPUT_PATH" ]] || fail "Path input tidak ditemukan: $INPUT_PATH"

# Ubah menjadi absolute path tanpa membutuhkan realpath.
if [[ -d "$INPUT_PATH" ]]; then
    INPUT_PATH="$(cd -- "$INPUT_PATH" && pwd -P)"
    INPUT_IS_DIR=1
else
    INPUT_DIR="$(cd -- "$(dirname -- "$INPUT_PATH")" && pwd -P)"
    INPUT_PATH="$INPUT_DIR/$(basename -- "$INPUT_PATH")"
    INPUT_IS_DIR=0
fi

if [[ $# -ge 2 ]]; then
    OUTPUT_DIR="$2"
else
    if (( INPUT_IS_DIR )); then
        OUTPUT_DIR="$INPUT_PATH/output"
    else
        OUTPUT_DIR="$(dirname -- "$INPUT_PATH")/output"
    fi
fi
mkdir -p -- "$OUTPUT_DIR"
OUTPUT_DIR="$(cd -- "$OUTPUT_DIR" && pwd -P)"
[[ "$OUTPUT_DIR" != "$INPUT_PATH" ]] \
    || fail "OUTPUT_DIR tidak boleh sama persis dengan folder input. Gunakan subfolder, misalnya: $INPUT_PATH/output"

if [[ $# -ge 3 ]]; then
    PYTHON_SCRIPT="$3"
else
    if [[ -f "$SCRIPT_DIR/pdf_to_rag_md.py" ]]; then
        PYTHON_SCRIPT="$SCRIPT_DIR/pdf_to_rag_md.py"
    elif [[ -f "$SCRIPT_DIR/pdf_to_rag_md(2).py" ]]; then
        PYTHON_SCRIPT="$SCRIPT_DIR/pdf_to_rag_md(2).py"
    else
        fail "Script Python tidak ditemukan. Letakkan pdf_to_rag_md.py di $SCRIPT_DIR atau berikan argumen PYTHON_SCRIPT."
    fi
fi

[[ -f "$PYTHON_SCRIPT" ]] || fail "Script Python tidak ditemukan: $PYTHON_SCRIPT"
PYTHON_SCRIPT="$(cd -- "$(dirname -- "$PYTHON_SCRIPT")" && pwd -P)/$(basename -- "$PYTHON_SCRIPT")"

# Pastikan script Python valid dan dependensinya tersedia sebelum batch dimulai.
python3 -m py_compile "$PYTHON_SCRIPT" \
    || fail "Script Python gagal diperiksa dengan py_compile."

python3 - <<'PY' \
    || fail "Dependensi Python belum lengkap. Instal dengan: python3 -m pip install pymupdf pyyaml"
try:
    import pymupdf  # noqa: F401
except ImportError:
    import fitz  # noqa: F401
import yaml  # noqa: F401
PY

find_input_files() {
    if [[ "$OUTPUT_DIR" == "$INPUT_PATH/"* ]]; then
        find "$INPUT_PATH" -path "$OUTPUT_DIR" -prune -o -type f "$@" -print0
    else
        find "$INPUT_PATH" -type f "$@" -print0
    fi
}

printf '\n=== Daftar semua file pada input ===\n'
if (( INPUT_IS_DIR )); then
    while IFS= read -r -d '' file; do
        printf '%s\n' "$file"
    done < <(find_input_files | sort -z)
else
    printf '%s\n' "$INPUT_PATH"
fi
printf '=== Akhir daftar file ===\n\n'

PDF_FILES=()
if (( INPUT_IS_DIR )); then
    while IFS= read -r -d '' pdf; do
        PDF_FILES+=("$pdf")
    done < <(find_input_files -iname '*.pdf' | sort -z)
else
    [[ "${INPUT_PATH,,}" == *.pdf ]] \
        || fail "File input bukan PDF: $INPUT_PATH"
    PDF_FILES+=("$INPUT_PATH")
fi

TOTAL=${#PDF_FILES[@]}
if (( TOTAL == 0 )); then
    log "Tidak ada file PDF yang ditemukan."
    exit 0
fi

LOG_FILE="$OUTPUT_DIR/batch_conversion.log"
SUMMARY_FILE="$OUTPUT_DIR/batch_summary.tsv"
printf 'status\tinput_pdf\toutput_folder\n' > "$SUMMARY_FILE"
: > "$LOG_FILE"

log "Ditemukan $TOTAL file PDF."
log "Output utama: $OUTPUT_DIR"
log "Log batch: $LOG_FILE"

SUCCESS=0
FAILED=0
INDEX=0

for PDF_FILE in "${PDF_FILES[@]}"; do
    ((INDEX += 1))

    if (( INPUT_IS_DIR )); then
        RELATIVE_PATH="${PDF_FILE#"$INPUT_PATH"/}"
    else
        RELATIVE_PATH="$(basename -- "$PDF_FILE")"
    fi

    RELATIVE_PARENT="$(dirname -- "$RELATIVE_PATH")"
    PDF_NAME="$(basename -- "$RELATIVE_PATH")"
    PDF_STEM="${PDF_NAME%.*}"

    if [[ "$RELATIVE_PARENT" == "." ]]; then
        PDF_OUTPUT_DIR="$OUTPUT_DIR/$PDF_STEM"
    else
        PDF_OUTPUT_DIR="$OUTPUT_DIR/$RELATIVE_PARENT/$PDF_STEM"
    fi

    mkdir -p -- "$PDF_OUTPUT_DIR"

    printf '\n[%d/%d] Konversi: %s\n' "$INDEX" "$TOTAL" "$PDF_FILE"
    printf '        Output   : %s\n' "$PDF_OUTPUT_DIR"

    printf '\n===== [%d/%d] %s =====\n' "$INDEX" "$TOTAL" "$PDF_FILE" \
        | tee -a "$LOG_FILE"

    set +e
    python3 "$PYTHON_SCRIPT" \
        "$PDF_FILE" \
        --output-dir "$PDF_OUTPUT_DIR" \
        --overwrite 2>&1 | tee -a "$LOG_FILE"
    EXIT_CODE=${PIPESTATUS[0]}
    set -e

    if (( EXIT_CODE == 0 )); then
        printf 'SUCCESS\t%s\t%s\n' "$PDF_FILE" "$PDF_OUTPUT_DIR" \
            >> "$SUMMARY_FILE"
        ((SUCCESS += 1))
        log "BERHASIL: $PDF_FILE" | tee -a "$LOG_FILE"
    else
        printf 'FAILED(%d)\t%s\t%s\n' "$EXIT_CODE" "$PDF_FILE" "$PDF_OUTPUT_DIR" \
            >> "$SUMMARY_FILE"
        ((FAILED += 1))
        log "GAGAL (exit $EXIT_CODE): $PDF_FILE" | tee -a "$LOG_FILE"
    fi
done

printf '\n=== Ringkasan batch ===\n'
printf 'Total PDF : %d\n' "$TOTAL"
printf 'Berhasil  : %d\n' "$SUCCESS"
printf 'Gagal     : %d\n' "$FAILED"
printf 'Output    : %s\n' "$OUTPUT_DIR"
printf 'Ringkasan : %s\n' "$SUMMARY_FILE"
printf 'Log       : %s\n' "$LOG_FILE"

(( FAILED == 0 ))