Onnowpurbo: Created page with "

  #!/usr/bin/env bash # Batch wrapper untuk pdf_to_rag_md.py # # Penggunaan: #   ./batch_pdf_to_md.sh /path/folder_pdf #   ./batch_pdf_to_md.sh /path/folder_pdf /path/fol..."

2026-07-26T01:47:44Z

Created page with "<pre> #!/usr/bin/env bash # Batch wrapper untuk pdf_to_rag_md.py # # Penggunaan: # ./batch_pdf_to_md.sh /path/folder_pdf # ./batch_pdf_to_md.sh /path/folder_pdf /path/fol..."

New page

<pre>

#!/usr/bin/env bash
# Batch wrapper untuk pdf_to_rag_md.py
#
# Penggunaan:
# ./batch_pdf_to_md.sh /path/folder_pdf
# ./batch_pdf_to_md.sh /path/folder_pdf /path/folder_output
# ./batch_pdf_to_md.sh /path/folder_pdf /path/folder_output /path/pdf_to_rag_md.py
#
# Hasil:
# OUTPUT_DIR/<path-relatif>/<nama-pdf>/
# ├── *.md
# └── manifest.json

set -Eeuo pipefail
IFS=$'\n\t'

PROGRAM_NAME="$(basename "$0")"
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"

usage() {
cat <<EOF
Penggunaan:
$PROGRAM_NAME INPUT_PATH [OUTPUT_DIR] [PYTHON_SCRIPT]

Argumen:
INPUT_PATH Folder yang akan dicari secara rekursif, atau satu file PDF.
OUTPUT_DIR Folder induk hasil konversi.
Default: INPUT_PATH/output untuk input folder,
atau folder_file/output untuk input satu PDF.
PYTHON_SCRIPT Lokasi script Python konverter.
Default: dicari di folder yang sama dengan shell script.

Contoh:
$PROGRAM_NAME /opt/ai-stack/documents
$PROGRAM_NAME /opt/ai-stack/documents /opt/ai-stack/output
$PROGRAM_NAME ./pdf ./output ./pdf_to_rag_md.py
EOF
}

log() {
printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"
}

fail() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}

if [[ ${1:-} == "-h" || ${1:-} == "--help" ]]; then
usage
exit 0
fi

[[ $# -ge 1 && $# -le 3 ]] || {
usage >&2
exit 2
}

command -v python3 >/dev/null 2>&1 || fail "python3 tidak ditemukan."
command -v find >/dev/null 2>&1 || fail "Perintah find tidak ditemukan."

INPUT_PATH="$1"
[[ -e "$INPUT_PATH" ]] || fail "Path input tidak ditemukan: $INPUT_PATH"

# Ubah menjadi absolute path tanpa membutuhkan realpath.
if [[ -d "$INPUT_PATH" ]]; then
INPUT_PATH="$(cd -- "$INPUT_PATH" && pwd -P)"
INPUT_IS_DIR=1
else
INPUT_DIR="$(cd -- "$(dirname -- "$INPUT_PATH")" && pwd -P)"
INPUT_PATH="$INPUT_DIR/$(basename -- "$INPUT_PATH")"
INPUT_IS_DIR=0
fi

if [[ $# -ge 2 ]]; then
OUTPUT_DIR="$2"
else
if (( INPUT_IS_DIR )); then
OUTPUT_DIR="$INPUT_PATH/output"
else
OUTPUT_DIR="$(dirname -- "$INPUT_PATH")/output"
fi
fi
mkdir -p -- "$OUTPUT_DIR"
OUTPUT_DIR="$(cd -- "$OUTPUT_DIR" && pwd -P)"
[[ "$OUTPUT_DIR" != "$INPUT_PATH" ]] \
|| fail "OUTPUT_DIR tidak boleh sama persis dengan folder input. Gunakan subfolder, misalnya: $INPUT_PATH/output"

if [[ $# -ge 3 ]]; then
PYTHON_SCRIPT="$3"
else
if [[ -f "$SCRIPT_DIR/pdf_to_rag_md.py" ]]; then
PYTHON_SCRIPT="$SCRIPT_DIR/pdf_to_rag_md.py"
elif [[ -f "$SCRIPT_DIR/pdf_to_rag_md(2).py" ]]; then
PYTHON_SCRIPT="$SCRIPT_DIR/pdf_to_rag_md(2).py"
else
fail "Script Python tidak ditemukan. Letakkan pdf_to_rag_md.py di $SCRIPT_DIR atau berikan argumen PYTHON_SCRIPT."
fi
fi

[[ -f "$PYTHON_SCRIPT" ]] || fail "Script Python tidak ditemukan: $PYTHON_SCRIPT"
PYTHON_SCRIPT="$(cd -- "$(dirname -- "$PYTHON_SCRIPT")" && pwd -P)/$(basename -- "$PYTHON_SCRIPT")"

# Pastikan script Python valid dan dependensinya tersedia sebelum batch dimulai.
python3 -m py_compile "$PYTHON_SCRIPT" \
|| fail "Script Python gagal diperiksa dengan py_compile."

python3 - <<'PY' \
|| fail "Dependensi Python belum lengkap. Instal dengan: python3 -m pip install pymupdf pyyaml"
try:
import pymupdf # noqa: F401
except ImportError:
import fitz # noqa: F401
import yaml # noqa: F401
PY

find_input_files() {
if [[ "$OUTPUT_DIR" == "$INPUT_PATH/"* ]]; then
find "$INPUT_PATH" -path "$OUTPUT_DIR" -prune -o -type f "$@" -print0
else
find "$INPUT_PATH" -type f "$@" -print0
fi
}

printf '\n=== Daftar semua file pada input ===\n'
if (( INPUT_IS_DIR )); then
while IFS= read -r -d '' file; do
printf '%s\n' "$file"
done < <(find_input_files | sort -z)
else
printf '%s\n' "$INPUT_PATH"
fi
printf '=== Akhir daftar file ===\n\n'

PDF_FILES=()
if (( INPUT_IS_DIR )); then
while IFS= read -r -d '' pdf; do
PDF_FILES+=("$pdf")
done < <(find_input_files -iname '*.pdf' | sort -z)
else
[[ "${INPUT_PATH,,}" == *.pdf ]] \
|| fail "File input bukan PDF: $INPUT_PATH"
PDF_FILES+=("$INPUT_PATH")
fi

TOTAL=${#PDF_FILES[@]}
if (( TOTAL == 0 )); then
log "Tidak ada file PDF yang ditemukan."
exit 0
fi

LOG_FILE="$OUTPUT_DIR/batch_conversion.log"
SUMMARY_FILE="$OUTPUT_DIR/batch_summary.tsv"
printf 'status\tinput_pdf\toutput_folder\n' > "$SUMMARY_FILE"
: > "$LOG_FILE"

log "Ditemukan $TOTAL file PDF."
log "Output utama: $OUTPUT_DIR"
log "Log batch: $LOG_FILE"

SUCCESS=0
FAILED=0
INDEX=0

for PDF_FILE in "${PDF_FILES[@]}"; do
((INDEX += 1))

if (( INPUT_IS_DIR )); then
RELATIVE_PATH="${PDF_FILE#"$INPUT_PATH"/}"
else
RELATIVE_PATH="$(basename -- "$PDF_FILE")"
fi

RELATIVE_PARENT="$(dirname -- "$RELATIVE_PATH")"
PDF_NAME="$(basename -- "$RELATIVE_PATH")"
PDF_STEM="${PDF_NAME%.*}"

if [[ "$RELATIVE_PARENT" == "." ]]; then
PDF_OUTPUT_DIR="$OUTPUT_DIR/$PDF_STEM"
else
PDF_OUTPUT_DIR="$OUTPUT_DIR/$RELATIVE_PARENT/$PDF_STEM"
fi

mkdir -p -- "$PDF_OUTPUT_DIR"

printf '\n[%d/%d] Konversi: %s\n' "$INDEX" "$TOTAL" "$PDF_FILE"
printf ' Output : %s\n' "$PDF_OUTPUT_DIR"

printf '\n===== [%d/%d] %s =====\n' "$INDEX" "$TOTAL" "$PDF_FILE" \
| tee -a "$LOG_FILE"

set +e
python3 "$PYTHON_SCRIPT" \
"$PDF_FILE" \
--output-dir "$PDF_OUTPUT_DIR" \
--overwrite 2>&1 | tee -a "$LOG_FILE"
EXIT_CODE=${PIPESTATUS[0]}
set -e

if (( EXIT_CODE == 0 )); then
printf 'SUCCESS\t%s\t%s\n' "$PDF_FILE" "$PDF_OUTPUT_DIR" \
>> "$SUMMARY_FILE"
((SUCCESS += 1))
log "BERHASIL: $PDF_FILE" | tee -a "$LOG_FILE"
else
printf 'FAILED(%d)\t%s\t%s\n' "$EXIT_CODE" "$PDF_FILE" "$PDF_OUTPUT_DIR" \
>> "$SUMMARY_FILE"
((FAILED += 1))
log "GAGAL (exit $EXIT_CODE): $PDF_FILE" | tee -a "$LOG_FILE"
fi
done

printf '\n=== Ringkasan batch ===\n'
printf 'Total PDF : %d\n' "$TOTAL"
printf 'Berhasil : %d\n' "$SUCCESS"
printf 'Gagal : %d\n' "$FAILED"
printf 'Output : %s\n' "$OUTPUT_DIR"
printf 'Ringkasan : %s\n' "$SUMMARY_FILE"
printf 'Log : %s\n' "$LOG_FILE"

(( FAILED == 0 ))

</pre>

Batch pdf to md.sh - Revision history

Onnowpurbo: Created page with "
#!/usr/bin/env bash # Batch wrapper untuk pdf_to_rag_md.py # # Penggunaan: # ./batch_pdf_to_md.sh /path/folder_pdf # ./batch_pdf_to_md.sh /path/folder_pdf /path/fol..."