#!/bin/bash
# Scrapes all book cat_ids from avkf.org across all alpha categories.
# Outputs newline-delimited cat_ids to stdout (or saves to avkf_links.txt).
# Usage: ./avkf-links.sh [output_file]
#        ./avkf-links.sh | db-push avkf_books

BASE_URL="https://www.avkf.org/BookLink/view_titles.php"
ALPHA_IDS="A-B C-D E-F G-H I-J K-L M-N O-P Q-R S-T U-V W-X Y-Z"
OUTPUT="${1:-avkf_links.txt}"
COOKIE_JAR=$(mktemp)
DELAY=0.5  # seconds between requests — be polite

> "$OUTPUT"  # clear output file
total_found=0

for alpha in $ALPHA_IDS; do
    echo "▶ Processing $alpha ..." >&2

    # Step 1: Hit alpha page to set session cookie + get total count
    first_page=$(curl -s -c "$COOKIE_JAR" -b "$COOKIE_JAR" \
        -A "Mozilla/5.0" \
        "$BASE_URL?alpha_id=$alpha")

    total=$(echo "$first_page" | grep -oiE 'there are [0-9]+ book' | grep -oE '[0-9]+')
    if [ -z "$total" ]; then
        echo "  ⚠ Could not find book count for $alpha, skipping" >&2
        continue
    fi
    echo "  Found $total books" >&2

    # Step 2: Extract cat_ids from page 1
    echo "$first_page" | grep -oE 'cat_id=[0-9]+' | sed 's/cat_id=//' >> "$OUTPUT"

    # Step 3: Paginate through remaining pages (step of 7)
    page_num=8
    while [ "$page_num" -le "$total" ]; do
        sleep "$DELAY"
        curl -s -b "$COOKIE_JAR" \
            -A "Mozilla/5.0" \
            "$BASE_URL?t_current_number=$page_num" \
            | grep -oE 'cat_id=[0-9]+' | sed 's/cat_id=//' >> "$OUTPUT"
        page_num=$((page_num + 7))
    done

    count=$(wc -l < "$OUTPUT")
    echo "  Running total: $count cat_ids" >&2
    sleep 1
done

rm -f "$COOKIE_JAR"

# Deduplicate and report
sort -u "$OUTPUT" -o "$OUTPUT"
total_found=$(wc -l < "$OUTPUT")
echo "✓ Done. $total_found unique book links saved to $OUTPUT" >&2

# Emit as JSON array for piping to db-push
jq -Rn '[inputs | {cat_id: .}]' "$OUTPUT"