#!/bin/bash

# Configuration
DATA_FILE="/Users/bobby/web/shrewsbury/indian_usa.txt.gz"
SCRAPE_DIR="/Users/bobby/web/shrewsbury/data/scraped_html"
CONCURRENCY_LIMIT=30

mkdir -p "$SCRAPE_DIR"

# 1. Extract URLs and Names from the gzipped dataset
echo "Extracting targets..."
targets=$(gunzip -c "$DATA_FILE" | awk -F'|' '$7 ~ /^http/ {print $1"|"$7}')
total=$(echo "$targets" | wc -l)
echo "Found $total websites to scrape."

# 2. Parallel Download Loop
count=0
temp_targets=$(mktemp)
echo "$targets" > "$temp_targets"

while IFS='|' read -r name url; do
    # Sanitize filename
    filename=$(echo "$name" | sed 's/[^a-zA-Z0-9]/_/g' | tr '[:upper:]' '[:lower:]')
    filepath="$SCRAPE_DIR/${filename}_${count}.html"

    if [ ! -f "$filepath" ]; then
        # Direct curl for high speed
        curl -s -L -m 15 -A "Mozilla/5.0" "$url" > "$filepath" &
    fi
    
    ((count++))

    # Progress report
    if (( count % 10 == 0 )); then
        printf "\rProgress: %d/%d" "$count" "$total"
    fi

    # Concurrency control for Bash 3.2 (macOS default)
    while [ $(jobs -p | wc -l) -ge $CONCURRENCY_LIMIT ]; do
        sleep 0.1
    done

done < "$temp_targets"

wait
rm "$temp_targets"
echo -e "\nDownload phase complete. Files are in $SCRAPE_DIR"