#!/bin/bash

OUTPUT_DIR="$HOME/scrape_data/builtinboston"
TOTAL_PAGES=158
CONCURRENCY=10

mkdir -p "$OUTPUT_DIR"

fetch_page() {
    local page=$1
    local out="$OUTPUT_DIR/page_${page}.html"
    if [ -f "$out" ]; then
        echo "skip $page"
        return
    fi
    curl -s -L -A "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" \
        "https://www.builtinboston.com/companies?handler=SearchResults&page=${page}" \
        -o "$out"
    echo "done $page"
}

export -f fetch_page
export OUTPUT_DIR

for page in $(seq 1 $TOTAL_PAGES); do
    fetch_page "$page" &
    while [ $(jobs -p | wc -l) -ge $CONCURRENCY ]; do
        sleep 0.2
    done
done

wait
echo "All $TOTAL_PAGES pages downloaded to $OUTPUT_DIR"