#!/bin/bash

INPUT_DIR="$HOME/scrape_data/builtinboston"
OUTPUT="$INPUT_DIR/companies.jsonl"

> "$OUTPUT"

for page in $(seq 1 158); do
    file="$INPUT_DIR/page_${page}.html"
    [ -f "$file" ] || { echo "missing page $page" >&2; continue; }

    names_f=$(mktemp)
    slugs_f=$(mktemp)
    industries_f=$(mktemp)
    employees_f=$(mktemp)

    htmlq '.company-info-section h2' --text < "$file" > "$names_f"
    htmlq '.company-info-section a[data-track-id="title"]' --attribute href < "$file" > "$slugs_f"
    htmlq '.company-info-section .mb-xl-sm' --text < "$file" > "$industries_f"
    htmlq '.company-stats-grid > div:nth-child(2) span' --text < "$file" > "$employees_f"

    paste "$names_f" "$slugs_f" "$industries_f" "$employees_f" | while IFS=$'\t' read -r name slug industries employees; do
        [ -z "$name" ] && continue
        jq -n \
            --arg name "$name" \
            --arg url "https://www.builtinboston.com${slug}" \
            --arg industries "$industries" \
            --arg employees "$employees" \
            '{name: $name, url: $url, industries: $industries, employees: $employees}'
    done >> "$OUTPUT"

    rm -f "$names_f" "$slugs_f" "$industries_f" "$employees_f"
    echo "parsed page $page"
done

echo "Done. $(wc -l < "$OUTPUT") companies written to $OUTPUT"