#!/bin/bash

# Pipeline to scrape National Classification of Occupations (NCO) data
# Source: https://dge.gov.in/nat
# Output: pipelines/nat.csv

set -euo pipefail

OUTPUT="pipelines/nat.csv"
BASE_URL="https://dge.gov.in/nat"
PAGES=7

echo "Scraping NCO data from $BASE_URL ($PAGES pages)..."

# Temporary file for JSON data
TEMP_JSON=$(mktemp)
echo "[]" > "$TEMP_JSON"

# Fields to extract
SELECTOR="table.cols-8 tbody tr"
FIELDS="sno:.views-field-counter,title:.views-field-title,nco2015:.views-field-field-nco-2015,nco2004:.views-field-field-nco-2004,division:.views-field-field-div,subdivision:.views-field-field-sub-div,group:.views-field-field-grup,family:.views-field-field-fmly"

for page in $(seq 0 $((PAGES-1))); do
    echo "  Fetching page $((page+1))/$PAGES..."
    URL="${BASE_URL}?page=${page}"
    
    HTML_FILE=$(mktemp)
    curl -s -L -A "Mozilla/5.0" "$URL" > "$HTML_FILE"
    
    IFS=',' read -ra ADDR <<< "$FIELDS"
    TEMP_FILES=()
    FIELD_NAMES=()
    for i in "${ADDR[@]}"; do
        KEY="${i%%:*}"; VAL="${i#*:}"
        FIELD_NAMES+=("$KEY")
        TF=$(mktemp); TEMP_FILES+=("$TF")
        htmlq "$SELECTOR $VAL" --text < "$HTML_FILE" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | tr -s ' ' > "$TF"
    done

    PAGE_JSON=$(mktemp)
    paste "${TEMP_FILES[@]}" | while IFS=$'\t' read -r -a values; do
        if [ "${#values[@]}" -eq 0 ] || [ -z "${values[0]}" ]; then continue; fi
        JQ_ARGS=()
        JQ_OBJ="{"
        for j in "${!FIELD_NAMES[@]}"; do
            KEY="${FIELD_NAMES[$j]}"
            VAL="${values[$j]:-}"
            JQ_ARGS+=("--arg" "$KEY" "$VAL")
            JQ_OBJ="$JQ_OBJ\"$KEY\": \$$KEY,"
        done
        jq -n "${JQ_ARGS[@]}" "${JQ_OBJ%,}}"
    done | jq -s . > "$PAGE_JSON"
    
    # Merge
    NEW_JSON=$(mktemp)
    jq -s 'add' "$TEMP_JSON" "$PAGE_JSON" > "$NEW_JSON"
    mv "$NEW_JSON" "$TEMP_JSON"
    
    rm "$HTML_FILE" "${TEMP_FILES[@]}" "$PAGE_JSON"
done

echo "Saving to CSV: $OUTPUT"
jq -r '(.[0] | keys_unsorted) as $keys | $keys, map([.[$keys[]]])[] | @csv' "$TEMP_JSON" > "$OUTPUT"

echo "Scrape complete. $(jq '. | length' "$TEMP_JSON") records saved to $OUTPUT"
rm "$TEMP_JSON"