import os
import sys
import json
import glob
import inspect
import logging
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed

SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))
ROOT_DIR = os.path.dirname(SCRIPTS_DIR)

if ROOT_DIR not in sys.path:
    sys.path.insert(0, ROOT_DIR)

from pinscrape.v2 import fetch_related_keywords
from pinscrape.database import get_pending_relatedkw_keywords, BatchWriter
from pinscrape.utils import load_proxies


def _verify_compat():
    if "include_empty" not in inspect.signature(get_pending_relatedkw_keywords).parameters:
        raise SystemExit(
            "ERROR: pinscrape/database.py di mesin ini versi lama (tidak punya parameter include_empty).\n"
            "File project tidak sinkron. Salin SEMUA file dari zip update "
            "(pinscrape/v2.py, pinscrape/database.py, scripts/scrape.py, scripts/scrape_related_kw.py,\n"
            "scripts/scrape_description.py, scripts/cli.py, run.bat) ke mesin ini, jangan sebagian, lalu jalankan ulang."
        )


_verify_compat()

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    datefmt="%H:%M:%S"
)
logger = logging.getLogger(__name__)


def load_config():
    config_path = os.path.join(ROOT_DIR, "config.json")
    defaults = {
        "concurrency": 5,
        "max_retries": 3,
        "retry_delay": 3,
        "max_related_kw": 10,
        "suggest_hl": "en",
        "suggest_gl": "US",
        "db_batch_size": 100,
    }
    if os.path.exists(config_path):
        with open(config_path, "r") as f:
            cfg = json.load(f)
        defaults.update(cfg)
    return defaults


def scrape_keyword_related(db_path, item, writer, proxy_list, config):
    keyword_id = item["id"]
    keyword = item["keyword"]
    max_related = config.get("max_related_kw", 10)
    hl = config.get("suggest_hl", "en")
    gl = config.get("suggest_gl", "US")

    related_kw = fetch_related_keywords(
        keyword, proxy_list=proxy_list, max_results=max_related, hl=hl, gl=gl
    )

    writer.update_related_kw(keyword_id, related_kw)

    return {
        "id": keyword_id,
        "keyword": keyword,
        "related_count": len(related_kw),
        "error": None if related_kw else "empty (re-scrape on demand)",
    }


def scrape_database_related(db_path, proxy_list, config, include_empty=False):
    db_name = os.path.basename(db_path)
    logger.info(f"\n{'='*50}")
    logger.info(f"Processing: {db_name}")
    logger.info(f"{'='*50}")

    pending = get_pending_relatedkw_keywords(db_path, include_empty=include_empty)
    if not pending:
        logger.info(f"No pending related keywords in {db_name}")
        return

    mode = "re-scrape empty + first-time" if include_empty else "first-time only"
    logger.info(f"Found {len(pending)} keywords pending related_kw ({mode})")
    concurrency = config["concurrency"]
    completed = 0
    lock = threading.Lock()
    writer = BatchWriter(db_path, batch_size=config.get("db_batch_size", 100)).start()

    def progress_callback(future):
        nonlocal completed
        result = future.result()
        with lock:
            completed += 1
            status = "OK" if not result["error"] else f"FAIL ({result['error']})"
            logger.info(f"  [{completed}/{len(pending)}] {result['keyword']} -> "
                        f"{result['related_count']} related [{status}]")

    try:
        with ThreadPoolExecutor(max_workers=concurrency) as executor:
            futures = {}
            for item in pending:
                future = executor.submit(
                    scrape_keyword_related,
                    db_path, item, writer,
                    proxy_list, config
                )
                futures[future] = item
                future.add_done_callback(progress_callback)

            for future in as_completed(futures):
                pass
    finally:
        writer.flush()
        writer.close()

    logger.info(f"Completed {db_name}: {completed}/{len(pending)} keywords processed")


def find_databases():
    db_dir = os.path.join(ROOT_DIR, "database")
    if not os.path.exists(db_dir):
        return []
    return sorted(glob.glob(os.path.join(db_dir, "*.sqlite")))


def main(include_empty=True):
    config = load_config()
    logger.info(f"Config: max_related_kw={config.get('max_related_kw', 10)}, "
                f"suggest_hl={config.get('suggest_hl', 'en')}, suggest_gl={config.get('suggest_gl', 'US')}, "
                f"concurrency={config['concurrency']}, include_empty={include_empty}, "
                f"db_batch_size={config.get('db_batch_size', 100)}")

    proxy_list = load_proxies(os.path.join(ROOT_DIR, "proxies.txt"))
    logger.info(f"Loaded {len(proxy_list)} proxies")

    databases = find_databases()
    if not databases:
        logger.warning("No .sqlite files found in database/ folder")
        return

    logger.info(f"Found {len(databases)} database(s): {[os.path.basename(d) for d in databases]}")

    for db_path in databases:
        scrape_database_related(db_path, proxy_list, config, include_empty=include_empty)

    logger.info("\nAll databases processed!")


if __name__ == "__main__":
    main()