#!/usr/bin/env python3
"""Collect wiki statistics and append to time-series data.

Measures: total pages, new/updated/removed pages, categories,
external sites, internal links, pages accessed, edited pages, and
visitor counts for the public wiki site (/wiki/) and the MCP endpoint
(/mcp/), derived from the nginx access log.

Output: /home/dev/git/wiki/stats/data.json — time-series JSON.

Intended to run every 15 min via wiki-stats.timer.

Performance notes
-----------------
The corpus is ~300k raw .md files on disk, so we do ALL per-file work in a
single traversal (file count, categories, link counts, access/edit mtimes)
instead of the old 4 separate full scans. That keeps a run to roughly one
pass over the tree instead of four, and caps memory (we never accumulate
the whole corpus in RAM — we only ever hold per-file text, released
immediately).
"""
from __future__ import annotations

import json
import os
import re
import subprocess
import sys
import time
from datetime import date, datetime
from pathlib import Path
from typing import Any

WIKI_DIR = Path("/home/dev/git/wiki").resolve()
STATS_DIR = WIKI_DIR / "stats"
DATA_FILE = STATS_DIR / "data.json"

# Previous data file — used for computing deltas (new, updated, removed)
PREV_FILE = STATS_DIR / "data-prev.json"

# nginx access log (combined format). Read access requires the user running
# collect.py to be able to read it (www-data:adm, mode 640). If unreadable we
# record visits as null and flag visits_available=False rather than failing.
NGINX_ACCESS_LOG = Path("/home/dev/git/wiki/stats/wiki-access.log")
# Fallback if the dedicated log isn't set up yet: the main nginx log.
NGINX_ACCESS_LOG_MAIN = Path("/var/log/nginx/access.log")

# Cache for the expensive link counts (they change slowly). Refreshed at most
# every LINK_CACHE_TTL seconds so the 15-min timer run stays fast.
LINK_CACHE_FILE = STATS_DIR / "linkcounts.cache.json"
LINK_CACHE_TTL = 6 * 3600  # 6 hours

# Disk usage (`du` over ~300k files) is slow (minutes), so cache it and only
# refresh every DISK_CACHE_TTL seconds — the value changes slowly.
DISK_CACHE_FILE = STATS_DIR / "diskusage.cache.json"
DISK_CACHE_TTL = 3 * 3600  # 3 hours


# Compiled patterns (module-level so they are compiled once per run)
WIKILINK_RE = re.compile(r"\[\[([^\]]+)\]\]")
URL_RE = re.compile(r"https?://[^\s\)\]<>\"']+")
# Combined-format request line:  ... "GET /wiki/foo HTTP/1.1" 200 ...
REQUEST_RE = re.compile(r'"(?:GET|POST|HEAD|OPTIONS|PUT|DELETE)\s+(\S+)\s+HTTP')
# Combined-format time_local bracket:  [... 18/Jul/2026:22:47:35 +0200]
TIME_RE = re.compile(r"\[([^\]]+)\]")


def _ensure_dir() -> None:
    STATS_DIR.mkdir(parents=True, exist_ok=True)


def _disk_usage() -> dict[str, Any]:
    """Wiki corpus disk usage via `du -h -d 1 <WIKI_DIR> | tail -1`.

    `du` over ~300k files is slow (minutes), so the result is cached and only
    refreshed every DISK_CACHE_TTL seconds. Returns {human, bytes}; on any
    failure (including a slow `du` exceeding the timeout) returns the cached
    value if present, else {human: None, bytes: None} so the page shows a
    graceful "unavailable" rather than crashing the whole collect run.
    """
    # Serve from cache when fresh.
    if DISK_CACHE_FILE.exists():
        try:
            cached = json.loads(DISK_CACHE_FILE.read_text(encoding="utf-8"))
            if time.time() - cached.get("timestamp", 0) < DISK_CACHE_TTL:
                return {"human": cached.get("human"), "bytes": cached.get("bytes")}
        except (json.JSONDecodeError, OSError, KeyError):
            pass
    # Compute fresh (slow). The subprocess timeout is generous because `du`
    # over the full corpus takes minutes; the result is cached afterwards.
    try:
        out = subprocess.run(
            ["du", "-h", "-d", "1", str(WIKI_DIR)],
            capture_output=True, text=True, timeout=900,
        )
        if out.returncode != 0 or not out.stdout.strip():
            cached = _read_disk_cache()
            return cached or {"human": None, "bytes": None}
        last = out.stdout.strip().splitlines()[-1]
        # du output: "<size>\t<path>"  e.g. "3.2G\t/home/dev/git/wiki"
        size = last.split("\t")[0].strip()
        if not size:
            cached = _read_disk_cache()
            return cached or {"human": None, "bytes": None}
        mult = {"K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4, "P": 1024**5}
        unit = size[-1] if size[-1] in mult else ""
        num = float(size[:-1]) if unit else float(size)
        bytes_ = int(num * mult.get(unit, 1)) if unit else int(num)
        result = {"human": size, "bytes": bytes_}
        try:
            DISK_CACHE_FILE.write_text(
                json.dumps({"timestamp": time.time(), **result}, indent=2),
                encoding="utf-8",
            )
        except OSError:
            pass
        return result
    except (subprocess.SubprocessError, ValueError, OSError):
        cached = _read_disk_cache()
        return cached or {"human": None, "bytes": None}


def _read_disk_cache() -> dict[str, Any] | None:
    if DISK_CACHE_FILE.exists():
        try:
            data = json.loads(DISK_CACHE_FILE.read_text(encoding="utf-8"))
            return {"human": data.get("human"), "bytes": data.get("bytes")}
        except (json.JSONDecodeError, OSError, KeyError):
            pass
    return None


def _walk_stats_only() -> dict[str, Any]:
    """Fast traversal: file counts, categories, access/edit mtimes only.

    Does NOT read file bodies, so it stays cheap (one lstat-class stat per
    file). This is what the frequent (15-min) timer run uses.
    """
    total = 0
    categories: set[str] = set()
    accessed = 0
    edited = 0

    today = date.today()
    today_ts = time.mktime(today.timetuple())

    for root, dirs, files in os.walk(WIKI_DIR):
        rel_root = Path(root).relative_to(WIKI_DIR)
        parts = rel_root.parts
        if parts and parts[0] == "stats":
            dirs[:] = []
            continue
        dirs[:] = [d for d in dirs if not d.startswith(".")]

        for name in files:
            if name.startswith("."):
                continue
            suffix = os.path.splitext(name)[1].lower()
            if suffix not in (".md", ".html"):
                continue
            total += 1
            if len(parts) >= 1:
                categories.add(parts[-1])
            try:
                st = os.stat(os.path.join(root, name))
            except OSError:
                continue
            if st.st_atime >= today_ts:
                accessed += 1
            if st.st_mtime >= today_ts:
                edited += 1

    return {
        "total": total,
        "categories": len(categories),
        "accessed": accessed,
        "edited": edited,
    }


def _walk_links() -> dict[str, int]:
    """Expensive traversal: read every page body and count link occurrences.

    Runs rarely (cached for LINK_CACHE_TTL). Returns internal/external counts.
    """
    internal_links = 0
    external_links = 0
    for root, dirs, files in os.walk(WIKI_DIR):
        rel_root = Path(root).relative_to(WIKI_DIR)
        parts = rel_root.parts
        if parts and parts[0] == "stats":
            dirs[:] = []
            continue
        dirs[:] = [d for d in dirs if not d.startswith(".")]
        for name in files:
            if name.startswith("."):
                continue
            suffix = os.path.splitext(name)[1].lower()
            if suffix not in (".md", ".html"):
                continue
            fpath = Path(root) / name
            try:
                text = fpath.read_text(encoding="utf-8", errors="ignore")
            except OSError:
                continue
            internal_links += len(WIKILINK_RE.findall(text))
            external_links += len(URL_RE.findall(text))
    return {"internal_links": internal_links, "external_links": external_links}


def _load_link_cache() -> tuple[dict[str, int], float]:
    """Return (counts, age_seconds). Empty/old cache => refresh needed."""
    if LINK_CACHE_FILE.exists():
        try:
            data = json.loads(LINK_CACHE_FILE.read_text(encoding="utf-8"))
            age = time.time() - data.get("timestamp", 0)
            return data.get("counts", {}), age
        except (json.JSONDecodeError, OSError, KeyError):
            pass
    return {}, 1e9


def _save_link_cache(counts: dict[str, int]) -> None:
    LINK_CACHE_FILE.write_text(
        json.dumps({"timestamp": time.time(), "counts": counts}, indent=2),
        encoding="utf-8",
    )


def _link_counts() -> dict[str, int]:
    """Return cached link counts, refreshing the cache if stale/missing."""
    cached, age = _load_link_cache()
    if cached and age < LINK_CACHE_TTL:
        return cached
    counts = _walk_links()
    _save_link_cache(counts)
    return counts


def _count_visits() -> tuple[int | None, int | None, bool]:
    """Count today's requests to /wiki/ and /mcp/ from the nginx access log.

    Prefers the dedicated, dev-readable log (wiki-access.log) written by nginx
    for the /wiki/ and /mcp/ locations; falls back to the main access log if
    the dedicated one isn't set up. Only hits stamped with today's date count.

    Returns (wiki_visits, mcp_visits, available). When no readable log exists,
    returns (None, None, False) and the page shows a graceful "unavailable"
    notice instead of crashing.
    """
    log_path = NGINX_ACCESS_LOG if NGINX_ACCESS_LOG.is_file() else NGINX_ACCESS_LOG_MAIN
    if not log_path.is_file():
        return None, None, False
    try:
        today = date.today()
        wiki = 0
        mcp = 0
        with log_path.open("r", encoding="utf-8", errors="ignore") as fh:
            for line in fh:
                # Parse the date from the leading bracketed field so we only
                # count today's hits (the log may span many days).
                tm = TIME_RE.search(line)
                if tm:
                    try:
                        ts = datetime.strptime(tm.group(1), "%d/%b/%Y:%H:%M:%S %z")
                        if ts.date() != today:
                            continue
                    except ValueError:
                        # If the timestamp is unparseable, still count it so a
                        # weird line doesn't silently drop a hit.
                        pass
                m = REQUEST_RE.search(line)
                if not m:
                    continue
                path = m.group(1)
                if path.startswith("/wiki"):
                    wiki += 1
                elif path.startswith("/mcp"):
                    mcp += 1
        return wiki, mcp, True
    except (OSError, PermissionError):
        # No read access (e.g. log is www-data:adm mode 640 and we are not in
        # adm). Degrade gracefully instead of crashing the whole collector.
        return None, None, False


def _load_data() -> list[dict[str, Any]]:
    if DATA_FILE.exists():
        try:
            return json.loads(DATA_FILE.read_text(encoding="utf-8"))
        except (json.JSONDecodeError, OSError):
            return []
    return []


def _save_data(entries: list[dict[str, Any]]) -> None:
    DATA_FILE.write_text(json.dumps(entries, indent=2), encoding="utf-8")


def _load_prev() -> dict[str, Any]:
    if PREV_FILE.exists():
        try:
            return json.loads(PREV_FILE.read_text(encoding="utf-8"))
        except (json.JSONDecodeError, OSError):
            return {}
    return {}


def _save_prev(entry: dict[str, Any]) -> None:
    PREV_FILE.write_text(json.dumps(entry, indent=2), encoding="utf-8")


def main() -> int:
    _ensure_dir()

    corp = _walk_stats_only()
    total = corp["total"]
    categories = corp["categories"]
    accessed = corp["accessed"]
    edited = corp["edited"]

    # Link counts are expensive (reads every page body); served from a cache
    # that refreshes at most every LINK_CACHE_TTL seconds.
    links = _link_counts()
    internal_links = links["internal_links"]
    external_links = links["external_links"]

    wiki_visits, mcp_visits, visits_available = _count_visits()

    disk = _disk_usage()

    # Compute deltas from previous run
    prev = _load_prev()
    prev_total = prev.get("total_pages", total)
    new_pages = max(0, total - prev_total)
    removed_pages = max(0, prev_total - total)

    entry: dict[str, Any] = {
        "date": datetime.now().strftime("%Y-%m-%d"),
        "timestamp": time.time(),
        "total_pages": total,
        "new_pages": new_pages,
        "removed_pages": removed_pages,
        "updated_pages": edited,  # files modified today
        "categories": categories,
        "external_links": external_links,
        "internal_links": internal_links,
        "pages_accessed": accessed,
        "pages_edited": edited,
        "wiki_visits": wiki_visits,
        "mcp_visits": mcp_visits,
        "visits_available": visits_available,
        "disk_usage_human": disk["human"],
        "disk_usage_bytes": disk["bytes"],
    }

    entries = _load_data()
    # Append every sample (time-series). Do NOT collapse to one row per day —
    # the dashboard plots the staircase; collapsing made "new_pages" look
    # permanently 0 because only the last run's row survived. The per-run
    # delta vs data-prev.json already captures real growth between samples.
    entries.append(entry)

    _save_data(entries)
    _save_prev(entry)

    if visits_available:
        vis = f", {wiki_visits} wiki visits, {mcp_visits} mcp visits"
    else:
        vis = " (visitor counts unavailable — cannot read nginx access log)"
    print(
        f"✅ Wiki stats collected: {total} pages, {categories} categories, "
        f"{internal_links} internal links, {external_links} external links{vis}"
    )

    # Prune to last 365 entries
    if len(entries) > 365:
        _save_data(entries[-365:])

    return 0


if __name__ == "__main__":
    raise SystemExit(main())