import os
import requests
import time
from random import uniform

API_URL = "https://teengovernment.miraheze.org/w/api.php"
OUTPUT_DIR = "wiki_pages"
SLEEP_INTERVAL = 60 * 60 * 6  # 6 hours between fetches
MAX_RETRIES = 5
TIMEOUT = 10  # seconds
BACKOFF_FACTOR = 2

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/118.0.5993.118 Safari/537.36"
}

if not os.path.exists(OUTPUT_DIR):
    os.makedirs(OUTPUT_DIR)

def robust_get(params):
    attempt = 0
    while attempt < MAX_RETRIES:
        try:
            response = requests.get(API_URL, params=params, headers=HEADERS, timeout=TIMEOUT)
            response.raise_for_status()
            return response.json()
        except requests.exceptions.RequestException as e:
            wait = BACKOFF_FACTOR ** attempt + uniform(0, 1)
            print(f"Request failed ({e}), retrying in {wait:.1f}s...")
            time.sleep(wait)
            attempt += 1
    print(f"Failed after {MAX_RETRIES} attempts for params: {params}")
    return None

def fetch_all_pages():
    pages = []
    apcontinue = ""
    while True:
        params = {
            "action": "query",
            "list": "allpages",
            "aplimit": "max",
            "format": "json",
        }
        if apcontinue:
            params["apcontinue"] = apcontinue

        response = robust_get(params)
        if not response:
            break

        pages.extend(response.get("query", {}).get("allpages", []))
        if "continue" in response:
            apcontinue = response["continue"]["apcontinue"]
        else:
            break
        time.sleep(uniform(0.5, 1.5))  # short random delay between requests

    return [p["title"] for p in pages]

def fetch_page_content(title):
    params = {
        "action": "parse",
        "page": title,
        "prop": "wikitext",
        "format": "json"
    }
    response = robust_get(params)
    if response and "parse" in response:
        return response["parse"]["wikitext"]["*"]
    return None

def save_page(title, content):
    safe_title = title.replace("/", "_").replace("\\", "_")
    filepath = os.path.join(OUTPUT_DIR, f"{safe_title}.txt")
    with open(filepath, "w", encoding="utf-8") as f:
        f.write(content)

def fetch_and_store_all_pages():
    all_pages = fetch_all_pages()
    print(f"Found {len(all_pages)} pages.")
    for title in all_pages:
        content = fetch_page_content(title)
        if content:
            save_page(title, content)
            print(f"Saved: {title}")
        else:
            print(f"Failed to fetch: {title}")
        time.sleep(uniform(1, 3))  # polite delay between page fetches

if __name__ == "__main__":
    while True:
        try:
            fetch_and_store_all_pages()
        except Exception as e:
            print("Unexpected error:", e)
        print(f"Sleeping for {SLEEP_INTERVAL} seconds...")
        time.sleep(SLEEP_INTERVAL)