#!/usr/bin/env python3
"""NC-05 · What an automated scanner asks a business WordPress site for, and what the site answers.

Input : the privacy-stripped nginx access-log extract made by ../nc-02/extract_server.py (columns: ts_utc, method, path, has_query,
        status, bytes, peer, robots_disallowed, ua). The extract holds every visitor's path and time, so it is never published.
Output: aggregate files only, written to this folder (the page reads them, so a clean checkout of main can rebuild it):
        tong-quan-<stamp>.json            window, totals, the moment the scanner-facing layer changed
        nhom-do-quet-<stamp>.csv          per group: requests, share of all requests, status classes
        theo-ngay-<stamp>.csv             per day (UTC+7): all requests, scanner-type requests, per group
        truoc-sau-<stamp>.csv             per group and phase (before / after the layer changed): status, requests, median bytes
        duong-dan-hay-gap-<stamp>.csv     the most requested paths per group
        request-do-quet-<stamp>.csv.gz    one row per scanner-type request: time, method, path, status, bytes, group (no IP, no user-agent)

Usage : python3 analyze.py <extract.tsv.gz> <stamp>
Method: a request is "scanner-type" when its PATH matches one of the groups below, checked in the order listed (each request is counted
        once). The path says what was asked for, not who asked: a request can come from a person, a monitoring tool or a bot, and the
        log cannot tell. Status classes are what the origin server answered; Cloudflare may stop some requests before they reach it.
"""
import collections
import csv
import gzip
import json
import os
import re
import statistics
import sys
from datetime import datetime, timedelta, timezone

HERE = os.path.dirname(os.path.abspath(__file__))
ICT = timezone(timedelta(hours=7))

# (key, label, what the path says, regex on the lower-cased path). Order matters: first match wins.
GROUPS = [
    ('xmlrpc', 'XML-RPC', 'Tệp xmlrpc.php của WordPress, nơi thử đăng nhập hàng loạt', re.compile(r'/xmlrpc\.php$')),
    ('login', 'Cửa đăng nhập và quản trị', 'wp-login.php và hai lối vào /wp-admin/ và /wp-admin/index.php',
     re.compile(r'^/wp-login\.php$|^/wp-admin/?$|^/wp-admin/index\.php$')),
    ('secret', 'Tệp cấu hình và khoá bí mật', '.env, .git, .aws, .ssh, wp-config, .htaccess và tương tự',
     re.compile(r'/\.(env|git|aws|ssh|svn|hg|docker|htaccess|htpasswd|ds_store)|/wp-config|/config\.(php|json|yml|yaml|ini)$|/credentials$')),
    ('backup', 'Tệp sao lưu và dữ liệu', 'Tệp .sql, .bak, .zip, .tar, .gz, .7z, .rar và thư mục backup, dump để lộ dữ liệu',
     re.compile(r'^/(?!wp-content/uploads/).*\.(sql|bak|old|orig|zip|tar|tgz|gz|7z|rar|swp)$|^/(backup|backups|bak|old|db|dump|database)(/|\.|$)')),
    ('dbtool', 'Công cụ quản trị cơ sở dữ liệu', 'phpMyAdmin, Adminer và tên tương tự', re.compile(r'phpmyadmin|/pma(/|$)|adminer|/dbadmin|/mysql(/|$)')),
    ('users', 'Dò tên người dùng qua REST', '/wp-json/wp/v2/users để lấy tên đăng nhập', re.compile(r'/wp-json/wp/v2/users')),
    ('version', 'Dò phiên bản plugin và giao diện', 'readme.txt, license, changelog trong wp-content/plugins và themes',
     re.compile(r'/wp-content/(plugins|themes|mu-plugins)/[^/]+/(readme|license|changelog)')),
    ('core', 'Tệp lõi WordPress gọi trực tiếp', 'Tệp .php có trong lõi WordPress (wp-links-opml.php, wp-signup.php, wp-load.php…) mà người xem bình thường không gọi', None),
    ('shell', 'Tệp PHP lạ và lỗ hổng thường gặp', 'Tệp .php không có trong lõi WordPress (alfa.php, file.php, 1.php…), vendor/phpunit, cgi-bin, ../',
     re.compile(r'\.\./|%2e%2e|/cgi-bin|/vendor/phpunit|eval-stdin|/boaform|/actuator|/bin/sh|/etc/passwd|/shell|'
                r'^/wp-(content|includes)/.*\.php$|^/[^/]+\.php$')),
]
# the .php files WordPress 7.1.2 itself ships (root, wp-includes, wp-content), from https://api.wordpress.org/core/checksums/1.0/?version=7.1.2
CORE_PHP = set(open(os.path.join(HERE, 'wp-core-php-7.1.2.txt'), encoding='utf-8').read().split())
# WordPress's own front controllers are not scanner-type even though they end in .php
LEGIT_PHP = {'/index.php', '/wp-cron.php', '/wp-comments-post.php'}
STATUS_COLS = [('200', '200'), ('3xx', None), ('401', '401'), ('403', '403'), ('404', '404'), ('405', '405'), ('khac', None)]


def klass(status):
    if status == '200':
        return '200'
    if status.startswith('3'):
        return '3xx'
    if status in ('401', '403', '404', '405'):
        return status
    return 'khac'


def classify(path):
    p = re.sub(r'/+', '/', path.lower())
    if p in LEGIT_PHP:
        return None
    for key, _l, _d, rx in GROUPS:
        if key == 'core':
            if p.lstrip('/') in CORE_PHP:
                return 'core'
        elif rx.search(p):
            return key
    return None


def main(extract, stamp):
    rows = []
    for line in gzip.open(extract, 'rt', encoding='utf-8', errors='replace'):
        c = line.rstrip('\n').split('\t')
        if len(c) < 9 or c[0] == 'ts_utc' or c[6] != 'cf':           # only requests that came through Cloudflare (real visitors)
            continue
        rows.append((datetime.strptime(c[0], '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc), c[1], c[2], c[4], int(c[5]) if c[5].isdigit() else 0))
    rows.sort(key=lambda r: r[0])
    n = len(rows)
    t0, t1 = rows[0][0], rows[-1][0]
    hours = (t1 - t0).total_seconds() / 3600
    scan = [(r, classify(r[2])) for r in rows]
    scan = [(r, g) for r, g in scan if g]

    # the moment the layer in front of WordPress changed: the first time xmlrpc.php answered a POST with 403
    t_change = next(r[0] for r, g in scan if g == 'xmlrpc' and r[1] == 'POST' and r[3] == '403')

    by_group = collections.defaultdict(list)
    for r, g in scan:
        by_group[g].append(r)
    glabel = {k: (l, d) for k, l, d, _ in GROUPS}

    # 1 · per group
    with open(os.path.join(HERE, f'nhom-do-quet-{stamp}.csv'), 'w', newline='', encoding='utf-8') as fh:
        w = csv.writer(fh, lineterminator='\n')
        w.writerow(['nhom', 'ten_nhom', 'mo_ta', 'so_request', 'ty_le_tren_tong_request', 'so_request_POST'] + [f'ma_{c}' for c, _ in STATUS_COLS])
        for key, label, desc, _ in GROUPS:
            rs = by_group.get(key, [])
            cnt = collections.Counter(klass(r[3]) for r in rs)
            w.writerow([key, label, desc, len(rs), f'{len(rs) / n:.6f}', sum(1 for r in rs if r[1] == 'POST')] + [cnt.get(c, 0) for c, _ in STATUS_COLS])

    # 2 · per day (UTC+7)
    days = collections.OrderedDict()
    for r in rows:
        d = r[0].astimezone(ICT).strftime('%Y-%m-%d')
        days.setdefault(d, [0, collections.Counter()])[0] += 1
    for r, g in scan:
        days[r[0].astimezone(ICT).strftime('%Y-%m-%d')][1][g] += 1
    with open(os.path.join(HERE, f'theo-ngay-{stamp}.csv'), 'w', newline='', encoding='utf-8') as fh:
        w = csv.writer(fh, lineterminator='\n')
        w.writerow(['ngay_gio_VN', 'tong_request', 'request_do_quet'] + [k for k, *_ in GROUPS])
        for d, (tot, cnt) in days.items():
            w.writerow([d, tot, sum(cnt.values())] + [cnt.get(k, 0) for k, *_ in GROUPS])

    # 3 · before / after the layer changed
    ph = lambda r: 'truoc' if r[0] < t_change else 'sau'
    with open(os.path.join(HERE, f'truoc-sau-{stamp}.csv'), 'w', newline='', encoding='utf-8') as fh:
        w = csv.writer(fh, lineterminator='\n')
        w.writerow(['nhom', 'giai_doan', 'ma_tra_loi', 'so_request', 'byte_trung_vi'])
        for key, *_ in GROUPS:
            for p in ('truoc', 'sau'):
                cnt = collections.defaultdict(list)
                for r in by_group.get(key, []):
                    if ph(r) == p:
                        cnt[r[3]].append(r[4])
                for st in sorted(cnt, key=lambda s: -len(cnt[s])):
                    w.writerow([key, p, st, len(cnt[st]), int(statistics.median(cnt[st]))])

    # 4 · most requested paths
    with open(os.path.join(HERE, f'duong-dan-hay-gap-{stamp}.csv'), 'w', newline='', encoding='utf-8') as fh:
        w = csv.writer(fh, lineterminator='\n')
        w.writerow(['nhom', 'duong_dan', 'so_request'])
        for key, *_ in GROUPS:
            top = collections.Counter(r[2][:80] for r in by_group.get(key, [])).most_common(8)
            for p, k in top:
                w.writerow([key, p, k])

    # 5 · scanner-type request list (no IP, no user-agent)
    with gzip.open(os.path.join(HERE, f'request-do-quet-{stamp}.csv.gz'), 'wt', encoding='utf-8', compresslevel=9) as fh:
        w = csv.writer(fh, lineterminator='\n')
        w.writerow(['ts_utc', 'method', 'path', 'status', 'bytes', 'nhom'])
        for r, g in scan:
            w.writerow([r[0].strftime('%Y-%m-%dT%H:%M:%SZ'), r[1], r[2][:200], r[3], r[4], g])

    # 6 · overview
    pre = [r for r, _ in scan if r[0] < t_change]
    xml = by_group['xmlrpc']
    xp = [r for r in xml if r[1] == 'POST']
    d = {
        'stamp': stamp, 'requests': n, 'scan_requests': len(scan), 'hours': round(hours, 2),
        't0_utc': t0.strftime('%Y-%m-%dT%H:%M:%SZ'), 't1_utc': t1.strftime('%Y-%m-%dT%H:%M:%SZ'),
        't0_vn': t0.astimezone(ICT).strftime('%d/%m/%Y %H:%M'), 't1_vn': t1.astimezone(ICT).strftime('%d/%m/%Y %H:%M'),
        't_change_utc': t_change.strftime('%Y-%m-%dT%H:%M:%SZ'), 't_change_vn': t_change.astimezone(ICT).strftime('%d/%m/%Y %H:%M'),
        'hours_before': round((t_change - t0).total_seconds() / 3600, 2), 'hours_after': round((t1 - t_change).total_seconds() / 3600, 2),
        'scan_before': len(pre), 'scan_after': len(scan) - len(pre),
        'requests_before': sum(1 for r in rows if r[0] < t_change), 'requests_after': sum(1 for r in rows if r[0] >= t_change),
        'xmlrpc_post': len(xp), 'xmlrpc_post_258': sum(1 for r in xp if r[0] < t_change and r[3] == '200' and r[4] == 258),
        'xmlrpc_post_200_before': sum(1 for r in xp if r[0] < t_change and r[3] == '200'),
        'xmlrpc_post_403_after': sum(1 for r in xp if r[0] >= t_change and r[3] == '403'),
        'xmlrpc_post_403_after_bytes': sorted({r[4] for r in xp if r[0] >= t_change and r[3] == '403'}),
        'login_post': sum(1 for r in by_group['login'] if r[1] == 'POST'),
        'distinct_scan_paths': len({r[2] for r, _ in scan}),
    }
    json.dump(d, open(os.path.join(HERE, f'tong-quan-{stamp}.json'), 'w', encoding='utf-8'), ensure_ascii=False, indent=1)
    print(json.dumps(d, ensure_ascii=False, indent=1))


if __name__ == '__main__':
    if len(sys.argv) != 3:
        sys.exit(__doc__)
    main(sys.argv[1], sys.argv[2])