sciencefair2023/analyze.py

import sys
import requests
from math import radians, cos, sin, acos, asin, sqrt
from pathlib import Path
from os import chdir

def filter_logs(filename: str):
    with open(filename, 'r') as f:
        for line in f:
            # IP,YYMMDD hhmmss TIMEZONE,STATUS,METHOD,FILE
            ip, timethings, status, method, filepath = line.split(',', 4)
            file = '/'.join(filepath.split('/')[4:]).strip()
            if (
                status != '200'
                or method != 'GET'
                or file not in validnames
            ): continue
            date, time, timezone = timethings.split(' ')
            yield ip, date, time, timezone, status, method, file

def get_ip_latlon(ip: str) -> (int, int):
    # make a reqest to ip-api.com to associate an ip to a
    # latitude and longitude
    addr = f'http://ip-api.com/json/{ip}'
    response = requests.get(addr)
    if not response.ok:
        raise RuntimeError(f'request for ip failed with {response.status_code}')
    resulting_dict = eval(response.content)
    if resulting_dict['status'] != 'fail':
        return resulting_dict['lat'], resulting_dict['lon']

def value_sort(d: dict, reverse=True) -> dict:
    return sorted(d.items(), key=lambda kv_pair: kv_pair[1], reverse=reverse)

def latlon_distance(p1, p2) -> float:
    # black magic do not touch. use the haversine formula to find the distance
    lat1, lon1 = p1
    lat2, lon2 = p2
    lon1 = radians(lon1)
    lat1 = radians(lat1)
    lon2 = radians(lon2)
    lat2 = radians(lat2)

    dlon = lon2 - lon1
    dlat = lat2 - lat1
    a = sin(dlat / 2)**2 + cos(lat1) * cos(lat2) * sin(dlon / 2)**2

    c = 2 * asin(sqrt(a))
    earth_radius_km = 6371

    return c * earth_radius_km

def analyze_server(server: Path, serverip: str) -> None:
    if not server.is_dir():
        return

    filename = f'{server.name}/access.log'
    hitfiles = {}
    requesters = {}
    distances = {}
    times = {}
    self_latlon = get_ip_latlon(serverip)
    for ip, date, time_str, timezone, status, method, file in filter_logs(filename):
        if file in hitfiles:
            hitfiles[file] += 1
        else:
            hitfiles[file] = 1

        if ip in requesters:
            requesters[ip] += 1
        else:
            requesters[ip] = 1
            latlon = get_ip_latlon(ip)
            if latlon is not None:
                distances[ip] = latlon_distance(self_latlon, latlon)

        hour = time_str.split(':')[0]
        if hour in times:
            times[hour] += 1
        else:
            times[hour] = 1

    print(f'\n\n--- ANALYSIS FOR {server.name.upper()} ---\n')

    for dict_name in ['hitfiles', 'requesters', 'distances', 'times']:
        print(
            dict_name + ': {\n    '
            + ',\n    '.join(
                f'{k!r}: {v!r}'
                for k, v in value_sort(eval(dict_name))
            )
            + '\n}'
        )

    print(f'average: {sum(distances.values())/len(distances)}')

def get_server_ip(servername: str) -> str:
    # associate servers with ips
    if servername == 'nova':
        return '184.73.25.153'
    elif servername == 'singapore':
        return '18.139.108.77'
    elif servername == 'sydney':
        return '54.206.216.118'
    elif servername == 'dublin':
        return '54.194.92.137'
    elif servername == 'brazil':
        return '18.228.245.48'

    raise ValueError(f'{servername} is not a known server')

def main(args: list) -> int:
    if len(args) == 1:
        # no log dirs provided
        print('no logdir provided')
        return 1

    start_dir = Path('.').resolve()

    for logdir in args[1:]:
        chdir(logdir)
        serverdir = Path('.')
        for subdir in serverdir.iterdir():
            serverip = get_server_ip(subdir.name)
            analyze_server(subdir, serverip)
        chdir(start_dir)

    return 0

validnames = {
    'wp-login.php',
    '.env',
    'plugins/system/debug/debug.xml',
    'administrator/language/en-GB/en-GB.xml',
    'administrator/help/en-GB/toc.json',
    '.git/config',
    'vendor/phpunit/phpunit/src/Util/PHP/eval-stdin.php',
    'xmlrpc.php',
    'wp1/wp-includes/wlwmanifest.xml',
    'wp/wp-includes/wlwmanifest.xml',
    'wordpress/wp-includes/wlwmanifest.xml',
    'web/wp-includes/wlwmanifest.xml',
    'test/wp-includes/wlwmanifest.xml',
    'site/wp-includes/wlwmanifest.xml',
    'shop/wp-includes/wlwmanifest.xml',
    'cms/wp-includes/wlwmanifest.xml',
    'blog/wp-includes/wlwmanifest.xml',
    '2019/wp-includes/wlwmanifest.xml',
    'wp-load.php',
    'public/_ignition/health-check',
    '_ignition/health-check',
    'admin/.env',
    'protected/.env',
    'wp-includes/wp-class.php',
    'wp-commentin.php',
    'wp-signin.php'
}
if __name__ == '__main__':
    sys.exit(main(sys.argv))