|
| 1 | +from typing import Dict, Union |
| 2 | +from netunicorn.base import Task, Failure |
| 3 | +import subprocess |
| 4 | +import pprint |
| 5 | +from ping3 import ping |
| 6 | +import csv |
| 7 | +import json |
| 8 | + |
| 9 | +class AlexaWebsitesTask(Task): |
| 10 | + # Measure network metrics for a list of Alexa top websites. |
| 11 | + requirements = [ |
| 12 | + "sudo apt-get install -y curl dnsutils traceroute", |
| 13 | + "pip install ping3" |
| 14 | + ] |
| 15 | + |
| 16 | + def __init__(self, domain: str = None, filepath: str = "alexa_websites.csv", output_path: str = None, top_k: int = 100, *args, **kwargs): |
| 17 | + super().__init__(*args, **kwargs) |
| 18 | + self.domain = domain |
| 19 | + self.filepath = filepath |
| 20 | + self.output_path = output_path |
| 21 | + self.top_k = top_k |
| 22 | + |
| 23 | + def get_traceroute(self) -> Union[str, Failure]: |
| 24 | + try: |
| 25 | + result = subprocess.run(["traceroute", "-m", "10", self.domain], capture_output=True, text=True, check=True) |
| 26 | + return result.stdout |
| 27 | + except Exception as e: |
| 28 | + return Failure(f"Traceroute failed: {e}") |
| 29 | + |
| 30 | + def measure_ping(self) -> Union[Dict[str, float], Failure]: |
| 31 | + try: |
| 32 | + ping_value = ping(self.domain) |
| 33 | + if ping_value is None: |
| 34 | + return Failure("Ping returned None.") |
| 35 | + return {"value": ping_value * 1000, "unit": "ms"} |
| 36 | + except Exception as e: |
| 37 | + return Failure(f"Ping failed: {e}") |
| 38 | + |
| 39 | + def measure_dns_time(self) -> Union[Dict[str, float], Failure]: |
| 40 | + try: |
| 41 | + result = subprocess.run(["dig", self.domain], capture_output=True, text=True, check=True) |
| 42 | + for line in result.stdout.splitlines(): |
| 43 | + if "Query time" in line: |
| 44 | + return {"value": float(line.split(":")[1].strip().split(" ")[0]), "unit": "ms"} |
| 45 | + return Failure("Query time not found in DNS response.") |
| 46 | + except Exception as e: |
| 47 | + return Failure(f"DNS resolution failed: {e}") |
| 48 | + |
| 49 | + def measure_timing(self) -> Union[Dict[str, Dict[str,float]], Failure]: |
| 50 | + try: |
| 51 | + result = subprocess.run([ |
| 52 | + "curl", |
| 53 | + "-o", "/dev/null", |
| 54 | + "-s", |
| 55 | + "-w", |
| 56 | + ( |
| 57 | + "time_appconnect: %{time_appconnect}\n" |
| 58 | + "time_connect: %{time_connect}\n" |
| 59 | + "time_namelookup: %{time_namelookup}\n" |
| 60 | + "time_pretransfer: %{time_pretransfer}\n" |
| 61 | + "time_redirect: %{time_redirect}\n" |
| 62 | + "time_starttransfer: %{time_starttransfer}\n" |
| 63 | + "time_total: %{time_total}\n" |
| 64 | + ), |
| 65 | + "-H", "Cache-Control: no-cache", |
| 66 | + f"https://{self.domain}", |
| 67 | + ], capture_output=True, text=True, check=True) |
| 68 | + metrics = { |
| 69 | + key.strip(): {"value": float(value.strip()) * 1000, "unit": "ms"} |
| 70 | + for line in result.stdout.splitlines() |
| 71 | + for key, value in [line.split(": ", 1)] |
| 72 | + } |
| 73 | + return metrics |
| 74 | + except Exception as e: |
| 75 | + return Failure(f"Network Timing measurement failed: {e}") |
| 76 | + |
| 77 | + @staticmethod |
| 78 | + def load_websites(filepath: str, top_k: int) -> list: |
| 79 | + # Load top k websites from a CSV file |
| 80 | + websites = [] |
| 81 | + with open(filepath, 'r') as file: |
| 82 | + reader = csv.reader(file) |
| 83 | + for row in reader: |
| 84 | + if len(websites) < top_k: |
| 85 | + websites.append(row[1]) |
| 86 | + else: |
| 87 | + break |
| 88 | + return websites |
| 89 | + |
| 90 | + def run(self) -> Union[Dict[str, Dict], Failure]: |
| 91 | + if self.domain: |
| 92 | + # Run for a single domain |
| 93 | + return { |
| 94 | + "traceroute": self.get_traceroute(), |
| 95 | + "ping_time": self.measure_ping(), |
| 96 | + "dns_time": self.measure_dns_time(), |
| 97 | + "measure_timing": self.measure_timing(), |
| 98 | + } |
| 99 | + else: |
| 100 | + # Run for all websites in a file |
| 101 | + websites = self.load_websites(self.filepath, self.top_k) |
| 102 | + print(f"Loaded {len(websites)} websites.") |
| 103 | + |
| 104 | + results = {} |
| 105 | + for website in websites: |
| 106 | + print(f"Processing: {website}") |
| 107 | + try: |
| 108 | + self.domain = website |
| 109 | + results[website] = self.run() |
| 110 | + except Exception as e: |
| 111 | + results[website] = Failure(f"Failed to process {website}: {e}") |
| 112 | + |
| 113 | + # Save results to a JSON file if output_path is provided |
| 114 | + if self.output_path: |
| 115 | + print(f"Saving results to {self.output_path}") |
| 116 | + try: |
| 117 | + with open(self.output_path, "w") as f: |
| 118 | + json.dump(results, f, indent=4) |
| 119 | + except Exception as e: |
| 120 | + return Failure(f"Failed to write results to file: {e}") |
| 121 | + else: |
| 122 | + pprint.pp(results) |
| 123 | + |
| 124 | + return results |
0 commit comments