Skip to content

Commit e5920a0

Browse files
maybe-hello-worldmaybe-hello-world
committed
updates
1 parent ce09fb6 commit e5920a0

File tree

11 files changed

+110
-46
lines changed

11 files changed

+110
-46
lines changed

‎pipelines/measurements/__init__.py‎

Whitespace-only changes.

‎pipelines/measurements/ookla_speedtest.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
from netunicorn.base import Pipeline
2+
from netunicorn.library.tasks.capture.tcpdump import StartCapture, StopNamedCapture
3+
from netunicorn.library.tasks.measurements.ookla_speedtest import SpeedTest
4+
from netunicorn.library.tasks.upload.fileio import UploadToFileIO
5+
6+
7+
def simple_speedtest_pipeline() -> Pipeline:
8+
"""
9+
Run a speedtest while capturing the traffic with tcpdump and upload the capture file to file.io
10+
:return: pipeline to run
11+
"""
12+
pipeline = (
13+
Pipeline()
14+
.then(StartCapture(filepath="/tmp/capture.pcap", name="capture"))
15+
.then(SpeedTest())
16+
.then(StopNamedCapture(start_capture_task_name="capture"))
17+
.then(UploadToFileIO(filepath="/tmp/capture.pcap", expires="1d"))
18+
)
19+
return pipeline

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "netunicorn-library"
3-
version = "0.3.1"
3+
version = "0.3.2"
44
authors = [
55
{name = "Roman Beltiukov", email = "[email protected]"},
66
]

‎tasks/capture/tcpdump.py‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,17 @@ def __init__(self, filepath: str, arguments: Optional[List[str]] = None, *args,
1313
super().__init__(*args, **kwargs)
1414
self.filepath = filepath
1515
self.arguments = arguments
16-
self.args = args
17-
self.kwargs = kwargs
16+
17+
self.linux_implementation = StartCaptureLinuxImplementation(
18+
filepath=self.filepath,
19+
arguments=self.arguments,
20+
*args,
21+
**kwargs
22+
)
1823

1924
def dispatch(self, node: Node) -> Task:
2025
if node.architecture in {Architecture.LINUX_AMD64, Architecture.LINUX_ARM64}:
21-
return StartCaptureLinuxImplementation(filepath=self.filepath, arguments=self.arguments, *self.args,
22-
**self.kwargs)
26+
return self.linux_implementation
2327

2428
raise NotImplementedError(
2529
f'StartCapture is not implemented for {node.architecture}'
@@ -49,24 +53,25 @@ def run(self) -> Result:
4953
class StopNamedCapture(TaskDispatcher):
5054
def __init__(self, start_capture_task_name: str, *args, **kwargs):
5155
super().__init__(*args, **kwargs)
52-
self.args = args
53-
self.kwargs = kwargs
5456
self.start_capture_task_name = start_capture_task_name
57+
self.linux_implementation = StopNamedCaptureLinuxImplementation(
58+
capture_task_name=self.start_capture_task_name,
59+
*args,
60+
**kwargs,
61+
)
5562

5663
def dispatch(self, node: Node) -> Task:
5764
if node.architecture in {Architecture.LINUX_AMD64, Architecture.LINUX_ARM64}:
58-
return StopNamedCaptureLinuxImplementation(
59-
*self.args,
60-
capture_task_name=self.start_capture_task_name,
61-
**self.kwargs,
62-
)
65+
return self.linux_implementation
6366

6467
raise NotImplementedError(
6568
f'StopCapture is not implemented for {node.architecture}'
6669
)
6770

6871

6972
class StopNamedCaptureLinuxImplementation(Task):
73+
requirements = ["sudo apt-get update", "sudo apt-get install -y tcpdump"]
74+
7075
def __init__(self, capture_task_name: str, *args, **kwargs):
7176
super().__init__(*args, **kwargs)
7277
self.capture_task_name = capture_task_name
@@ -84,12 +89,11 @@ def run(self):
8489
class StopAllTCPDumps(TaskDispatcher):
8590
def __init__(self, *args, **kwargs):
8691
super().__init__(*args, **kwargs)
87-
self.args = args
88-
self.kwargs = kwargs
92+
self.linux_implementation = StopAllTCPDumpsLinuxImplementation(*args, **kwargs)
8993

9094
def dispatch(self, node: Node) -> Task:
9195
if node.architecture in {Architecture.LINUX_AMD64, Architecture.LINUX_ARM64}:
92-
return StopAllTCPDumpsLinuxImplementation(*self.args, **self.kwargs)
96+
return self.linux_implementation
9397

9498
raise NotImplementedError(
9599
f'StopAllTCPDumps is not implemented for {node.architecture}'

‎tasks/measurements/ookla_speedtest.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,13 @@
77

88

99
class SpeedTest(TaskDispatcher):
10+
def __init__(self, *args, **kwargs):
11+
super().__init__(*args, **kwargs)
12+
self.linux_instance = SpeedTestLinuxImplementation(name=self.name)
13+
1014
def dispatch(self, node: Node) -> Task:
1115
if node.architecture in {Architecture.LINUX_AMD64, Architecture.LINUX_ARM64}:
12-
return SpeedTestLinuxImplementation()
16+
return self.linux_instance
1317

1418
raise NotImplementedError(
1519
f'SpeedTest is not implemented for architecture: {node.architecture}'
@@ -20,7 +24,7 @@ class SpeedTestLinuxImplementation(Task):
2024
requirements = ["pip install speedtest-cli"]
2125

2226
def run(self):
23-
result = subprocess.run(["speedtest-cli", "--simple"], capture_output=True)
27+
result = subprocess.run(["speedtest-cli", "--simple", "--secure"], capture_output=True)
2428
if result.returncode != 0:
2529
return Failure(
2630
result.stdout.decode("utf-8").strip()

‎tasks/upload/fileio.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
"""
2+
Uploads files to file.io -- temporary file storage
3+
"""
4+
5+
import os
6+
import subprocess
7+
from typing import Literal, Optional, Set
8+
9+
from netunicorn.base.nodes import Architecture, Node
10+
from netunicorn.base.task import Task, TaskDispatcher
11+
12+
13+
class UploadToFileIO(TaskDispatcher):
14+
def __init__(self, filepath: str, expires: str = "14d", *args, **kwargs):
15+
super().__init__(*args, **kwargs)
16+
self.linux_implementation = UploadToFileIOCurlImplementation(
17+
filepath=filepath, expires=expires, name=self.name
18+
)
19+
self.linux_implementation.requirements = ["sudo apt-get install -y curl"]
20+
21+
def dispatch(self, node: Node) -> Task:
22+
if node.architecture in {Architecture.LINUX_AMD64, Architecture.LINUX_ARM64}:
23+
return self.linux_implementation
24+
25+
raise NotImplementedError(
26+
f"UploadToFileIO is not implemented for architecture: {node.architecture}"
27+
)
28+
29+
30+
class UploadToFileIOCurlImplementation(Task):
31+
def __init__(self, filepath: str, expires: str = "14d", *args, **kwargs):
32+
super().__init__(*args, **kwargs)
33+
self.filepath = filepath
34+
self.expires = expires
35+
36+
def run(self):
37+
command = ["curl", "-F", f"file=@{self.filepath}", f"https://file.io?expires={self.expires}"]
38+
return subprocess.run(command, check=True, capture_output=True).stdout.decode("utf-8")
39+
40+

‎tasks/upload/webdav.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,19 +27,19 @@ def __init__(
2727

2828
super().__init__(*args, **kwargs)
2929

30-
def dispatch(self, node: Node) -> Task:
31-
result = UploadToWebDavImplementation(
30+
self.linux_implementation = UploadToWebDavImplementation(
3231
self.filepaths,
3332
self.endpoint,
3433
self.username,
3534
self.password,
3635
self.authentication,
3736
name=self.name,
3837
)
38+
self.linux_implementation.requirements = ["sudo apt-get install -y curl"]
3939

40+
def dispatch(self, node: Node) -> Task:
4041
if node.architecture in {Architecture.LINUX_AMD64, Architecture.LINUX_ARM64}:
41-
result.requirements = ["sudo apt-get install -y curl"]
42-
return result
42+
return self.linux_implementation
4343

4444
raise NotImplementedError(
4545
f"UploadToWebDav is not implemented for {node.architecture}"

‎tasks/video_watchers/__init__.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
from youtube_watcher import WatchYouTubeVideo, WatchYouTubeVideoLinuxImplementation
2-
from vimeo_watcher import WatchVimeoVideo, WatchVimeoVideoLinuxImplementation
3-
from twitch_watcher import WatchTwitchStream, WatchTwitchStreamLinuxImplementation
1+
from .youtube_watcher import WatchYouTubeVideo, WatchYouTubeVideoLinuxImplementation
2+
from .vimeo_watcher import WatchVimeoVideo, WatchVimeoVideoLinuxImplementation
3+
from .twitch_watcher import WatchTwitchStream, WatchTwitchStreamLinuxImplementation

‎tasks/video_watchers/twitch_watcher.py‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
"""
2-
Selenium-based Vimeo watcher
2+
Selenium-based Twitch watcher
33
"""
44
import os
55
import random
@@ -28,6 +28,7 @@ def watch(
2828
options = Options()
2929
options.add_argument("--no-sandbox")
3030
options.add_argument("--autoplay-policy=no-user-gesture-required")
31+
options.add_argument("--disable-dev-shm-usage")
3132
if chrome_location:
3233
options.binary_location = chrome_location
3334
driver = webdriver.Chrome(service=Service(), options=options)
@@ -44,13 +45,14 @@ def watch(
4445

4546
class WatchTwitchStream(TaskDispatcher):
4647
def __init__(self, video_url: str, duration: Optional[int] = None, *args, **kwargs):
48+
super().__init__(*args, **kwargs)
4749
self.video_url = video_url
4850
self.duration = duration
49-
super().__init__(*args, **kwargs)
51+
self.linux_implementation = WatchTwitchStreamLinuxImplementation(self.video_url, self.duration, name=self.name)
5052

5153
def dispatch(self, node: Node) -> Task:
5254
if node.architecture in {Architecture.LINUX_AMD64, Architecture.LINUX_ARM64}:
53-
return WatchTwitchStreamLinuxImplementation(self.video_url, self.duration, name=self.name)
55+
return self.linux_implementation
5456

5557
raise NotImplementedError(
5658
f'WatchTwitchVideo is not implemented for architecture: {node.architecture}'
@@ -59,11 +61,8 @@ def dispatch(self, node: Node) -> Task:
5961

6062
class WatchTwitchStreamLinuxImplementation(Task):
6163
requirements = [
62-
"sudo apt update",
63-
"sudo apt install -y python3-pip wget xvfb",
64+
"apt install -y python3-pip wget xvfb procps chromium chromium-driver",
6465
"pip3 install selenium webdriver-manager",
65-
"sudo apt install -y chromium-browser",
66-
"from webdriver_manager.chrome import ChromeDriverManager; from webdriver_manager.core.utils import ChromeType; ChromeDriverManager(chrome_type=ChromeType.CHROMIUM,path='/usr/bin/').install()",
6766
]
6867

6968
def __init__(
@@ -73,7 +72,7 @@ def __init__(
7372
self.duration = duration
7473
self.chrome_location = chrome_location
7574
if not self.chrome_location:
76-
self.chrome_location = "/usr/bin/chromium-browser"
75+
self.chrome_location = "/usr/bin/chromium"
7776
super().__init__(*args, **kwargs)
7877

7978
def run(self):

‎tasks/video_watchers/vimeo_watcher.py‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ def watch(
2828
options = Options()
2929
options.add_argument("--no-sandbox")
3030
options.add_argument("--autoplay-policy=no-user-gesture-required")
31+
options.add_argument("--disable-dev-shm-usage")
3132
if chrome_location:
3233
options.binary_location = chrome_location
3334
driver = webdriver.Chrome(service=Service(), options=options)
@@ -74,13 +75,14 @@ def watch(
7475

7576
class WatchVimeoVideo(TaskDispatcher):
7677
def __init__(self, video_url: str, duration: Optional[int] = None, *args, **kwargs):
78+
super().__init__(*args, **kwargs)
7779
self.video_url = video_url
7880
self.duration = duration
79-
super().__init__(*args, **kwargs)
81+
self.linux_implementation = WatchVimeoVideoLinuxImplementation(self.video_url, self.duration, name=self.name)
8082

8183
def dispatch(self, node: Node) -> Task:
8284
if node.architecture in {Architecture.LINUX_AMD64, Architecture.LINUX_ARM64}:
83-
return WatchVimeoVideoLinuxImplementation(self.video_url, self.duration, name=self.name)
85+
return self.linux_implementation
8486

8587
raise NotImplementedError(
8688
f'WatchVimeoVideo is not implemented for architecture: {node.architecture}'
@@ -89,11 +91,8 @@ def dispatch(self, node: Node) -> Task:
8991

9092
class WatchVimeoVideoLinuxImplementation(Task):
9193
requirements = [
92-
"sudo apt update",
93-
"sudo apt install -y python3-pip wget xvfb",
94+
"apt install -y python3-pip wget xvfb procps chromium chromium-driver",
9495
"pip3 install selenium webdriver-manager",
95-
"sudo apt install -y chromium-browser",
96-
"python3 -c \"from webdriver_manager.chrome import ChromeDriverManager; from webdriver_manager.core.utils import ChromeType; ChromeDriverManager(chrome_type=ChromeType.CHROMIUM,path='/usr/bin/').install()\"",
9796
]
9897

9998
def __init__(
@@ -108,7 +107,7 @@ def __init__(
108107
self.duration = duration
109108
self.chrome_location = chrome_location
110109
if not self.chrome_location:
111-
self.chrome_location = "/usr/bin/chromium-browser"
110+
self.chrome_location = "/usr/bin/chromium"
112111
super().__init__(*args, **kwargs)
113112

114113
def run(self):

0 commit comments

Comments
 (0)