Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.9.10
rev: v0.16.3
hooks:
- id: ruff
args: [--fix]
Expand Down
23 changes: 15 additions & 8 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,10 @@ Be sure to review the breaking changes before upgrading.

bench = MicroBench()


@bench
def my_function():
...
def my_function(): ...


for _ in range(10):
my_function()
Expand Down Expand Up @@ -150,6 +151,7 @@ Be sure to review the breaking changes before upgrading.
async def fetch():
await asyncio.sleep(0.01)


asyncio.run(fetch())

async with bench.arecord('load'):
Expand Down Expand Up @@ -288,10 +290,12 @@ Be sure to review the breaking changes before upgrading.
```python
from microbench import MicroBench, FileOutput, RedisOutput

bench = MicroBench(outputs=[
FileOutput('/home/user/results.jsonl'),
RedisOutput('microbench:mykey', host='redis-host', port=6379),
])
bench = MicroBench(
outputs=[
FileOutput('/home/user/results.jsonl'),
RedisOutput('microbench:mykey', host='redis-host', port=6379),
]
)
```

`get_results()` delegates to the first sink that supports reading back
Expand Down Expand Up @@ -395,19 +399,22 @@ Be sure to review the breaking changes before upgrading.
```python
from microbench import MicroBenchRedis


class RedisBench(MicroBenchRedis):
redis_connection = {'host': 'localhost', 'port': 6379}
redis_key = 'microbench:mykey'


bench = RedisBench()
```

After:
```python
from microbench import MicroBench, RedisOutput

bench = MicroBench(outputs=[RedisOutput('microbench:mykey',
host='localhost', port=6379)])
bench = MicroBench(
outputs=[RedisOutput('microbench:mykey', host='localhost', port=6379)]
)
```

- **`LiveStream` updated for v2 record schema**: field references updated from
Expand Down
20 changes: 12 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -146,15 +146,17 @@ from microbench import MicroBench

bench = MicroBench(outfile='/home/user/results.jsonl', experiment='baseline')


@bench
def my_function(n):
return sum(range(n))


my_function(1_000_000)

results = bench.get_results() # list of dicts — no extra dependencies
results = bench.get_results() # list of dicts — no extra dependencies
results = bench.get_results(format='df') # pandas DataFrame
bench.summary() # quick stats printout
bench.summary() # quick stats printout
```

Each call produces one record. With `get_results(flat=True)` the record looks
Expand Down Expand Up @@ -183,17 +185,19 @@ like:
from microbench import MicroBench, MBFunctionCall, MBHostInfo, MBSlurmInfo
import numpy, pandas, time


class MyBench(MicroBench, MBFunctionCall, MBHostInfo, MBSlurmInfo):
outfile = '/home/user/my-benchmarks.jsonl'
capture_versions = (numpy, pandas) # record live module versions
env_vars = ('CUDA_VISIBLE_DEVICES',) # capture env vars as env.<NAME>
capture_versions = (numpy, pandas) # record live module versions
env_vars = ('CUDA_VISIBLE_DEVICES',) # capture env vars as env.<NAME>


benchmark = MyBench(experiment='run-1', iterations=3, duration_counter=time.monotonic)

benchmark = MyBench(experiment='run-1', iterations=3,
duration_counter=time.monotonic)

@benchmark
def myfunction(arg1, arg2):
...
def myfunction(arg1, arg2): ...


myfunction(x, y)
```
Expand Down
17 changes: 12 additions & 5 deletions docs/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,7 @@ Read the results:

```python
from microbench import FileOutput

df = FileOutput('/scratch/user/results.jsonl').get_results(flat=True, format='df')
df['total_duration'] = df['call.durations'].apply(sum)
df.groupby('slurm.job_id')['total_duration'].describe()
Expand Down Expand Up @@ -444,6 +445,7 @@ Read results back from Redis with Python:

```python
import redis, json

client = redis.StrictRedis(host='redis.example.com')
records = [json.loads(r) for r in client.lrange('bench:results', 0, -1)]
```
Expand All @@ -452,6 +454,7 @@ Or via microbench's `RedisOutput.get_results()`:

```python
from microbench import RedisOutput

results = RedisOutput('bench:results', host='redis.example.com').get_results()
```

Expand Down Expand Up @@ -548,15 +551,19 @@ Analyse with `get_results()`:

```python
from microbench import FileOutput

results = FileOutput('results.jsonl').get_results()

# Flatten all samples for the first iteration across all records
import pandas
samples = pandas.DataFrame([
s
for r in results
for s in r['call']['monitor'][0] # [0] = first iteration
])

samples = pandas.DataFrame(
[
s
for r in results
for s in r['call']['monitor'][0] # [0] = first iteration
]
)
samples['rss_mb'] = samples['rss_bytes'] / 1024 / 1024
print(samples[['timestamp', 'cpu_percent', 'rss_mb']])
```
29 changes: 19 additions & 10 deletions docs/getting-started.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,11 @@ from microbench import MicroBench

bench = MicroBench()


@bench
def my_function(x):
return x ** 2
return x**2


my_function(42)
```
Expand All @@ -57,7 +59,7 @@ By default results are captured into an in-memory buffer. Read them back as
a list of dicts:

```python
results = bench.get_results() # list of dicts — no extra dependencies
results = bench.get_results() # list of dicts — no extra dependencies
results = bench.get_results(format='df') # pandas DataFrame
```

Expand Down Expand Up @@ -87,21 +89,22 @@ Every record contains these fields automatically (all nested under `mb` or `call
Here's an extended example to give you an idea of real-world usage.

```python
from microbench import MicroBench, MBFunctionCall, \
MBHostInfo, MBSlurmInfo
from microbench import MicroBench, MBFunctionCall, MBHostInfo, MBSlurmInfo
import numpy, pandas, time


class MyBench(MicroBench, MBFunctionCall, MBHostInfo, MBSlurmInfo):
outfile = '/home/user/my-benchmarks.jsonl'
capture_versions = (numpy, pandas)
env_vars = ('CUDA_VISIBLE_DEVICES',)

benchmark = MyBench(experiment='run-1', iterations=3,
duration_counter=time.monotonic)

benchmark = MyBench(experiment='run-1', iterations=3, duration_counter=time.monotonic)


@benchmark
def myfunction(arg1, arg2):
...
def myfunction(arg1, arg2): ...


myfunction(x, y)
```
Expand Down Expand Up @@ -153,13 +156,14 @@ object per line). Read them back with pandas:

```python
import pandas

results = pandas.read_json('/home/user/results.jsonl', lines=True)
```

Or via `get_results()`, which works regardless of the output destination:

```python
results = bench.get_results() # list of dicts — no extra dependencies
results = bench.get_results() # list of dicts — no extra dependencies
results = bench.get_results(format='df') # pandas DataFrame
```

Expand All @@ -173,6 +177,7 @@ bench.summary()

# or pass any list of result dicts:
from microbench import summary

summary(bench.get_results())
```

Expand All @@ -196,7 +201,7 @@ Use `flat=True` to flatten nested fields (e.g. `slurm`, `git`,
into pandas or a spreadsheet:

```python
results = bench.get_results(flat=True) # list of flat dicts
results = bench.get_results(flat=True) # list of flat dicts
results = bench.get_results(format='df', flat=True) # flat DataFrame
# 'call' dict becomes: call.name, call.durations, call.start_time, ...
# 'slurm' dict becomes: slurm.job_id, slurm.cpus_on_node, ...
Expand All @@ -213,9 +218,11 @@ section of a script:
```python
from microbench import MicroBench, MBHostInfo


class MyBench(MicroBench, MBHostInfo):
outfile = '/home/user/results.jsonl'


bench = MyBench(experiment='run-1')

with bench.record('data_loading'):
Expand Down Expand Up @@ -256,10 +263,12 @@ process exits — no restructuring of the script is required:
```python
from microbench import MicroBench, MBHostInfo, MBSlurmInfo


class MyBench(MicroBench, MBHostInfo, MBSlurmInfo):
outfile = '/scratch/results.jsonl'
capture_optional = True # recommended: don't let a failed capture abort exit


bench = MyBench(experiment='baseline')
bench.record_on_exit('simulation')

Expand Down
2 changes: 2 additions & 0 deletions docs/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,12 @@ from microbench import MicroBench

bench = MicroBench(outfile='results.jsonl')


@bench
def my_function(n):
return sum(range(n))


my_function(1_000_000)
```

Expand Down
21 changes: 18 additions & 3 deletions docs/user-guide/advanced.md
Original file line number Diff line number Diff line change
Expand Up @@ -120,11 +120,13 @@ contain `N` entries per named phase:
```python
bench = MicroBench(iterations=3)


@bench
def pipeline():
with bench.time('step'):
...


pipeline()
# call.timings → [{"name": "step", ...}, {"name": "step", ...}, {"name": "step", ...}]
```
Expand Down Expand Up @@ -159,6 +161,7 @@ analysis:

```python
import pandas

results = pandas.read_json('/home/user/results.jsonl', lines=True)

# Records where the call raised
Expand All @@ -178,6 +181,7 @@ a class attribute to catch failures instead and record them in
```python
from microbench import MicroBench, MBNvidiaSmi, MBCondaPackages


class MyBench(MicroBench, MBNvidiaSmi, MBCondaPackages):
capture_optional = True # missing nvidia-smi or conda won't abort the run
```
Expand Down Expand Up @@ -217,21 +221,26 @@ To handle custom types, subclass `JSONEncoder`:
import microbench as mb
from igraph import Graph


class MyEncoder(mb.JSONEncoder):
def default(self, o):
if isinstance(o, Graph):
return str(o)
return super().default(o)


class MyBench(mb.MicroBench, mb.MBReturnValue):
pass


bench = MyBench(json_encoder=MyEncoder)


@bench
def make_graph():
return Graph(2, ((0, 1), (0, 2)))


make_graph() # no warning
```

Expand All @@ -250,11 +259,13 @@ running in the same Python process:
```python
from microbench.livestream import LiveStream


class MyStream(LiveStream):
def process_alert(self, data):
if sum(data.get('call', {}).get('durations', [])) > 10.0:
host = data.get('host', {}).get('hostname', 'unknown')
print(f"Slow call on {host}: {data['call']['durations']}")
print(f'Slow call on {host}: {data["call"]["durations"]}')


stream = MyStream('/home/user/results.jsonl')
# ... runs in background while your job continues ...
Expand All @@ -270,6 +281,7 @@ your benchmark job writes to the file:
from microbench.livestream import LiveStream
import time


class Watcher(LiveStream):
def filter(self, data):
# Only show records from GPU nodes
Expand All @@ -279,7 +291,8 @@ class Watcher(LiveStream):
name = data.get('call', {}).get('name', '?')
host = data.get('host', {}).get('hostname', '?')
durs = data.get('call', {}).get('durations', [])
print(f"{name} | {host} | {durs}")
print(f'{name} | {host} | {durs}')


stream = Watcher('/home/user/results.jsonl')
try:
Expand Down Expand Up @@ -314,7 +327,9 @@ import pandas
results = pandas.read_json('/home/user/results.jsonl', lines=True)

# Compare the environment of the slowest and fastest calls
results_flat = pandas.DataFrame(FileOutput('/home/user/results.jsonl').get_results(flat=True))
results_flat = pandas.DataFrame(
FileOutput('/home/user/results.jsonl').get_results(flat=True)
)
slowest = results_flat.loc[results_flat['call.durations'].apply(sum).idxmax()]
fastest = results_flat.loc[results_flat['call.durations'].apply(sum).idxmin()]

Expand Down
Loading