Skip to content

Commit 3e7565f

Browse files
nevansnevans
committed
📉 Add SequenceSet benchmarks
This adds: * a simple ruby script for measuring `ObjectSpace.memsize_of`, * several benchmark-driver scripts for: * `SequenceSet.new` (via `::[]`, which also calls `#freeze`) * `SequenceSet#slice` (aka `#[]`) * `SequenceSet#normalize` * Various set ops: `&`, `|`, `-`, `^`, `~` * Various set predicates: `#intersect?`, `#disjoint?`, `#cover?` * Several alternate implementations of: * AND — `#&` and `#intersect!` * NOT — `#~` and `#complement!` * XOR — `#^` and `#xor!`
1 parent 30e7aa3 commit 3e7565f

9 files changed

+701
-0
lines changed

‎benchmarks/seqset-memsize.rb‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# frozen_string_literal: true
2+
3+
$LOAD_PATH.unshift "./lib"
4+
require "net/imap"
5+
require "objspace"
6+
7+
def seqset(n, min: 1, max: (n * 1.25).to_i)
8+
inputs = Array.new(n) { rand(min..max) }
9+
Net::IMAP::SequenceSet[inputs]
10+
end
11+
12+
def obj_tree(obj, seen: Set.new)
13+
seen << obj
14+
children = ObjectSpace.reachable_objects_from(obj)
15+
.reject { _1 in Module or seen.include?(_1) }
16+
.flat_map { obj_tree(_1, seen:) }
17+
[obj, *children]
18+
end
19+
20+
def memsize(obj) = obj_tree(obj).sum { ObjectSpace.memsize_of _1 }
21+
22+
def avg(ary) = ary.sum / ary.count.to_f
23+
24+
def print_avg(n, count: 10, **)
25+
print "Average memsize of SequenceSet with %6d inputs: " % [n]
26+
sizes = Array.new(count) {
27+
print "."
28+
memsize seqset(n, **)
29+
}
30+
puts "%9.1f" % [avg(sizes)]
31+
end
32+
33+
# pp obj_tree(seqset(200, min: 1_000_000, max: 1_000_999)).to_h { [_1, memsize(_1)] }
34+
print_avg 1
35+
print_avg 10
36+
print_avg 100
37+
38+
print_avg 1_000
39+
print_avg 10_000
40+
print_avg 100_000

‎benchmarks/sequence_set-and.yml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
---
2+
prelude: |
3+
require "yaml"
4+
require "net/imap"
5+
6+
INPUT_COUNT = Integer ENV.fetch("BENCHMARK_INPUT_COUNT", 1000)
7+
MAX_INPUT = Integer ENV.fetch("BENCHMARK_MAX_INPUT", 1400)
8+
WARMUP_RUNS = Integer ENV.fetch("BENCHMARK_WARMUP_RUNS", 200)
9+
10+
SETS = Array.new(1000) {
11+
Net::IMAP::SequenceSet[Array.new(INPUT_COUNT) { rand(1..MAX_INPUT) }]
12+
}
13+
14+
def sets
15+
l, r = SETS.sample(2)
16+
[l.dup, r]
17+
end
18+
19+
class Net::IMAP
20+
class SequenceSet
21+
def and0(other) remain_frozen dup.and0! other end
22+
def and1(other) remain_frozen dup.and1! other end
23+
def and2(other) remain_frozen dup.and2! other end
24+
25+
# L - ~R
26+
def and0!(other)
27+
modifying!
28+
subtract SequenceSet.new(other).complement!
29+
end
30+
31+
# L - (L - R)
32+
def and1!(other)
33+
modifying!
34+
subtract dup.subtract(SequenceSet.new(other))
35+
end
36+
37+
# TODO: add this as a public method
38+
def xor!(other) # :nodoc:
39+
modifying!
40+
copy = dup
41+
other = SequenceSet.new(other)
42+
merge(other).subtract(other.subtract(copy.complement!))
43+
end
44+
45+
# L - (L ^ R)
46+
def and2!(other)
47+
modifying!
48+
subtract SequenceSet.new(other).xor! self
49+
end
50+
end
51+
end
52+
53+
# warmup (esp. for JIT)
54+
WARMUP_RUNS.times do
55+
lhs, rhs = sets
56+
lhs | rhs
57+
lhs & rhs
58+
lhs - rhs
59+
lhs ^ rhs
60+
~lhs
61+
lhs.and0 rhs
62+
lhs.and1 rhs
63+
lhs.and2 rhs
64+
end
65+
66+
benchmark:
67+
" L & R": l, r = sets; l & r
68+
" L - ~R": l, r = sets; l - ~r
69+
"and0 L - ~R": l, r = sets; l.and0 r
70+
"and0! L - ~R": l, r = sets; l.and0! r
71+
" L - (L - R)": l, r = sets; l - (l - r)
72+
"and1 L - (L - R)": l, r = sets; l.and1 r
73+
"and1! L - (L - R)": l, r = sets; l.and1! r
74+
" L - (L ^ R)": l, r = sets; l - (l ^ r)
75+
"and2 L - (L ^ R)": l, r = sets; l.and2 r
76+
"and2! L - (L ^ R)": l, r = sets; l.and2! r
77+
78+
contexts:
79+
- name: local
80+
prelude: |
81+
$LOAD_PATH.unshift "./lib"
82+
require: false
83+
- name: v0.5.9
84+
gems:
85+
net-imap: 0.5.9
86+
require: false
87+
- name: v0.5.0
88+
gems:
89+
net-imap: 0.5.0
90+
require: false
91+
- name: v0.4.21
92+
gems:
93+
net-imap: 0.4.21
94+
require: false

‎benchmarks/sequence_set-new.yml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
---
2+
prelude: |
3+
require "net/imap"
4+
SeqSet = Net::IMAP::SequenceSet
5+
6+
N_RAND = 100
7+
8+
def rand_nums(n, min: 1, max: (n * 1.25).to_i) = Array.new(n) { rand(1..max) }
9+
def rand_entries(...) = SeqSet[rand_nums(...)].elements.shuffle
10+
def rand_string(...) = SeqSet[rand_nums(...)].string.split(?,).shuffle.join(?,)
11+
12+
def build_string_inputs(n, n_rand, **)
13+
Array.new(n_rand) { rand_string(n, **) }
14+
end
15+
16+
def build_int_inputs(n, n_rand, **)
17+
Array.new(n_rand) { rand_entries(n, **) }
18+
end
19+
20+
inputs = nil
21+
i = 0
22+
23+
# warm up, especially for YJIT
24+
1000.times do
25+
ints = rand_nums(1000)
26+
seqset = SeqSet[ints]
27+
string = seqset.string.split(?,).shuffle.join(?,)
28+
SeqSet[string]
29+
end
30+
31+
benchmark:
32+
33+
- name: n=10 ints
34+
prelude: inputs = build_int_inputs 10, N_RAND
35+
script: SeqSet[inputs[i = (i+1) % N_RAND]]
36+
37+
- name: n=10 string
38+
prelude: inputs = build_string_inputs 10, N_RAND
39+
script: SeqSet[inputs[i = (i+1) % N_RAND]]
40+
41+
- name: n=100 ints
42+
prelude: inputs = build_int_inputs 100, N_RAND
43+
script: SeqSet[inputs[i = (i+1) % N_RAND]]
44+
45+
- name: n=100 string
46+
prelude: inputs = build_string_inputs 100, N_RAND
47+
script: SeqSet[inputs[i = (i+1) % N_RAND]]
48+
49+
- name: n=1000 ints
50+
prelude: inputs = build_int_inputs 1000, N_RAND
51+
script: SeqSet[inputs[i = (i+1) % N_RAND]]
52+
53+
- name: n=1000 string
54+
prelude: inputs = build_string_inputs 1000, N_RAND
55+
script: SeqSet[inputs[i = (i+1) % N_RAND]]
56+
57+
- name: n=10,000 ints
58+
prelude: inputs = build_int_inputs 10_000, N_RAND
59+
script: SeqSet[inputs[i = (i+1) % N_RAND]]
60+
61+
- name: n=10,000 string
62+
prelude: inputs = build_string_inputs 10_000, N_RAND
63+
script: SeqSet[inputs[i = (i+1) % N_RAND]]
64+
65+
- name: n=100,000 ints
66+
prelude: inputs = build_int_inputs 100_000, N_RAND / 2
67+
script: SeqSet[inputs[i = (i+1) % N_RAND]]
68+
69+
- name: n=100,000 string
70+
prelude: inputs = build_string_inputs 100_000, N_RAND / 2
71+
script: SeqSet[inputs[i = (i+1) % (N_RAND / 2)]]
72+
73+
# - name: n=1,000,000 ints
74+
# prelude: inputs = build_int_inputs 1_000_000
75+
# script: SeqSet[inputs[i = (i+1) % N_RAND]]
76+
77+
# - name: n=10,000,000 ints
78+
# prelude: inputs = build_int_inputs 10_000_000
79+
# script: SeqSet[inputs[i = (i+1) % N_RAND]]
80+
81+
contexts:
82+
- name: local
83+
prelude: |
84+
$LOAD_PATH.unshift "./lib"
85+
require: false
86+
- name: v0.5.9
87+
gems:
88+
net-imap: 0.5.9
89+
require: false
90+
- name: v0.5.0
91+
gems:
92+
net-imap: 0.5.0
93+
require: false
94+
- name: v0.4.21
95+
gems:
96+
net-imap: 0.4.21
97+
require: false

‎benchmarks/sequence_set-normalize.yml‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
---
2+
prelude: |
3+
require "yaml"
4+
require "net/imap"
5+
6+
INPUT_COUNT = Integer ENV.fetch("BENCHMARK_INPUT_COUNT", 1000)
7+
MAX_INPUT = Integer ENV.fetch("BENCHMARK_MAX_INPUT", 1400)
8+
WARMUP_RUNS = Integer ENV.fetch("BENCHMARK_WARMUP_RUNS", 200)
9+
10+
def init_sets(count: 100, set_size: INPUT_COUNT, max: MAX_INPUT)
11+
Array.new(count) {
12+
Net::IMAP::SequenceSet.new(Array.new(set_size) { rand(1..max) })
13+
}
14+
end
15+
16+
def init_normal_sets(...)
17+
init_sets(...)
18+
end
19+
20+
def init_frozen_normal_sets(...)
21+
init_sets(...)
22+
.map(&:freeze)
23+
end
24+
25+
def init_unsorted_sets(...)
26+
init_sets(...)
27+
.each do |seqset|
28+
entries = seqset.entries.shuffle
29+
seqset.clear
30+
entries.each do |entry|
31+
seqset.append entry
32+
end
33+
end
34+
end
35+
36+
def init_abnormal_sets(...)
37+
init_sets(...)
38+
.each do |seqset|
39+
entries = seqset.entries.shuffle
40+
seqset.clear
41+
entries.each do |entry|
42+
if [true, false].sample
43+
seqset.append entry
44+
elsif entry.is_a? Range
45+
seqset.append "#{entry.end || "*"}:#{entry.begin}"
46+
else
47+
seqset.append "#{entry}:#{entry}"
48+
end
49+
end
50+
end
51+
end
52+
53+
# warmup (esp. for JIT)
54+
WARMUP_RUNS.times do
55+
init_sets(count: 20, set_size: 100, max: 120).each do |set|
56+
set.normalize
57+
end
58+
end
59+
60+
benchmark:
61+
- name: "normal"
62+
prelude: $sets = init_normal_sets
63+
script: $sets.sample.normalize
64+
- name: "frozen and normal"
65+
prelude: $sets = init_frozen_normal_sets
66+
script: $sets.sample.normalize
67+
- name: "unsorted"
68+
prelude: $sets = init_unsorted_sets
69+
script: $sets.sample.normalize
70+
- name: "abnormal"
71+
prelude: $sets = init_abnormal_sets
72+
script: $sets.sample.normalize
73+
74+
contexts:
75+
# n.b: can't use anything newer as the baseline: it's over 500x faster!
76+
- name: v0.5.9
77+
gems:
78+
net-imap: 0.5.9
79+
require: false
80+
- name: local
81+
prelude: |
82+
$LOAD_PATH.unshift "./lib"
83+
require: false
84+
- name: v0.5.0
85+
gems:
86+
net-imap: 0.5.0
87+
require: false
88+
- name: v0.4.21
89+
gems:
90+
net-imap: 0.4.21
91+
require: false

0 commit comments

Comments
 (0)