-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcache_padded.zig
More file actions
233 lines (193 loc) · 8.46 KB
/
cache_padded.zig
File metadata and controls
233 lines (193 loc) · 8.46 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
const std = @import("std");
const builtin = @import("builtin");
///
/// CachePadded library for Zig
///
/// - Architecture-aware cache line at COMPTIME for alignment.
/// - Optional runtime detection (Linux sysfs) for padding.
/// - Designed for use in MPMC queues, Chase-Lev, S3-FIFO, TinyUFO, EBR, etc.
///
/// Exposed variants:
/// - Static(T, LINE?) : compile-time line size (default: arch-tuned)
/// - Auto(T) : runtime-detected line size (padding only; no align())
/// - Numa(T, LINE?) : double-line padding
/// - NumaAuto(T) : Numa with arch-tuned line (no param)
/// - Atomic(T, LINE?) : cache-line isolated integer atomic
/// - AtomicAuto(T) : Atomic with arch-tuned line (no param)
///
/// Cached runtime line size to avoid repeated sysfs reads on Linux.
/// Thread-local to avoid cross-thread data races during initialization.
threadlocal var g_cached_line_size: usize = 0;
pub const CachePadded = struct {
// ================================================================
// COMPTIME ARCH-BASED CACHE LINE (for align())
// ================================================================
/// Use std.atomic.cache_line as the single source of truth for
/// the architectural cache line size used for alignment.
fn archLine() usize {
return std.atomic.cache_line;
}
// ================================================================
// RUNTIME DETECTION (Linux sysfs) FOR Auto(T)
// ================================================================
fn detectLineSizeRuntime() usize {
// Fast path: use cached value if already initialized.
if (g_cached_line_size != 0) return g_cached_line_size;
// For production use, we avoid any filesystem access and rely
// solely on std.atomic.cache_line (via archLine()). This keeps
// initialization O(1) and prevents a sysfs read on the first
// Auto(T) usage per thread.
const line = archLine();
g_cached_line_size = line;
return line;
}
fn detectSysfs() ?usize {
var f = std.fs.openFileAbsolute(
"/sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size",
.{}
) catch return null;
defer f.close();
var buf: [32]u8 = undefined;
const n = f.read(&buf) catch return null;
if (n == 0) return null;
const trimmed = std.mem.trim(u8, buf[0..n], " \n\r\t");
const parsed = std.fmt.parseUnsigned(usize, trimmed, 10) catch return null;
if (parsed == 0) return null;
return parsed;
}
/// Public helper if you ever want the chosen line size at runtime.
pub fn lineSize() usize {
return detectLineSizeRuntime();
}
// ================================================================
// 1) STATIC PADDING (compile-time line, with optional override)
// ================================================================
/// Static padding using the architecture-tuned cache line size.
pub fn Static(comptime T: type) type {
return StaticWithLine(T, archLine());
}
/// Static padding with an explicit compile-time line size.
pub fn StaticWithLine(comptime T: type, comptime LINE: usize) type {
comptime {
if (LINE == 0 or (LINE & (LINE - 1)) != 0)
@compileError("cache line must be a non-zero power-of-two");
}
return struct {
value: T align(LINE),
pad: [padCalc(T, LINE)]u8 = undefined,
fn padCalc(comptime X: type, comptime L: usize) usize {
const sz = @sizeOf(X);
if (sz >= L) return 0;
return L - sz;
}
pub fn init(v: T) @This() {
return .{ .value = v };
}
};
}
// ================================================================
// 2) AUTO (runtime detection, padding only; no align() change)
// ================================================================
pub fn Auto(comptime T: type) type {
return struct {
value: T,
pad: []u8,
// NOTE:
// - This type increases the *size* of each instance to at least one cache line,
// but does NOT change its alignment.
// - When used in arrays/slices, elements may still share cache lines depending
// on the allocator's base address.
// - For strict per-element isolation in arrays, prefer Static/Numa-based types.
pub fn init(allocator: std.mem.Allocator, v: T) !@This() {
const line = detectLineSizeRuntime();
const sz = @sizeOf(T);
const need = if (sz >= line) 0 else line - sz;
// Avoid pointless allocation when no padding is needed.
const buf: []u8 = if (need == 0)
&[_]u8{}
else
try allocator.alloc(u8, need);
return .{ .value = v, .pad = buf };
}
pub fn deinit(self: *@This(), allocator: std.mem.Allocator) void {
// Only free when we actually allocated padding.
if (self.pad.len != 0) {
allocator.free(self.pad);
}
}
};
}
// ================================================================
// 3) NUMA MODE (double-line padding; compile-time line)
// ================================================================
/// NUMA-style double-line padding using the architecture-tuned line size.
pub fn Numa(comptime T: type) type {
return NumaWithLine(T, archLine());
}
/// NUMA-style double-line padding with explicit compile-time line size.
pub fn NumaWithLine(comptime T: type, comptime LINE: usize) type {
return struct {
value: T align(LINE),
pad: [calc(T, LINE)]u8 = undefined,
fn calc(comptime X: type, comptime L: usize) usize {
const total = 2 * L;
const sz = @sizeOf(X);
if (sz >= total) return 0;
return total - sz;
}
pub fn init(v: T) @This() {
return .{ .value = v };
}
};
}
/// Convenience: NUMA padding with arch-based line, no param.
pub fn NumaAuto(comptime T: type) type {
return Numa(T);
}
// ================================================================
// 4) ATOMIC MODE (cache-line isolated integer atomic; compile-time line)
// ================================================================
/// Cache-line isolated atomic integer using architecture-tuned line size.
pub fn Atomic(comptime T: type) type {
return AtomicWithLine(T, archLine());
}
/// Cache-line isolated atomic integer with explicit compile-time line size.
pub fn AtomicWithLine(comptime T: type, comptime LINE: usize) type {
comptime {
const info = @typeInfo(T);
// Restrict to integer types so fetchAdd/fetchSub are well-defined.
if (info != .int and info != .comptime_int) {
@compileError("CachePadded.Atomic(T) only supports integer types");
}
}
return struct {
atom: std.atomic.Value(T) align(LINE),
pad: [calc(T, LINE)]u8 = undefined,
fn calc(comptime X: type, comptime L: usize) usize {
const sz = @sizeOf(std.atomic.Value(X));
if (sz >= L) return 0;
return L - sz;
}
pub fn init(v: T) @This() {
const a = std.atomic.Value(T).init(v);
return .{ .atom = a };
}
pub fn load(self: *const @This(), comptime order: std.builtin.AtomicOrder) T {
return self.atom.load(order);
}
pub fn store(self: *@This(), v: T, comptime order: std.builtin.AtomicOrder) void {
self.atom.store(v, order);
}
pub fn fetchAdd(self: *@This(), v: T, comptime order: std.builtin.AtomicOrder) T {
return self.atom.fetchAdd(v, order);
}
pub fn fetchSub(self: *@This(), v: T, comptime order: std.builtin.AtomicOrder) T {
return self.atom.fetchSub(v, order);
}
};
}
/// Convenience: Atomic padding with arch-based line, no param.
pub fn AtomicAuto(comptime T: type) type {
return Atomic(T);
}
};