Skip to content

Commit 2c9620e

Browse files
WonshtrumWonshtrum
committed
Performance and stability improvements
- remove kawa delimiters (it overly fragments the writes and slows h2 tremendously) - check rustls buffers before the socket (to reduce syscalls) - ignore empty data frames with end stream on close stream (all the stream management should be revised honestly) - only count the active streams when checking if opening a new one would overflow the max concurrent allowed (again... stream management = bad) - log the precise reason of any goaway - properly reset metrics - display total time and backend response time in access logs (will soon changed when rebase on main) Signed-off-by: Eloi DEMOLIS <[email protected]>
1 parent 07040fb commit 2c9620e

File tree

8 files changed

+168
-107
lines changed

8 files changed

+168
-107
lines changed

‎command/src/logging/display.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ impl fmt::Display for LogDuration {
8484

8585
let ms = duration.as_millis();
8686
if ms < 10 {
87-
let us = duration.as_millis();
87+
let us = duration.as_micros();
8888
if us >= 10 {
8989
return write!(f, "{us}μs");
9090
}

‎lib/src/protocol/mux/converter.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ impl<'a, T: AsBuffer> BlockConverter<T> for H2BlockConverter<'a> {
150150
.unwrap();
151151
kawa.push_out(Store::from_slice(&header));
152152
kawa.push_out(data);
153-
kawa.push_delimiter();
153+
// kawa.push_delimiter();
154154
return can_continue;
155155
}
156156
Block::Flags(Flags {
@@ -189,7 +189,7 @@ impl<'a, T: AsBuffer> BlockConverter<T> for H2BlockConverter<'a> {
189189
kawa.push_out(Store::from_slice(&header));
190190
}
191191
if end_header || end_stream {
192-
kawa.push_delimiter()
192+
// kawa.push_delimiter()
193193
}
194194
}
195195
}

‎lib/src/protocol/mux/h1.rs‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
use std::time::Instant;
2+
13
use sozu_command::ready::Ready;
24

35
use crate::{
@@ -42,6 +44,9 @@ impl<Front: SocketHandler> ConnectionH1<Front> {
4244
println_!("======= MUX H1 READABLE {:?}", self.position);
4345
self.timeout_container.reset();
4446
let stream = &mut context.streams[self.stream];
47+
if stream.metrics.start.is_none() {
48+
stream.metrics.start = Some(Instant::now());
49+
}
4550
let parts = stream.split(&self.position);
4651
let kawa = parts.rbuffer;
4752
let (size, status) = self.socket.socket_read(kawa.storage.space());
@@ -144,19 +149,19 @@ impl<Front: SocketHandler> ConnectionH1<Front> {
144149
let kawa = &mut stream.back;
145150
match kawa.detached.status_line {
146151
kawa::StatusLine::Response { code: 101, .. } => {
147-
println!("============== HANDLE UPGRADE!");
152+
debug!("============== HANDLE UPGRADE!");
148153
return MuxResult::Upgrade;
149154
}
150155
kawa::StatusLine::Response { code: 100, .. } => {
151-
println!("============== HANDLE CONTINUE!");
156+
debug!("============== HANDLE CONTINUE!");
152157
// after a 100 continue, we expect the client to continue with its request
153158
self.timeout_container.reset();
154159
self.readiness.interest.insert(Ready::READABLE);
155160
kawa.clear();
156161
return MuxResult::Continue;
157162
}
158163
kawa::StatusLine::Response { code: 103, .. } => {
159-
println!("============== HANDLE EARLY HINT!");
164+
debug!("============== HANDLE EARLY HINT!");
160165
if let StreamState::Linked(token) = stream.state {
161166
// after a 103 early hints, we expect the backend to send its response
162167
endpoint
@@ -181,9 +186,7 @@ impl<Front: SocketHandler> ConnectionH1<Front> {
181186
let old_state = std::mem::replace(&mut stream.state, StreamState::Unlinked);
182187
if stream.context.keep_alive_frontend {
183188
self.timeout_container.reset();
184-
// println!("{old_state:?} {:?}", self.readiness);
185189
if let StreamState::Linked(token) = old_state {
186-
// println!("{:?}", endpoint.readiness(token));
187190
endpoint.end_stream(token, self.stream, context);
188191
}
189192
self.readiness.interest.insert(Ready::READABLE);
@@ -285,7 +288,7 @@ impl<Front: SocketHandler> ConnectionH1<Front> {
285288
}
286289
(false, false) => {
287290
// we do not have an answer, but the request is untouched so we can retry
288-
println!("H1 RECONNECT");
291+
debug!("H1 RECONNECT");
289292
stream.state = StreamState::Link;
290293
}
291294
(false, true) => unreachable!(),

‎lib/src/protocol/mux/h2.rs‎

Lines changed: 75 additions & 43 deletions
Large diffs are not rendered by default.

‎lib/src/protocol/mux/mod.rs‎

Lines changed: 28 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,9 @@ pub use crate::protocol::mux::{
5252
#[macro_export]
5353
macro_rules! println_ {
5454
($($t:expr),*) => {
55-
print!("{}:{} ", file!(), line!());
56-
println!($($t),*)
57-
// $(let _ = &$t;)*
55+
// print!("{}:{} ", file!(), line!());
56+
// println!($($t),*)
57+
$(let _ = &$t;)*
5858
};
5959
}
6060
fn debug_kawa(_kawa: &GenericHttpStream) {
@@ -737,7 +737,7 @@ impl Stream {
737737
tags,
738738
client_rtt: None, //socket_rtt(self.front_socket()),
739739
server_rtt: None, //self.backend_socket.as_ref().and_then(socket_rtt),
740-
service_time: Duration::from_secs(0), // self.metrics.service_time(),
740+
service_time: self.metrics.backend_connection_time().unwrap_or(Duration::from_secs(0)), // self.metrics.service_time(),
741741
response_time: self.metrics.response_time(),
742742
bytes_in: self.metrics.bin,
743743
bytes_out: self.metrics.bout,
@@ -770,6 +770,13 @@ impl<L: ListenerHandler + L7ListenerHandler> Context<L> {
770770
}
771771
}
772772

773+
pub fn active_len(&self) -> usize {
774+
self.streams
775+
.iter()
776+
.filter(|s| !matches!(s.state, StreamState::Recycle))
777+
.count()
778+
}
779+
773780
pub fn create_stream(&mut self, request_id: Ulid, window: u32) -> Option<GlobalStreamId> {
774781
let listener = self.listener.borrow();
775782
let http_context = HttpContext::new(
@@ -784,12 +791,15 @@ impl<L: ListenerHandler + L7ListenerHandler> Context<L> {
784791
println_!("Reuse stream: {stream_id}");
785792
stream.state = StreamState::Idle;
786793
stream.attempts = 0;
794+
stream.received_end_of_stream = false;
787795
stream.window = window as i32;
788796
stream.context = http_context;
789797
stream.back.clear();
790798
stream.back.storage.clear();
791799
stream.front.clear();
792800
stream.front.storage.clear();
801+
stream.metrics.reset();
802+
stream.metrics.start = Some(Instant::now());
793803
return Some(stream_id);
794804
}
795805
}
@@ -919,8 +929,8 @@ impl Router {
919929

920930
let token = if let Some(token) = reuse_token {
921931
println_!("reused backend: {:#?}", self.backends.get(&token).unwrap());
922-
stream.metrics.backend_start();
923-
stream.metrics.backend_connected();
932+
// stream.metrics.backend_start();
933+
// stream.metrics.backend_connected();
924934
token
925935
} else {
926936
let (mut socket, backend) = self.backend_from_request(
@@ -995,7 +1005,7 @@ impl Router {
9951005
Err(cluster_error) => {
9961006
// we are past kawa parsing if it succeeded this can't fail
9971007
// if the request was malformed it was caught by kawa and we sent a 400
998-
panic!("{cluster_error}");
1008+
unreachable!("{cluster_error}");
9991009
}
10001010
};
10011011

@@ -1004,7 +1014,7 @@ impl Router {
10041014
let route = match route_result {
10051015
Ok(route) => route,
10061016
Err(frontend_error) => {
1007-
println!("{}", frontend_error);
1017+
println_!("{}", frontend_error);
10081018
// self.set_answer(DefaultAnswerStatus::Answer404, None);
10091019
return Err(RetrieveClusterError::RetrieveFrontend(frontend_error));
10101020
}
@@ -1013,7 +1023,7 @@ impl Router {
10131023
let cluster_id = match route {
10141024
Route::Cluster(id) => id,
10151025
Route::Deny => {
1016-
println!("Route::Deny");
1026+
println_!("Route::Deny");
10171027
// self.set_answer(DefaultAnswerStatus::Answer401, None);
10181028
return Err(RetrieveClusterError::UnauthorizedRoute);
10191029
}
@@ -1038,7 +1048,7 @@ impl Router {
10381048
proxy,
10391049
)
10401050
.map_err(|backend_error| {
1041-
println!("{backend_error}");
1051+
println_!("{backend_error}");
10421052
// self.set_answer(DefaultAnswerStatus::Answer503, None);
10431053
BackendConnectionError::Backend(backend_error)
10441054
})?;
@@ -1135,7 +1145,6 @@ impl<Front: SocketHandler + std::fmt::Debug, L: ListenerHandler + L7ListenerHand
11351145
let mut dead_backends = Vec::new();
11361146
for (token, client) in self.router.backends.iter_mut() {
11371147
let readiness = client.readiness_mut();
1138-
// println!("{token:?} -> {readiness:?}");
11391148
let dead = readiness.filter_interest().is_hup()
11401149
|| readiness.filter_interest().is_error();
11411150
if dead {
@@ -1292,8 +1301,6 @@ impl<Front: SocketHandler + std::fmt::Debug, L: ListenerHandler + L7ListenerHand
12921301
}
12931302
if !proxy_borrow.remove_session(*token) {
12941303
error!("session {:?} was already removed!", token);
1295-
} else {
1296-
// println!("SUCCESS: session {token:?} was removed!");
12971304
}
12981305
}
12991306
println_!("FRONTEND: {:#?}", self.frontend);
@@ -1384,7 +1391,7 @@ impl<Front: SocketHandler + std::fmt::Debug, L: ListenerHandler + L7ListenerHand
13841391
}
13851392

13861393
fn update_readiness(&mut self, token: Token, events: Ready) {
1387-
println!("EVENTS: {events:?} on {token:?}");
1394+
println_!("EVENTS: {events:?} on {token:?}");
13881395
if token == self.frontend_token {
13891396
self.frontend.readiness_mut().event |= events;
13901397
} else if let Some(c) = self.router.backends.get_mut(&token) {
@@ -1451,7 +1458,7 @@ impl<Front: SocketHandler + std::fmt::Debug, L: ListenerHandler + L7ListenerHand
14511458
if token == back_token {
14521459
// This stream is linked to the backend that timedout
14531460
if stream.back.is_terminated() || stream.back.is_error() {
1454-
println!(
1461+
println_!(
14551462
"Stream terminated or in error, do nothing, just wait a bit more"
14561463
);
14571464
// Nothing to do, simply wait for the remaining bytes to be proxied
@@ -1460,11 +1467,13 @@ impl<Front: SocketHandler + std::fmt::Debug, L: ListenerHandler + L7ListenerHand
14601467
}
14611468
} else if !stream.back.consumed {
14621469
// The response has not started yet
1463-
println!("Stream still waiting for response, send 504");
1470+
println_!("Stream still waiting for response, send 504");
14641471
set_default_answer(stream, front_readiness, 504);
14651472
should_write = true;
14661473
} else {
1467-
println!("Stream waiting for end of response, forcefully terminate it");
1474+
println_!(
1475+
"Stream waiting for end of response, forcefully terminate it"
1476+
);
14681477
forcefully_terminate_answer(
14691478
stream,
14701479
front_readiness,
@@ -1527,6 +1536,7 @@ impl<Front: SocketHandler + std::fmt::Debug, L: ListenerHandler + L7ListenerHand
15271536
}
15281537

15291538
fn close<P: L7Proxy>(&mut self, proxy: Rc<RefCell<P>>, _metrics: &mut SessionMetrics) {
1539+
println!("MUX CLOSE");
15301540
println_!("FRONTEND: {:#?}", self.frontend);
15311541
println_!("BACKENDS: {:#?}", self.router.backends);
15321542

@@ -1544,8 +1554,6 @@ impl<Front: SocketHandler + std::fmt::Debug, L: ListenerHandler + L7ListenerHand
15441554
}
15451555
if !proxy_borrow.remove_session(*token) {
15461556
error!("session {:?} was already removed!", token);
1547-
} else {
1548-
// println!("SUCCESS: session {token:?} was removed!");
15491557
}
15501558

15511559
match client.position() {
@@ -1573,6 +1581,7 @@ impl<Front: SocketHandler + std::fmt::Debug, L: ListenerHandler + L7ListenerHand
15731581
Position::Server => unreachable!(),
15741582
}
15751583
}
1584+
return;
15761585
let s = match &mut self.frontend {
15771586
Connection::H1(c) => &mut c.socket,
15781587
Connection::H2(c) => &mut c.socket,

‎lib/src/protocol/mux/parser.rs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ pub fn frame_header(input: &[u8], max_frame_size: u32) -> IResult<&[u8], FrameHe
207207
FrameType::WindowUpdate => true,
208208
};
209209
if !valid_stream_id {
210-
println!("invalid stream_id: {stream_id}");
210+
error!("invalid stream_id: {}", stream_id);
211211
return Err(Err::Failure(ParserError::new_h2(i, H2Error::ProtocolError)));
212212
}
213213

@@ -223,7 +223,7 @@ pub fn frame_header(input: &[u8], max_frame_size: u32) -> IResult<&[u8], FrameHe
223223
}
224224

225225
fn convert_frame_type(t: u8) -> Option<FrameType> {
226-
info!("got frame type: {}", t);
226+
debug!("got frame type: {}", t);
227227
match t {
228228
0 => Some(FrameType::Data),
229229
1 => Some(FrameType::Headers),
@@ -353,7 +353,6 @@ pub fn data_frame<'a>(
353353
header: &FrameHeader,
354354
) -> IResult<&'a [u8], Frame, ParserError<'a>> {
355355
let (remaining, i) = take(header.payload_len)(input)?;
356-
println!("{i:?}");
357356

358357
let (i, pad_length) = if header.flags & 0x8 != 0 {
359358
let (i, pad_length) = be_u8(i)?;

‎lib/src/protocol/mux/pkawa.rs‎

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,21 @@ where
3333
return handle_trailer(kawa, input, end_stream, decoder);
3434
}
3535
kawa.push_block(Block::StatusLine);
36+
// kawa.detached.status_line = match kawa.kind {
37+
// Kind::Request => StatusLine::Request {
38+
// version: Version::V20,
39+
// method: Store::Static(b"GET"),
40+
// uri: Store::Static(b"/"),
41+
// authority: Store::Static(b"lolcatho.st:8443"),
42+
// path: Store::Static(b"/"),
43+
// },
44+
// Kind::Response => StatusLine::Response {
45+
// version: Version::V20,
46+
// code: 200,
47+
// status: Store::Static(b"200"),
48+
// reason: Store::Static(b"FromH2"),
49+
// },
50+
// };
3651
kawa.detached.status_line = match kawa.kind {
3752
Kind::Request => {
3853
let mut method = Store::Empty;
@@ -87,12 +102,15 @@ where
87102
invalid_headers = true;
88103
}
89104
} else if compare_no_case(&k, b"priority") {
90-
todo!("decode priority");
105+
// todo!("decode priority");
106+
warn!("DECODE PRIORITY: {}", unsafe {
107+
std::str::from_utf8_unchecked(v.as_ref())
108+
});
91109
prioriser.push_priority(
92110
stream_id,
93111
PriorityPart::Rfc9218 {
94-
urgency: todo!(),
95-
incremental: todo!(),
112+
urgency: 0,
113+
incremental: false,
96114
},
97115
);
98116
}
@@ -105,7 +123,7 @@ where
105123
}
106124
});
107125
if let Err(error) = decode_status {
108-
println!("INVALID FRAGMENT: {error:?}");
126+
error!("INVALID FRAGMENT: {:?}", error);
109127
return Err((H2Error::CompressionError, true));
110128
}
111129
if invalid_headers
@@ -114,7 +132,7 @@ where
114132
|| path.len() == 0
115133
|| scheme.len() == 0
116134
{
117-
println!("INVALID HEADERS");
135+
error!("INVALID HEADERS");
118136
return Err((H2Error::ProtocolError, false));
119137
}
120138
// uri is only used by H1 statusline, in most cases it only consists of the path
@@ -177,11 +195,11 @@ where
177195
}
178196
});
179197
if let Err(error) = decode_status {
180-
println!("INVALID FRAGMENT: {error:?}");
198+
error!("INVALID FRAGMENT: {:?}", error);
181199
return Err((H2Error::CompressionError, true));
182200
}
183201
if invalid_headers || status.len() == 0 {
184-
println!("INVALID HEADERS");
202+
error!("INVALID HEADERS");
185203
return Err((H2Error::ProtocolError, false));
186204
}
187205
StatusLine::Response {
@@ -195,7 +213,7 @@ where
195213

196214
// everything has been parsed
197215
kawa.storage.head = kawa.storage.end;
198-
println!(
216+
debug!(
199217
"index: {}/{}/{}",
200218
kawa.storage.start, kawa.storage.head, kawa.storage.end
201219
);

0 commit comments

Comments
 (0)