Implement package fetch retries

lib/std/http/Client.zig

@@ -1063,7 +1063,7 @@ pub const Request = struct {
         try w.writeAll("\r\n");
     }
 
-    pub const ReceiveHeadError = http.Reader.HeadError || ConnectError || error{
+    pub const ReceiveHeadError = http.Reader.HeadError || RequestError || error{
         /// Server sent headers that did not conform to the HTTP protocol.
         ///
         /// To find out more detailed diagnostics, `http.Reader.head_buffer` can be
@@ -1562,7 +1562,8 @@ pub fn connectProxied(
     };
 }
 
-pub const ConnectError = ConnectTcpError || RequestError;
+/// Deprecated. Please use `RequestError` instead.
+pub const ConnectError = RequestError;
 
 /// Connect to `host:port` using the specified protocol. This will reuse a
 /// connection if one is already open.
@@ -1576,7 +1577,7 @@ pub fn connect(
     host: []const u8,
     port: u16,
     protocol: Protocol,
-) ConnectError!*Connection {
+) ConnectTcpError!*Connection {
     const proxy = switch (protocol) {
         .plain => client.http_proxy,
         .tls => client.https_proxy,

src/Package/Fetch.zig

@@ -90,6 +90,12 @@ latest_commit: ?git.Oid,
 /// the root source file.
 module: ?*Package.Module,
 
+/// The number of times an HTTP request will retry if it fails
+retry_count: u16 = 3,
+
+/// The delay in milliseconds between HTTP request retries
+retry_delay_ms: u32 = 500,
+
 pub const LazyStatus = enum {
     /// Not lazy.
     eager,
@@ -994,9 +1000,18 @@ fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u
     if (ascii.eqlIgnoreCase(uri.scheme, "http") or
         ascii.eqlIgnoreCase(uri.scheme, "https"))
     {
+        var retries_left = f.retry_count;
         resource.* = .{ .http_request = .{
-            .request = http_client.request(.GET, uri, .{}) catch |err|
-                return f.fail(f.location_tok, try eb.printString("unable to connect to server: {t}", .{err})),
+            .request = while (true) {
+                break http_client.request(.GET, uri, .{}) catch |err| {
+                    if (retries_left > 0) {
+                        std.Thread.sleep(std.time.ns_per_ms * f.retry_delay_ms);
+                        retries_left -= 1;
+                        continue;
+                    }
+                    return f.fail(f.location_tok, try eb.printString("unable to connect to server: {t}", .{err}));
+                };
+            },
             .response = undefined,
             .transfer_buffer = reader_buffer,
             .decompress_buffer = &.{},
@@ -1005,39 +1020,62 @@ fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u
         const request = &resource.http_request.request;
         errdefer request.deinit();
 
-        request.sendBodiless() catch |err|
-            return f.fail(f.location_tok, try eb.printString("HTTP request failed: {t}", .{err}));
+        while (true) {
+            request.sendBodiless() catch |err| {
+                if (retries_left > 0) {
+                    std.Thread.sleep(std.time.ns_per_ms * f.retry_delay_ms);
+                    retries_left -= 1;
+                    continue;
+                }
+                return f.fail(f.location_tok, try eb.printString("HTTP request failed: {t}", .{err}));
+            };
 
-        var redirect_buffer: [1024]u8 = undefined;
-        const response = &resource.http_request.response;
-        response.* = request.receiveHead(&redirect_buffer) catch |err| switch (err) {
-            error.ReadFailed => {
-                return f.fail(f.location_tok, try eb.printString("HTTP response read failure: {t}", .{
-                    request.connection.?.getReadError().?,
-                }));
-            },
-            else => |e| return f.fail(f.location_tok, try eb.printString("invalid HTTP response: {t}", .{e})),
-        };
+            var redirect_buffer: [1024]u8 = undefined;
+            const response = &resource.http_request.response;
+            response.* = request.receiveHead(&redirect_buffer) catch |err| switch (err) {
+                error.ReadFailed => {
+                    return f.fail(f.location_tok, try eb.printString("HTTP response read failure: {t}", .{
+                        request.connection.?.getReadError().?,
+                    }));
+                },
+                else => |e| return f.fail(f.location_tok, try eb.printString("invalid HTTP response: {t}", .{e})),
+            };
 
-        if (response.head.status != .ok) return f.fail(f.location_tok, try eb.printString(
-            "bad HTTP response code: '{d} {s}'",
-            .{ response.head.status, response.head.status.phrase() orelse "" },
-        ));
+            if (response.head.status != .ok) {
+                if (retries_left > 0) {
+                    std.Thread.sleep(std.time.ns_per_ms * f.retry_delay_ms);
+                    retries_left -= 1;
+                    continue;
+                }
+                return f.fail(f.location_tok, try eb.printString(
+                    "bad HTTP response code: '{d} {s}'",
+                    .{ response.head.status, response.head.status.phrase() orelse "" },
+                ));
+            }
 
-        resource.http_request.decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
-        return;
+            resource.http_request.decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
+            return;
+        }
     }
 
     if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or
         ascii.eqlIgnoreCase(uri.scheme, "git+https"))
     {
+        var retries_left = f.retry_count;
         var transport_uri = uri;
         transport_uri.scheme = uri.scheme["git+".len..];
-        var session = git.Session.init(arena, http_client, transport_uri, reader_buffer) catch |err| {
-            return f.fail(
-                f.location_tok,
-                try eb.printString("unable to discover remote git server capabilities: {t}", .{err}),
-            );
+        var session = while (true) {
+            break git.Session.init(arena, http_client, transport_uri, reader_buffer) catch |err| {
+                if (retries_left > 0) {
+                    std.Thread.sleep(std.time.ns_per_ms * f.retry_delay_ms);
+                    retries_left -= 1;
+                    continue;
+                }
+                return f.fail(
+                    f.location_tok,
+                    try eb.printString("unable to discover remote git server capabilities: {t}", .{err}),
+                );
+            };
         };
 
         const want_oid = want_oid: {
@@ -1097,9 +1135,16 @@ fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u
             .want_oid = want_oid,
         } };
         const fetch_stream = &resource.git.fetch_stream;
-        session.fetch(fetch_stream, &.{&want_oid_buf}, reader_buffer) catch |err| {
-            return f.fail(f.location_tok, try eb.printString("unable to create fetch stream: {t}", .{err}));
-        };
+        while (true) {
+            break session.fetch(fetch_stream, &.{&want_oid_buf}, reader_buffer) catch |err| {
+                if (retries_left > 0) {
+                    std.Thread.sleep(std.time.ns_per_ms * f.retry_delay_ms);
+                    retries_left -= 1;
+                    continue;
+                }
+                return f.fail(f.location_tok, try eb.printString("unable to create fetch stream: {t}", .{err}));
+            };
+        }
         errdefer fetch_stream.deinit(fetch_stream);
 
         return;

src/Package/Fetch/git.zig

@@ -678,6 +678,7 @@ pub const Session = struct {
             .object_format = .sha1,
             .arena = arena,
         };
+        errdefer session.location.deinit();
         var capability_iterator: CapabilityIterator = undefined;
         try session.getCapabilities(&capability_iterator, response_buffer);
         defer capability_iterator.deinit();
@@ -703,6 +704,7 @@ pub const Session = struct {
     /// An owned `std.Uri` representing the location of the server (base URI).
     const Location = struct {
         uri: std.Uri,
+        arena: Allocator,
 
         fn init(arena: Allocator, uri: std.Uri) !Location {
             const scheme = try arena.dupe(u8, uri.scheme);
@@ -720,6 +722,7 @@ pub const Session = struct {
             });
             // The query and fragment are not used as part of the base server URI.
             return .{
+                .arena = arena,
                 .uri = .{
                     .scheme = scheme,
                     .user = if (user) |s| .{ .percent_encoded = s } else null,
@@ -730,6 +733,23 @@ pub const Session = struct {
                 },
             };
         }
+
+        fn deinit(location: *Location) void {
+            location.arena.free(location.uri.scheme);
+            if (location.uri.user) |user| {
+                location.arena.free(user.percent_encoded);
+            }
+
+            if (location.uri.password) |password| {
+                location.arena.free(password.percent_encoded);
+            }
+
+            if (location.uri.host) |host| {
+                location.arena.free(host.percent_encoded);
+            }
+
+            location.arena.free(location.uri.path.percent_encoded);
+        }
     };
 
     /// Returns an iterator over capabilities supported by the server.
@@ -993,6 +1013,7 @@ pub const Session = struct {
         }
         {
             const object_format_packet = try std.fmt.allocPrint(arena, "object-format={s}\n", .{@tagName(session.object_format)});
+            defer arena.free(object_format_packet);
             try Packet.write(.{ .data = object_format_packet }, &body);
         }
         try Packet.write(.delimiter, &body);
@@ -1033,6 +1054,7 @@ pub const Session = struct {
         if (response.head.status != .ok) return error.ProtocolError;
 
         const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
+        errdefer arena.free(decompress_buffer);
         const reader = response.readerDecompressing(response_buffer, &fs.decompress, decompress_buffer);
         // We are not interested in any of the sections of the returned fetch
         // data other than the packfile section, since we aren't doing anything