Skip to content

Commit 4d98284

Browse files
mgomesmgomes
committed
Add the ability to auto-resume downloads and use sparse files
1 parent b46474b commit 4d98284

File tree

8 files changed

+1325
-87
lines changed

8 files changed

+1325
-87
lines changed

‎.dlrc.example‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Sample .dlrc configuration file
2+
# Copy this file to ~/.dlrc and customize as needed
3+
4+
# Number of concurrent connections (default: 8)
5+
boost = 8
6+
7+
# Maximum retries for failed parts (default: 3)
8+
retries = 3

‎Makefile‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
# Makefile for dl - Fast download manager
2+
3+
# Variables
4+
BINARY_NAME=dl
5+
GO=go
6+
GO_BUILD=$(GO) build
7+
GO_TEST=$(GO) test
8+
GO_CLEAN=$(GO) clean
9+
GO_LINT=golangci-lint
10+
GO_MOD=$(GO) mod
11+
GO_FILES=$(wildcard *.go)
12+
VERSION?=$(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
13+
LDFLAGS=-ldflags "-X main.version=$(VERSION)"
14+
15+
# Build variables
16+
GOOS?=$(shell go env GOOS)
17+
GOARCH?=$(shell go env GOARCH)
18+
19+
# Default target
20+
.PHONY: all
21+
all: help
22+
23+
# Build the application
24+
.PHONY: build
25+
build:
26+
@echo "Building $(BINARY_NAME)..."
27+
@$(GO_BUILD) $(LDFLAGS) -o $(BINARY_NAME) .
28+
29+
# Build for all platforms
30+
.PHONY: build-all
31+
build-all: build-linux build-darwin build-windows
32+
33+
# Build for Linux
34+
.PHONY: build-linux
35+
build-linux:
36+
@echo "Building for Linux..."
37+
@GOOS=linux GOARCH=amd64 $(GO_BUILD) $(LDFLAGS) -o $(BINARY_NAME)-linux-amd64 .
38+
@GOOS=linux GOARCH=arm64 $(GO_BUILD) $(LDFLAGS) -o $(BINARY_NAME)-linux-arm64 .
39+
40+
# Build for macOS
41+
.PHONY: build-darwin
42+
build-darwin:
43+
@echo "Building for macOS..."
44+
@GOOS=darwin GOARCH=amd64 $(GO_BUILD) $(LDFLAGS) -o $(BINARY_NAME)-darwin-amd64 .
45+
@GOOS=darwin GOARCH=arm64 $(GO_BUILD) $(LDFLAGS) -o $(BINARY_NAME)-darwin-arm64 .
46+
47+
# Build for Windows
48+
.PHONY: build-windows
49+
build-windows:
50+
@echo "Building for Windows..."
51+
@GOOS=windows GOARCH=amd64 $(GO_BUILD) $(LDFLAGS) -o $(BINARY_NAME)-windows-amd64.exe .
52+
53+
# Run tests
54+
.PHONY: test
55+
test:
56+
@echo "Running tests..."
57+
@$(GO_TEST) -v ./...
58+
59+
# Run tests with coverage
60+
.PHONY: test-coverage
61+
test-coverage:
62+
@echo "Running tests with coverage..."
63+
@$(GO_TEST) -v -coverprofile=coverage.out ./...
64+
@$(GO) tool cover -html=coverage.out -o coverage.html
65+
@echo "Coverage report generated: coverage.html"
66+
67+
# Run linter
68+
.PHONY: lint
69+
lint:
70+
@if command -v $(GO_LINT) >/dev/null 2>&1; then \
71+
echo "Running linter..."; \
72+
$(GO_LINT) run; \
73+
else \
74+
echo "golangci-lint not installed. Install with:"; \
75+
echo " curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin"; \
76+
fi
77+
78+
# Format code
79+
.PHONY: fmt
80+
fmt:
81+
@echo "Formatting code..."
82+
@$(GO) fmt ./...
83+
84+
# Update dependencies
85+
.PHONY: deps
86+
deps:
87+
@echo "Updating dependencies..."
88+
@$(GO_MOD) download
89+
@$(GO_MOD) tidy
90+
91+
# Clean build artifacts
92+
.PHONY: clean
93+
clean:
94+
@echo "Cleaning..."
95+
@$(GO_CLEAN)
96+
@rm -f $(BINARY_NAME) $(BINARY_NAME)-*
97+
@rm -f coverage.out coverage.html
98+
@rm -f *.dl_progress
99+
100+
# Install locally
101+
.PHONY: install
102+
install: build
103+
@echo "Installing $(BINARY_NAME)..."
104+
@cp $(BINARY_NAME) $(GOPATH)/bin/$(BINARY_NAME) || cp $(BINARY_NAME) ~/go/bin/$(BINARY_NAME)
105+
106+
# Uninstall
107+
.PHONY: uninstall
108+
uninstall:
109+
@echo "Uninstalling $(BINARY_NAME)..."
110+
@rm -f $(GOPATH)/bin/$(BINARY_NAME) ~/go/bin/$(BINARY_NAME)
111+
112+
# Run a test download
113+
.PHONY: test-download
114+
test-download: build
115+
@echo "Testing download with 10MB file..."
116+
@./$(BINARY_NAME) -boost 8 http://speedtest.tele2.net/10MB.zip
117+
118+
# Run a test download with resume
119+
.PHONY: test-resume
120+
test-resume: build
121+
@echo "Testing resume functionality..."
122+
@if [ -f test_resume.sh ]; then \
123+
./test_resume.sh; \
124+
else \
125+
echo "test_resume.sh not found"; \
126+
fi
127+
128+
# Development run with race detector
129+
.PHONY: dev
130+
dev:
131+
@echo "Building with race detector..."
132+
@$(GO_BUILD) -race $(LDFLAGS) -o $(BINARY_NAME) .
133+
@echo "Ready for development testing"
134+
135+
# Show help
136+
.PHONY: help
137+
help:
138+
@echo "dl - Fast download manager"
139+
@echo ""
140+
@echo "Usage:"
141+
@echo " make build - Build the application"
142+
@echo " make build-all - Build for all platforms"
143+
@echo " make test - Run tests"
144+
@echo " make test-coverage - Run tests with coverage report"
145+
@echo " make lint - Run linter"
146+
@echo " make fmt - Format code"
147+
@echo " make deps - Update dependencies"
148+
@echo " make clean - Clean build artifacts"
149+
@echo " make install - Install locally"
150+
@echo " make uninstall - Uninstall from local system"
151+
@echo " make test-download - Test download functionality"
152+
@echo " make test-resume - Test resume functionality"
153+
@echo " make dev - Build with race detector for development"
154+
@echo " make help - Show this help message"

‎README.md‎

Lines changed: 145 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,19 @@
11
# dl
22

3-
Downloads files utilizing concurrent connections. For some ISPs, this can really increase download speeds.
3+
A fast, reliable download manager that utilizes concurrent connections to maximize download speeds.
4+
5+
## Features
6+
7+
- **Concurrent downloading** - Multiple connections for faster downloads
8+
- **Auto-resume by default** - Automatically continues interrupted downloads
9+
- **Progress persistence** - Tracks exact download state across restarts
10+
- **Sparse file support** - Efficient disk usage on macOS/Linux
11+
- **Bandwidth limiting** - Control download speed
12+
- **Checksum verification** - Verify file integrity (MD5/SHA256)
13+
- **Smart retry logic** - Exponential backoff for failed parts
14+
- **Configuration file** - Save your preferred settings
15+
- **Graceful cancellation** - Clean shutdown with Ctrl+C
16+
- **Progress tracking** - Real-time progress with ETA
417

518
## Install
619

@@ -10,35 +23,153 @@ Download the latest compiled release of `dl` from the [releases page](https://gi
1023

1124
### Compiling from source
1225

13-
After you've installed Go:
26+
Requirements: Go 1.23+
1427

15-
1. Clone this repo
16-
2. Run `go build` in the root directory where you cloned the repo
28+
```bash
29+
git clone https://github.com/mgomes/dl.git
30+
cd dl
31+
go build
32+
```
1733

1834
## Usage
1935

20-
```
36+
### Basic Usage
37+
38+
```bash
2139
dl <file url> [file2 url] [file3 url] ...
2240
```
2341

24-
### Custom Filename
42+
### Advanced Options
43+
44+
```bash
45+
dl [options] <file url>
46+
47+
Options:
48+
-filename string Custom filename for the download
49+
-boost int Number of concurrent connections (default: 8)
50+
-retries int Max retries for failed parts (default: 3)
51+
-resume Resume interrupted download (default: true)
52+
-no-resume Disable auto-resume functionality
53+
-limit string Bandwidth limit (e.g. 1M, 500K, 100KB/s)
54+
-checksum string Verify with checksum (format: algorithm:hash)
55+
```
56+
57+
### Examples
2558

26-
By default, `dl` will use the file's HTTP metadata when available for the filename. If not available it will fallback to using the filename from the URI path.
59+
```bash
60+
# Basic download
61+
dl https://example.com/file.zip
2762

28-
To override this and set your own filename you can:
63+
# Download with custom filename
64+
dl -filename myfile.zip https://example.com/file.zip
2965

66+
# Downloads auto-resume by default (just re-run the same command)
67+
dl https://example.com/file.zip
68+
69+
# Disable auto-resume for a fresh download
70+
dl -no-resume https://example.com/file.zip
71+
72+
# Limit bandwidth to 1 MB/s
73+
dl -limit 1M https://example.com/file.zip
74+
75+
# Verify download with SHA256
76+
dl -checksum sha256:abc123... https://example.com/file.zip
77+
78+
# Use 4 connections with 5 retries
79+
dl -boost 4 -retries 5 https://example.com/file.zip
80+
81+
# Combine multiple options
82+
dl -boost 4 -limit 500K -filename data.tar.gz https://example.com/file.tar.gz
3083
```
31-
dl -filename blah.zip <file url>
84+
85+
## Configuration File
86+
87+
Create a `.dlrc` file in your home directory to set default values:
88+
89+
```bash
90+
# ~/.dlrc
91+
boost = 8
92+
retries = 3
3293
```
3394

34-
### Boost
95+
## Advanced Features
3596

36-
The boost will set the concurrency level. In typical concurrency scenarios you want to set this to the number of CPU threads available... however, we recommend keeping this at the default value of `8`. A higher value doesn't always lead to faster downloads. At some concurrency level, your network throughput will saturate.
97+
### Concurrent Downloads
3798

99+
The `-boost` parameter controls how many simultaneous connections are used. Higher values aren't always better - your network throughput will eventually saturate. The default of 8 works well for most scenarios.
100+
101+
### Auto-Resume Capability
102+
103+
Downloads now automatically resume by default! If a download is interrupted (network failure, Ctrl+C, system crash), simply run the same command again:
104+
105+
```bash
106+
# Start a download
107+
dl https://example.com/large-file.zip
108+
# ... download interrupted at 45% ...
109+
110+
# Just run the same command to resume
111+
dl https://example.com/large-file.zip
112+
# Output: Resuming download using progress file (45.0% complete)
38113
```
39-
dl -boost 8 <file url>
114+
115+
Features:
116+
- **Automatic detection** - Finds incomplete downloads and resumes automatically
117+
- **Progress persistence** - Saves exact download state in `.filename.dl_progress` files
118+
- **Multi-part awareness** - Resumes each parallel connection from exact byte position
119+
- **Sparse file support** - Efficient disk usage on macOS/Linux filesystems
120+
- **Smart validation** - Verifies same URL/filesize before resuming
121+
- **Clean completion** - Removes progress files after successful download
122+
123+
Progress tracking details:
124+
- Progress saved every 2 seconds during download
125+
- Each download part tracked individually
126+
- Survives crashes, network failures, and interruptions
127+
- Works with multi-connection downloads (preserves boost setting)
128+
129+
To force a fresh download without resuming:
130+
```bash
131+
dl -no-resume https://example.com/file.zip
40132
```
41133

42-
### Custom Working Directory
134+
The tool shows clear status messages:
135+
```
136+
Resuming download using progress file (73.2% complete)
137+
Part 0 already completed, skipping
138+
Part 1 already completed, skipping
139+
Downloading: 45.23 MB / 128.45 MB [=========>--------] 73.2% 3.2 MB/s 00:27
140+
```
141+
142+
### Bandwidth Limiting
143+
144+
Control download speed with `-limit`:
145+
- `1M` or `1MB` = 1 megabyte per second
146+
- `500K` or `500KB` = 500 kilobytes per second
147+
- `100KB/s` = 100 kilobytes per second
148+
149+
When using multiple connections, the bandwidth is divided equally among all connections.
150+
151+
### Checksum Verification
152+
153+
Verify file integrity after download:
154+
```bash
155+
dl -checksum sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 https://example.com/file.zip
156+
dl -checksum md5:d41d8cd98f00b204e9800998ecf8427e https://example.com/file.zip
157+
```
158+
159+
Supported algorithms: `md5`, `sha256`
160+
161+
### Error Handling
162+
163+
`dl` includes robust error handling:
164+
- **Exponential backoff** - Failed parts are retried with increasing delays (1s, 2s, 4s)
165+
- **Configurable retries** - Set max retry attempts with `-retries`
166+
- **Context-aware errors** - Clear error messages with relevant details
167+
- **Graceful shutdown** - Ctrl+C cleanly cancels downloads
168+
169+
## Technical Details
43170

44-
As of `dl` version 1.1, temporary files are no longer generated.
171+
- Uses HTTP/HTTPS with proper timeout configuration
172+
- Connection pooling for efficient resource usage
173+
- Progress tracking with real-time speed and ETA
174+
- Smart chunk boundary calculation for optimal performance
175+
- Rate limiting with context-aware cancellation

‎go.mod‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
module github.com/mgomes/dl
22

3-
go 1.22.2
3+
go 1.23.0
4+
5+
toolchain go1.24.2
46

57
require github.com/schollz/progressbar/v3 v3.7.2
68

@@ -9,4 +11,5 @@ require (
911
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
1012
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9 // indirect
1113
golang.org/x/sys v0.0.0-20220325203850-36772127a21f // indirect
14+
golang.org/x/time v0.11.0 // indirect
1215
)

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,5 @@ golang.org/x/sys v0.0.0-20201113135734-0a15ea8d9b02/go.mod h1:h1NjWce9XRLGQEsW7w
2525
golang.org/x/sys v0.0.0-20220325203850-36772127a21f h1:TrmogKRsSOxRMJbLYGrB4SBbW+LJcEllYBLME5Zk5pU=
2626
golang.org/x/sys v0.0.0-20220325203850-36772127a21f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
2727
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
28+
golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
29+
golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=

0 commit comments

Comments
 (0)