Skip to content

Commit 489b70d

Browse files
author
Hugo H.
committed
fix(ftp): some issues with ftp storageclass for big files
1 parent 3050f2c commit 489b70d

3 files changed

Lines changed: 166 additions & 91 deletions

File tree

‎.env.example‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,12 @@ FTP_USER=myuser
6666
FTP_PASSWORD=mypassword
6767
# Inactivity timeout in ms for the FTP connection (0 to disable, default 120000)
6868
FTP_TIMEOUT=120000
69+
# TCP keep-alive in ms on the FTP control connection, keeps it alive during long uploads (0 to disable, default 15000)
70+
FTP_KEEPALIVE=15000
71+
# Number of retries on a dropped FTP connection (default 2)
72+
FTP_RETRIES=2
73+
# Delay in ms before retrying a failed FTP operation (default 5000)
74+
FTP_RETRY_DELAY=5000
6975

7076
# SFTP Configuration
7177
SFTP_HOST=sftp.example.com

‎README.md‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -338,20 +338,24 @@ Increase this value if your network and server can handle more simultaneous conn
338338
339339
### Storage Settings
340340
341-
| Variable | Description | Default |
342-
| ----------------------- | ------------------------------------------------------------ | ------------ |
343-
| `STORAGE_TYPE` | The storage type to use. Supported: `ftp`, `sftp`, `local`. | `local` |
344-
| `LOCAL_STORAGE_PATH` | Path to store backups when using local storage. | `backups` |
345-
| `FTP_HOST` | Your FTP server host. | `localhost` |
346-
| `FTP_PORT` | Your FTP server port. | `21` |
347-
| `FTP_USER` | The username for the FTP connection. | `myuser` |
348-
| `FTP_PASSWORD` | The password for the FTP connection. | `mypassword` |
349-
| `SFTP_HOST` | Your SFTP server host. | `localhost` |
350-
| `SFTP_PORT` | Your SFTP server port. | `22` |
351-
| `SFTP_USER` | The username for the SFTP connection. | `myuser` |
352-
| `SFTP_PASSWORD` | The password for the SFTP connection (if not using SSH key). | _(empty)_ |
353-
| `SFTP_PRIVATE_KEY_PATH` | Path to the SSH private key file for SFTP authentication. | _(empty)_ |
354-
| `SFTP_PASSPHRASE` | Optional passphrase for the SSH private key. | _(empty)_ |
341+
| Variable | Description | Default |
342+
| ----------------------- | ------------------------------------------------------------------------------------------------------------------- | ------------ |
343+
| `STORAGE_TYPE` | The storage type to use. Supported: `ftp`, `sftp`, `local`. | `local` |
344+
| `LOCAL_STORAGE_PATH` | Path to store backups when using local storage. | `backups` |
345+
| `FTP_HOST` | Your FTP server host. | `localhost` |
346+
| `FTP_PORT` | Your FTP server port. | `21` |
347+
| `FTP_USER` | The username for the FTP connection. | `myuser` |
348+
| `FTP_PASSWORD` | The password for the FTP connection. | `mypassword` |
349+
| `FTP_TIMEOUT` | Inactivity timeout in ms for the FTP connection (`0` to disable). | `120000` |
350+
| `FTP_KEEPALIVE` | TCP keep-alive in ms on the FTP control connection, keeps it alive while a large file is uploaded (`0` to disable). | `15000` |
351+
| `FTP_RETRIES` | Number of retries when the FTP connection drops during an operation. | `2` |
352+
| `FTP_RETRY_DELAY` | Delay in ms before retrying a failed FTP operation. | `5000` |
353+
| `SFTP_HOST` | Your SFTP server host. | `localhost` |
354+
| `SFTP_PORT` | Your SFTP server port. | `22` |
355+
| `SFTP_USER` | The username for the SFTP connection. | `myuser` |
356+
| `SFTP_PASSWORD` | The password for the SFTP connection (if not using SSH key). | _(empty)_ |
357+
| `SFTP_PRIVATE_KEY_PATH` | Path to the SSH private key file for SFTP authentication. | _(empty)_ |
358+
| `SFTP_PASSPHRASE` | Optional passphrase for the SSH private key. | _(empty)_ |
355359
356360
#### Using Local Storage
357361
You must also set `LOCAL_STORAGE_PATH` to the directory where backups will be stored.

‎src/services/storage/ftp/main.ts‎

Lines changed: 142 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { Client } from "basic-ftp";
2+
import { statSync } from "node:fs";
23
import { StorageClass } from "../StorageClass";
34
import { logger } from "../../log";
45
import { convertToIP } from '../../../utils/ip';
@@ -8,7 +9,39 @@ const { FTP_HOST, FTP_PORT, FTP_USER, FTP_PASSWORD } = process.env;
89

910
// 0 disables the inactivity timeout entirely
1011
const FTP_TIMEOUT = ConvertToNumber(process.env.FTP_TIMEOUT, 120_000);
11-
const UPLOAD_RETRIES = 2;
12+
const FTP_KEEPALIVE = ConvertToNumber(process.env.FTP_KEEPALIVE, 15_000);
13+
const FTP_RETRIES = ConvertToNumber(process.env.FTP_RETRIES, 2);
14+
const FTP_RETRY_DELAY = ConvertToNumber(process.env.FTP_RETRY_DELAY, 5_000);
15+
16+
const CONNECTION_ERROR_PATTERNS = [
17+
"client is closed",
18+
"fin packet",
19+
"socket hang up",
20+
"timeout",
21+
"econnreset",
22+
"econnaborted",
23+
"econnrefused",
24+
"epipe",
25+
"etimedout",
26+
"ehostunreach",
27+
"enetunreach",
28+
"data connection"
29+
];
30+
31+
class RetryableError extends Error { }
32+
33+
function isRetryable(error: any) {
34+
if (error instanceof RetryableError) {
35+
return true;
36+
}
37+
38+
const message = String(error?.message ?? error).toLowerCase();
39+
return CONNECTION_ERROR_PATTERNS.some(pattern => message.includes(pattern));
40+
}
41+
42+
function wait(ms: number) {
43+
return new Promise<void>(resolve => setTimeout(resolve, ms));
44+
}
1245

1346
export class FTPStorage extends StorageClass {
1447
private client: Client;
@@ -29,128 +62,160 @@ export class FTPStorage extends StorageClass {
2962
password: FTP_PASSWORD,
3063
});
3164

65+
if (FTP_KEEPALIVE > 0) {
66+
this.client.ftp.socket.setKeepAlive(true, FTP_KEEPALIVE);
67+
}
68+
3269
logger.info(`Connected to FTP server: ${hostIp}:${FTP_PORT}`);
3370
}
3471

35-
// basic-ftp closes the client for good after a timeout; reconnect instead of
36-
// letting every following operation fail with "Client is closed"
72+
private async reconnect() {
73+
try {
74+
this.client.close();
75+
} catch (error) {
76+
logger.debug(`Failed to close the previous FTP client: ${error}`);
77+
}
78+
79+
this.client = new Client(FTP_TIMEOUT);
80+
await this.connect();
81+
}
82+
3783
private async ensureConnected() {
3884
if (this.client.closed) {
3985
logger.warn(`FTP connection lost, reconnecting...`);
40-
this.client = new Client(FTP_TIMEOUT);
41-
await this.connect();
86+
await this.reconnect();
4287
}
4388
}
4489

45-
async deleteFile(filePath: string) {
46-
await this.ensureConnected();
47-
await this.client.cd('/');
90+
private async run<T>(label: string, operation: () => Promise<T>): Promise<T> {
91+
for (let attempt = 1; ; attempt++) {
92+
try {
93+
await this.ensureConnected();
94+
await this.client.cd('/');
95+
return await operation();
96+
} catch (error) {
97+
if (attempt > FTP_RETRIES || !isRetryable(error)) {
98+
logger.error(`Failed to ${label}: ${error}`);
99+
throw error;
100+
}
48101

49-
try {
50-
await this.client.remove(filePath);
51-
logger.debug(`Deleted file: ${filePath}`);
52-
} catch (error) {
53-
logger.error(`Failed to delete file ${filePath}: ${error}`);
54-
throw error;
102+
logger.warn(`Failed to ${label} (attempt ${attempt}/${FTP_RETRIES + 1}): ${error}, reconnecting and retrying...`);
103+
await wait(FTP_RETRY_DELAY);
104+
105+
try {
106+
await this.reconnect();
107+
} catch (reconnectError) {
108+
logger.warn(`Failed to reconnect to the FTP server: ${reconnectError}`);
109+
}
110+
}
55111
}
56112
}
57113

58-
async uploadFile(filePath: string, destination: string) {
59-
for (let attempt = 1; attempt <= UPLOAD_RETRIES + 1; attempt++) {
114+
private async removeQuietly(filePath: string) {
115+
try {
60116
await this.ensureConnected();
61117
await this.client.cd('/');
118+
await this.client.remove(filePath, true);
119+
} catch (error) {
120+
logger.debug(`Could not remove ${filePath}: ${error}`);
121+
}
122+
}
62123

63-
try {
64-
await this.client.uploadFrom(filePath, destination);
65-
logger.debug(`Uploaded file: ${filePath}, to: ${destination}`);
66-
return;
67-
} catch (error) {
68-
if (attempt <= UPLOAD_RETRIES) {
69-
logger.warn(`Failed to upload file ${filePath} to ${destination} (attempt ${attempt}/${UPLOAD_RETRIES + 1}): ${error}, retrying...`);
70-
continue;
71-
}
124+
private async assertUploadedSize(destination: string, expectedSize: number) {
125+
let uploadedSize: number;
72126

73-
logger.error(`Failed to upload file ${filePath} to ${destination}: ${error}`);
127+
try {
128+
uploadedSize = await this.client.size(destination);
129+
} catch (error) {
130+
if (isRetryable(error)) {
74131
throw error;
75132
}
133+
134+
logger.warn(`Could not verify the size of ${destination}: ${error}`);
135+
return;
136+
}
137+
138+
if (uploadedSize !== expectedSize) {
139+
throw new RetryableError(`Incomplete upload of ${destination}: ${uploadedSize} bytes stored out of ${expectedSize}`);
76140
}
77141
}
78142

79-
async createFolder(folderPath: string) {
80-
await this.ensureConnected();
81-
await this.client.cd('/');
143+
async deleteFile(filePath: string) {
144+
await this.run(`delete file ${filePath}`, async () => {
145+
await this.client.remove(filePath);
146+
logger.debug(`Deleted file: ${filePath}`);
147+
});
148+
}
149+
150+
async uploadFile(filePath: string, destination: string) {
151+
const localSize = statSync(filePath).size;
82152

83153
try {
84-
await this.client.ensureDir(folderPath);
85-
logger.debug(`Created folder: ${folderPath}`);
154+
await this.run(`upload file ${filePath} to ${destination}`, async () => {
155+
await this.removeQuietly(destination);
156+
await this.client.uploadFrom(filePath, destination);
157+
await this.assertUploadedSize(destination, localSize);
158+
logger.debug(`Uploaded file: ${filePath}, to: ${destination}`);
159+
});
86160
} catch (error) {
87-
logger.error(`Failed to create folder ${folderPath}: ${error}`);
161+
await this.removeQuietly(destination);
88162
throw error;
89163
}
90164
}
91165

166+
async createFolder(folderPath: string) {
167+
await this.run(`create folder ${folderPath}`, async () => {
168+
await this.client.ensureDir(folderPath);
169+
logger.debug(`Created folder: ${folderPath}`);
170+
});
171+
}
172+
92173
async deleteFolder(folderPath: string) {
93-
await this.ensureConnected();
94-
await this.client.cd('/');
95-
96-
try {
174+
await this.run(`delete folder ${folderPath}`, async () => {
97175
await this.client.removeDir(folderPath);
98176
logger.debug(`Deleted folder: ${folderPath}`);
99-
} catch (error) {
100-
logger.error(`Failed to delete folder ${folderPath}: ${error}`);
101-
throw error;
102-
}
177+
});
103178
}
104179

105180
async folderExists(folderPath: string): Promise<boolean> {
106-
await this.ensureConnected();
107-
await this.client.cd('/');
108-
109-
try {
110-
const list = await this.client.list(folderPath);
111-
return list.some(file => file.name === folderPath && file.isDirectory);
112-
} catch (error) {
113-
if (error.code === 550) { // 550 means "not found"
114-
return false;
181+
return this.run(`check if folder ${folderPath} exists`, async () => {
182+
try {
183+
const list = await this.client.list(folderPath);
184+
return list.some(file => file.name === folderPath && file.isDirectory);
185+
} catch (error) {
186+
if (error.code === 550) { // 550 means "not found"
187+
return false;
188+
}
189+
throw error; // rethrow other errors
115190
}
116-
throw error; // rethrow other errors
117-
}
191+
});
118192
}
119193

120194
async folderSizeBytes(folderPath: string) {
121-
await this.ensureConnected();
122-
const list = await this.client.list(folderPath);
123-
return list.reduce((total, file) => total + (file.size || 0), 0);
195+
return this.run(`get the size of folder ${folderPath}`, async () => {
196+
const list = await this.client.list(folderPath);
197+
return list.reduce((total, file) => total + (file.size || 0), 0);
198+
});
124199
}
125200

126201
async listFiles(folderPath: string) {
127-
await this.ensureConnected();
128-
await this.client.cd('/');
129-
130-
try {
202+
return this.run(`list files in folder ${folderPath}`, async () => {
131203
const list = await this.client.list(folderPath);
132204

133-
const result = [];
134-
for await (const file of list) {
135-
result.push({
136-
fileName: file.name,
137-
filePath: `${folderPath}/${file.name}`,
138-
size: file.size || 0,
139-
lastModified: file.modifiedAt || new Date(),
140-
isDirectory: file.isDirectory
141-
});
142-
}
143-
return result;
144-
} catch (error) {
145-
logger.error(`Failed to list files in folder ${folderPath}: ${error}`);
146-
throw error;
147-
}
205+
return list.map(file => ({
206+
fileName: file.name,
207+
filePath: `${folderPath}/${file.name}`,
208+
size: file.size || 0,
209+
lastModified: file.modifiedAt || new Date(),
210+
isDirectory: file.isDirectory
211+
}));
212+
});
148213
}
149-
214+
150215
async close() {
151-
await this.client.close();
216+
this.client.close();
152217
}
153218
async init() {
154219
await this.connect();
155-
}
156-
}
220+
}
221+
}

0 commit comments

Comments
 (0)