fix: occasional "socket hang up" errors in APM server intake requests (#179)

Use the 'agentkeepalive' alternative to Node.js core `http.Agent`
for keep-alive handling to get the `freeSocketTimeout` option that
can timeout kept-alive (aka "free") sockets before getting in the
range of the downstream APM Server (or Lambda extension) HTTP
Keep-Alive timeout. This avoids occasional ECONNRESET errors.

Refs: elastic/apm-agent-nodejs#2594

Author: trentm

CHANGELOG.md

@@ -19,6 +19,11 @@
   current elastic-apm-node with this version of the client would break
   behavior in a Lambda environment.
 
+- Add the `freeSocketTimeout` option, with a default of 4000 (ms), and switch
+  from Node.js's core `http.Agent` to the [agentkeepalive package](https://github.com/node-modules/agentkeepalive)
+  to fix ECONNRESET issues with HTTP Keep-Alive usage talking to APM Server
+  (https://github.com/elastic/apm-agent-nodejs/issues/2594).
+
 ## v10.4.0
 
 - Add APM Server version checking to the client. On creation the client will

README.md

@@ -104,6 +104,15 @@ HTTP client configuration:
   `Infinity`)
 - `maxFreeSockets` - Maximum number of sockets to leave open in a free
   state. Only relevant if `keepAlive` is set to `true` (default: `256`)
+- `freeSocketTimeout` - A number of milliseconds of inactivity on a free
+  (kept-alive) socket after which to timeout and recycle the socket. Set this to
+  a value less than the HTTP Keep-Alive timeout of the APM server to avoid
+  [ECONNRESET exceptions](https://medium.com/ssense-tech/reduce-networking-errors-in-nodejs-23b4eb9f2d83).
+  This defaults to 4000ms to be less than the [node.js HTTP server default of
+  5s](https://nodejs.org/api/http.html#serverkeepalivetimeout) (useful when
+  using a Node.js-based mock APM server) and the [Go lang Dialer `KeepAlive`
+  default of 15s](https://pkg.go.dev/net#Dialer) (when talking to the Elastic
+  APM Lambda extension). (default: `4000`)
 
 Cloud & Extra Metadata Configuration:
 

index.js

@@ -11,6 +11,8 @@ const { performance } = require('perf_hooks')
 const { URL } = require('url')
 const zlib = require('zlib')
 
+const HttpAgentKeepAlive = require('agentkeepalive')
+const HttpsAgentKeepAlive = HttpAgentKeepAlive.HttpsAgent
 const Filters = require('object-filter-sequence')
 const querystring = require('querystring')
 const Writable = require('readable-stream').Writable
@@ -244,6 +246,10 @@ Client.prototype.config = function (opts) {
   if (!this._conf.intakeResTimeoutOnEnd) this._conf.intakeResTimeoutOnEnd = 1000
   this._conf.keepAlive = this._conf.keepAlive !== false
   this._conf.centralConfig = this._conf.centralConfig || false
+  if (!('keepAliveMsecs' in this._conf)) this._conf.keepAliveMsecs = 1000
+  if (!('maxSockets' in this._conf)) this._conf.maxSockets = Infinity
+  if (!('maxFreeSockets' in this._conf)) this._conf.maxFreeSockets = 256
+  if (!('freeSocketTimeout' in this._conf)) this._conf.freeSocketTimeout = 4000
 
   // processed values
   this._conf.serverUrl = new URL(this._conf.serverUrl)
@@ -260,16 +266,19 @@ Client.prototype.config = function (opts) {
     }
   }
 
+  let AgentKeepAlive
   switch (this._conf.serverUrl.protocol) {
     case 'http:':
       this._transport = http
       this._transportRequest = httpRequest
       this._transportGet = httpGet
+      AgentKeepAlive = HttpAgentKeepAlive
       break
     case 'https:':
       this._transport = https
       this._transportRequest = httpsRequest
       this._transportGet = httpsGet
+      AgentKeepAlive = HttpsAgentKeepAlive
       break
     default:
       throw new Error('Unknown protocol ' + this._conf.serverUrl.protocol)
@@ -281,13 +290,14 @@ Client.prototype.config = function (opts) {
     if (this._agent) {
       this._agent.destroy()
     }
-    var agentOpts = {
+    this._agent = new AgentKeepAlive({
       keepAlive: this._conf.keepAlive,
       keepAliveMsecs: this._conf.keepAliveMsecs,
+      freeSocketTimeout: this._conf.freeSocketTimeout,
+      timeout: this._conf.serverTimeout,
       maxSockets: this._conf.maxSockets,
       maxFreeSockets: this._conf.maxFreeSockets
-    }
-    this._agent = new this._transport.Agent(agentOpts)
+    })
   }
 
   // http request options

package.json

@@ -18,6 +18,7 @@
   "author": "Thomas Watson <[email protected]> (https://twitter.com/wa7son)",
   "license": "MIT",
   "dependencies": {
+    "agentkeepalive": "^4.2.1",
     "breadth-filter": "^2.0.0",
     "container-info": "^1.0.1",
     "end-of-stream": "^1.4.4",

test/edge-cases.test.js

@@ -347,7 +347,7 @@ test('socket timeout - server response too slow', function (t) {
       t.ok(err, 'got a request-error from the client')
       const end = Date.now()
       const delta = end - start
-      t.ok(delta > 1000 && delta < 2000, 'timeout should occur between 1-2 seconds')
+      t.ok(delta > 1000 && delta < 2000, `timeout should occur between 1-2 seconds: delta=${delta}ms`)
       t.equal(err.message, 'APM Server response timeout (1000ms)')
       server.close()
       t.end()