Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions .fern/metadata.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"cliVersion": "1.0.4",
"generatorName": "fernapi/fern-java-sdk",
"generatorVersion": "3.15.1",
"generatorConfig": {
"publish-to": "central",
"client-class-name": "BaseClient",
"custom-dependencies": [
"api org.apache.commons:commons-text:1.13.1"
]
}
}
37 changes: 34 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,22 @@

The Pipedream Java library provides convenient access to the Pipedream APIs from Java.

## Table of Contents

- [Installation](#installation)
- [Usage](#usage)
- [Environments](#environments)
- [Base Url](#base-url)
- [Exception Handling](#exception-handling)
- [Advanced](#advanced)
- [Custom Client](#custom-client)
- [Retries](#retries)
- [Timeouts](#timeouts)
- [Custom Headers](#custom-headers)
- [Access Raw Response Data](#access-raw-response-data)
- [Contributing](#contributing)
- [Reference](#reference)

## Installation

### Gradle
Expand All @@ -25,7 +41,7 @@ Add the dependency in your `pom.xml` file:
<dependency>
<groupId>com.pipedream</groupId>
<artifactId>pipedream</artifactId>
<version>1.1.0</version>
<version>1.1.1</version>
</dependency>
```

Expand Down Expand Up @@ -104,7 +120,7 @@ try{

### Custom Client

This SDK is built to work with any instance of `OkHttpClient`. By default, if no client is provided, the SDK will construct one.
This SDK is built to work with any instance of `OkHttpClient`. By default, if no client is provided, the SDK will construct one.
However, you can pass your own client like so:

```java
Expand All @@ -123,7 +139,9 @@ BaseClient client = BaseClient

The SDK is instrumented with automatic retries with exponential backoff. A request will be retried as long
as the request is deemed retryable and the number of retry attempts has not grown larger than the configured
retry limit (default: 2).
retry limit (default: 2). Before defaulting to exponential backoff, the SDK will first attempt to respect
the `Retry-After` header (as either in seconds or as an HTTP date), and then the `X-RateLimit-Reset` header
(as a Unix timestamp in epoch seconds); failing both of those, it will fall back to exponential backoff.

A request is deemed retryable when any of the following HTTP status codes is returned:

Expand Down Expand Up @@ -192,6 +210,19 @@ client.actions().run(
);
```

### Access Raw Response Data

The SDK provides access to raw response data, including headers, through the `withRawResponse()` method.
The `withRawResponse()` method returns a raw client that wraps all responses with `body()` and `headers()` methods.
(A normal client's `response` is identical to a raw client's `response.body()`.)

```java
RunHttpResponse response = client.actions().withRawResponse().run(...);

System.out.println(response.body());
System.out.println(response.headers().get("X-My-Header"));
```

## Contributing

While we value open-source contributions to this SDK, this library is generated programmatically.
Expand Down
13 changes: 6 additions & 7 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,14 @@ repositories {
}

dependencies {
api 'com.squareup.okhttp3:okhttp:4.12.0'
api 'com.fasterxml.jackson.core:jackson-databind:2.17.2'
api 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.17.2'
api 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.17.2'
api 'com.squareup.okhttp3:okhttp:5.2.1'
api 'com.fasterxml.jackson.core:jackson-databind:2.18.2'
api 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.18.2'
api 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18.2'
testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.2'
testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.8.2'
testImplementation 'org.junit.jupiter:junit-jupiter-params:5.8.2'
api 'org.apache.commons:commons-text:1.13.1'
testImplementation 'com.squareup.okhttp3:mockwebserver:4.12.0'
}


Expand All @@ -49,7 +48,7 @@ java {

group = 'com.pipedream'

version = '1.1.0'
version = '1.1.1'

jar {
dependsOn(":generatePomFileForMavenPublication")
Expand Down Expand Up @@ -80,7 +79,7 @@ publishing {
maven(MavenPublication) {
groupId = 'com.pipedream'
artifactId = 'pipedream'
version = '1.1.0'
version = '1.1.1'
from components.java
pom {
name = 'pipedream'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,9 @@ public Map<String, List<String>> headers() {
return this.headers;
}

@java.lang.Override
@Override
public String toString() {
return "BaseClientApiException{" + "message: " + getMessage() + ", statusCode: " + statusCode + ", body: "
+ body + "}";
+ ObjectMappers.stringify(body) + "}";
}
}
20 changes: 17 additions & 3 deletions src/main/java/com/pipedream/api/core/ClientOptions.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ public final class ClientOptions {

private final int timeout;

private final int maxRetries;

private String projectId;

private ClientOptions(
Expand All @@ -29,21 +31,23 @@ private ClientOptions(
Map<String, Supplier<String>> headerSuppliers,
OkHttpClient httpClient,
int timeout,
int maxRetries,
String projectId) {
this.environment = environment;
this.headers = new HashMap<>();
this.headers.putAll(headers);
this.headers.putAll(new HashMap<String, String>() {
{
put("User-Agent", "com.pipedream:pipedream/1.1.0");
put("User-Agent", "com.pipedream:pipedream/1.1.1");
put("X-Fern-Language", "JAVA");
put("X-Fern-SDK-Name", "com.pipedream.fern:api-sdk");
put("X-Fern-SDK-Version", "1.1.0");
put("X-Fern-SDK-Version", "1.1.1");
}
});
this.headerSuppliers = headerSuppliers;
this.httpClient = httpClient;
this.timeout = timeout;
this.maxRetries = maxRetries;
this.projectId = projectId;
}

Expand Down Expand Up @@ -86,6 +90,10 @@ public OkHttpClient httpClientWithTimeout(RequestOptions requestOptions) {
.build();
}

public int maxRetries() {
return this.maxRetries;
}

public String projectId() {
return this.projectId;
}
Expand Down Expand Up @@ -181,7 +189,13 @@ public ClientOptions build() {
this.timeout = Optional.of(httpClient.callTimeoutMillis() / 1000);

return new ClientOptions(
environment, headers, headerSuppliers, httpClient, this.timeout.get(), this.projectId);
environment,
headers,
headerSuppliers,
httpClient,
this.timeout.get(),
this.maxRetries,
this.projectId);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ public boolean equals(Object o) {
}

private boolean isOptionalEmpty(Object o) {
return o instanceof Optional && !((Optional<?>) o).isPresent();
if (o instanceof Optional) {
return !((Optional<?>) o).isPresent();
}
return false;
}
}
9 changes: 9 additions & 0 deletions src/main/java/com/pipedream/api/core/ObjectMappers.java
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package com.pipedream.api.core;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
Expand Down Expand Up @@ -33,4 +34,12 @@ public static String stringify(Object o) {
return o.getClass().getName() + "@" + Integer.toHexString(o.hashCode());
}
}

public static Object parseErrorBody(String responseBodyString) {
try {
return JSON_MAPPER.readValue(responseBodyString, Object.class);
} catch (JsonProcessingException ignored) {
return responseBodyString;
}
}
}
118 changes: 110 additions & 8 deletions src/main/java/com/pipedream/api/core/RetryInterceptor.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,20 @@

import java.io.IOException;
import java.time.Duration;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Optional;
import java.util.Random;
import okhttp3.Interceptor;
import okhttp3.Response;

public class RetryInterceptor implements Interceptor {

private static final Duration ONE_SECOND = Duration.ofSeconds(1);
private static final Duration INITIAL_RETRY_DELAY = Duration.ofMillis(1000);
private static final Duration MAX_RETRY_DELAY = Duration.ofMillis(60000);
private static final double JITTER_FACTOR = 0.2;

private final ExponentialBackoff backoff;
private final Random random = new Random();

Expand All @@ -32,7 +38,7 @@ public Response intercept(Chain chain) throws IOException {
}

private Response retryChain(Response response, Chain chain) throws IOException {
Optional<Duration> nextBackoff = this.backoff.nextBackoff();
Optional<Duration> nextBackoff = this.backoff.nextBackoff(response);
while (nextBackoff.isPresent()) {
try {
Thread.sleep(nextBackoff.get().toMillis());
Expand All @@ -42,7 +48,7 @@ private Response retryChain(Response response, Chain chain) throws IOException {
response.close();
response = chain.proceed(chain.request());
if (shouldRetry(response.code())) {
nextBackoff = this.backoff.nextBackoff();
nextBackoff = this.backoff.nextBackoff(response);
} else {
return response;
}
Expand All @@ -51,6 +57,102 @@ private Response retryChain(Response response, Chain chain) throws IOException {
return response;
}

/**
* Calculates the retry delay from response headers, with fallback to exponential backoff.
* Priority: Retry-After > X-RateLimit-Reset > Exponential Backoff
*/
private Duration getRetryDelayFromHeaders(Response response, int retryAttempt) {
// Check for Retry-After header first (RFC 7231), with no jitter
String retryAfter = response.header("Retry-After");
if (retryAfter != null) {
// Parse as number of seconds...
Optional<Duration> secondsDelay = tryParseLong(retryAfter)
.map(seconds -> seconds * 1000)
.filter(delayMs -> delayMs > 0)
.map(delayMs -> Math.min(delayMs, MAX_RETRY_DELAY.toMillis()))
.map(Duration::ofMillis);
if (secondsDelay.isPresent()) {
return secondsDelay.get();
}

// ...or as an HTTP date; both are valid
Optional<Duration> dateDelay = tryParseHttpDate(retryAfter)
.map(resetTime -> resetTime.toInstant().toEpochMilli() - System.currentTimeMillis())
.filter(delayMs -> delayMs > 0)
.map(delayMs -> Math.min(delayMs, MAX_RETRY_DELAY.toMillis()))
.map(Duration::ofMillis);
if (dateDelay.isPresent()) {
return dateDelay.get();
}
}

// Then check for industry-standard X-RateLimit-Reset header, with positive jitter
String rateLimitReset = response.header("X-RateLimit-Reset");
if (rateLimitReset != null) {
// Assume Unix timestamp in epoch seconds
Optional<Duration> rateLimitDelay = tryParseLong(rateLimitReset)
.map(resetTimeSeconds -> (resetTimeSeconds * 1000) - System.currentTimeMillis())
.filter(delayMs -> delayMs > 0)
.map(delayMs -> Math.min(delayMs, MAX_RETRY_DELAY.toMillis()))
.map(this::addPositiveJitter)
.map(Duration::ofMillis);
if (rateLimitDelay.isPresent()) {
return rateLimitDelay.get();
}
}

// Fall back to exponential backoff, with symmetric jitter
long baseDelay = INITIAL_RETRY_DELAY.toMillis() * (1L << retryAttempt); // 2^retryAttempt
long cappedDelay = Math.min(baseDelay, MAX_RETRY_DELAY.toMillis());
return Duration.ofMillis(addSymmetricJitter(cappedDelay));
}

/**
* Attempts to parse a string as a long, returning empty Optional on failure.
*/
private Optional<Long> tryParseLong(String value) {
if (value == null) {
return Optional.empty();
}
try {
return Optional.of(Long.parseLong(value));
} catch (NumberFormatException e) {
return Optional.empty();
}
}

/**
* Attempts to parse a string as an HTTP date (RFC 1123), returning empty Optional on failure.
*/
private Optional<ZonedDateTime> tryParseHttpDate(String value) {
if (value == null) {
return Optional.empty();
}
try {
return Optional.of(ZonedDateTime.parse(value, DateTimeFormatter.RFC_1123_DATE_TIME));
} catch (DateTimeParseException e) {
return Optional.empty();
}
}

/**
* Adds positive jitter (100-120% of original value) to prevent thundering herd.
* Used for X-RateLimit-Reset header delays.
*/
private long addPositiveJitter(long delayMs) {
double jitterMultiplier = 1.0 + (random.nextDouble() * JITTER_FACTOR);
return (long) (delayMs * jitterMultiplier);
}

/**
* Adds symmetric jitter (90-110% of original value) to prevent thundering herd.
* Used for exponential backoff delays.
*/
private long addSymmetricJitter(long delayMs) {
double jitterMultiplier = 1.0 + ((random.nextDouble() - 0.5) * JITTER_FACTOR);
return (long) (delayMs * jitterMultiplier);
}

private static boolean shouldRetry(int statusCode) {
return statusCode == 408 || statusCode == 429 || statusCode >= 500;
}
Expand All @@ -65,14 +167,14 @@ private final class ExponentialBackoff {
this.maxNumRetries = maxNumRetries;
}

public Optional<Duration> nextBackoff() {
retryNumber += 1;
if (retryNumber > maxNumRetries) {
public Optional<Duration> nextBackoff(Response response) {
if (retryNumber >= maxNumRetries) {
return Optional.empty();
}

int upperBound = (int) Math.pow(2, retryNumber);
return Optional.of(ONE_SECOND.multipliedBy(random.nextInt(upperBound)));
Duration delay = getRetryDelayFromHeaders(response, retryNumber);
retryNumber += 1;
return Optional.of(delay);
}
}
}
Loading
Loading