diff --git a/.github/workflows/pullrequest.yml b/.github/workflows/pullrequest.yml index 0c4137ed0..be3b52fe5 100644 --- a/.github/workflows/pullrequest.yml +++ b/.github/workflows/pullrequest.yml @@ -16,7 +16,7 @@ jobs: - java: 11 profile: java11 name: with Java ${{ matrix.build.java }} - runs-on: ${{ matrix.os}} + runs-on: ${{ matrix.os }} steps: - name: Check out the code uses: actions/checkout@09d2acae674a48949e3602304ab46fd20ae0c42f @@ -54,5 +54,9 @@ jobs: fail_ci_if_error: true # optional (default = false) verbose: true # optional (default = false) + - if: matrix.build.java == '17' + name: Run JMH benchmark + run: mvn --activate-profiles benchmark jmh:benchmark "-Djmh.rf=json" "-Djmh.rff=benchmark/results/results.json" + - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@624d0bca90f761ffa7ce50c41875a1a226969a02 diff --git a/pom.xml b/pom.xml index c4461155f..211d4a038 100644 --- a/pom.xml +++ b/pom.xml @@ -167,7 +167,12 @@ 1.37 test - + @@ -298,7 +303,6 @@ - @@ -703,6 +707,24 @@ + diff --git a/src/test/java/dev/openfeature/sdk/benchmark/FlagEvaluationBenchmark.java b/src/test/java/dev/openfeature/sdk/benchmark/FlagEvaluationBenchmark.java new file mode 100644 index 000000000..c7a454124 --- /dev/null +++ b/src/test/java/dev/openfeature/sdk/benchmark/FlagEvaluationBenchmark.java @@ -0,0 +1,32 @@ +package dev.openfeature.sdk.benchmark; + +import dev.openfeature.sdk.benchmark.state.FlagEvaluationState; +import dev.openfeature.sdk.benchmark.state.HooksState; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Warmup; + +@BenchmarkMode(Mode.Throughput) +@Warmup(time = 1, timeUnit = TimeUnit.SECONDS, iterations = 1) +// @Warmup(time = 1, timeUnit = TimeUnit.SECONDS, iterations = 2) +@Measurement(time = 5, timeUnit = TimeUnit.SECONDS, iterations = 1) +// @Measurement(time = 5, timeUnit = TimeUnit.SECONDS, iterations = 4) +@Fork(1) +public class FlagEvaluationBenchmark { + + @Benchmark + public String flagEvaluations(FlagEvaluationState state) { + return state.client + .getStringDetails(FlagEvaluationState.FLAG_KEY, "default") + .getValue(); + } + + @Benchmark + public String hookExecution(HooksState state) { + return state.client.getStringDetails(HooksState.FLAG_KEY, "default").getValue(); + } +} diff --git a/src/test/java/dev/openfeature/sdk/benchmark/state/FlagEvaluationState.java b/src/test/java/dev/openfeature/sdk/benchmark/state/FlagEvaluationState.java new file mode 100644 index 000000000..53eb73765 --- /dev/null +++ b/src/test/java/dev/openfeature/sdk/benchmark/state/FlagEvaluationState.java @@ -0,0 +1,40 @@ +package dev.openfeature.sdk.benchmark.state; + +import dev.openfeature.sdk.Client; +import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.OpenFeatureAPI; +import dev.openfeature.sdk.providers.memory.Flag; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import java.util.Map; +import org.openjdk.jmh.annotations.*; + +@State(Scope.Benchmark) +public class FlagEvaluationState { + public static final String FLAG_KEY = "flag-key"; + public static final String DOMAIN = "jmh-domain"; + + public InMemoryProvider provider; + public Client client; + + @Setup(Level.Trial) + public void setup() { + provider = new InMemoryProvider(Map.of( + FLAG_KEY, + Flag.builder() + .variant("a", "a-value") + .variant("b", "b-value") + .defaultVariant("b") + .flagMetadata(ImmutableMetadata.builder() + .addString("meta", "data") + .build()) + .build())); + OpenFeatureAPI.getInstance().setProviderAndWait(DOMAIN, provider); + client = OpenFeatureAPI.getInstance().getClient(DOMAIN); + } + + @TearDown(Level.Trial) + public void teardown() { + OpenFeatureAPI.getInstance().shutdown(); + OpenFeatureAPI.getInstance().clearHooks(); + } +} diff --git a/src/test/java/dev/openfeature/sdk/benchmark/state/HooksState.java b/src/test/java/dev/openfeature/sdk/benchmark/state/HooksState.java new file mode 100644 index 000000000..cd1993afb --- /dev/null +++ b/src/test/java/dev/openfeature/sdk/benchmark/state/HooksState.java @@ -0,0 +1,47 @@ +package dev.openfeature.sdk.benchmark.state; + +import dev.openfeature.sdk.Client; +import dev.openfeature.sdk.Hook; +import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.OpenFeatureAPI; +import dev.openfeature.sdk.StringHook; +import dev.openfeature.sdk.providers.memory.Flag; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import java.util.Map; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; + +@State(Scope.Benchmark) +public class HooksState { + public static final String FLAG_KEY = "flag-key"; + public static final String DOMAIN = "jmh-domain"; + + public InMemoryProvider provider; + public Client client; + + @Setup(Level.Trial) + public void setup() { + provider = new InMemoryProvider(Map.of( + FLAG_KEY, + Flag.builder() + .variant("a", "a-value") + .variant("b", "b-value") + .defaultVariant("b") + .flagMetadata(ImmutableMetadata.builder() + .addString("meta", "data") + .build()) + .build())); + OpenFeatureAPI.getInstance().setProviderAndWait(DOMAIN, provider); + client = OpenFeatureAPI.getInstance().getClient(DOMAIN); + client.addHooks(new StringHook() {}, new Hook() {}); + } + + @TearDown(Level.Trial) + public void teardown() { + OpenFeatureAPI.getInstance().shutdown(); + OpenFeatureAPI.getInstance().clearHooks(); + } +}