chore: add e2e test suite (#211)

First pass at adding the standard gherkin suite to flagd ruby.

I was able to add quite a lot of coverage without changes - most of
what's missing is just the in-process stuff which isn't implemented yet.

---------

Signed-off-by: Todd Baert <todd.baert@dynatrace.com>

Author: toddbaert

.github/workflows/ruby.yml

@@ -44,6 +44,32 @@ jobs:
       run: bundle exec rubocop
     - name: Run tests
       run: bundle exec rspec
+
+  test_flagd_provider_e2e:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./providers/openfeature-flagd-provider
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version:
+          - "3.3"
+          - "3.4"
+          - "4.0"
+    steps:
+    - uses: actions/checkout@v7
+      with:
+        submodules: recursive
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true
+        working-directory: ./providers/openfeature-flagd-provider
+    - name: Run e2e (flagd-testbed / cucumber)
+      run: bundle exec rake e2e
+
   test_meta_provider:
     runs-on: ubuntu-latest
     defaults:

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "providers/openfeature-flagd-provider/schemas"]
 	path = providers/openfeature-flagd-provider/schemas
 	url = https://github.com/open-feature/flagd-schemas
+[submodule "providers/openfeature-flagd-provider/test-harness"]
+	path = providers/openfeature-flagd-provider/test-harness
+	url = https://github.com/open-feature/flagd-testbed.git

providers/openfeature-flagd-provider/Gemfile.lock

@@ -10,8 +10,37 @@ GEM
   specs:
     ast (2.4.3)
     bigdecimal (4.0.1)
+    builder (3.3.0)
+    cucumber (9.2.1)
+      builder (~> 3.2)
+      cucumber-ci-environment (> 9, < 11)
+      cucumber-core (> 13, < 14)
+      cucumber-cucumber-expressions (~> 17.0)
+      cucumber-gherkin (> 24, < 28)
+      cucumber-html-formatter (> 20.3, < 22)
+      cucumber-messages (> 19, < 25)
+      diff-lcs (~> 1.5)
+      mini_mime (~> 1.1)
+      multi_test (~> 1.1)
+      sys-uname (~> 1.2)
+    cucumber-ci-environment (10.0.1)
+    cucumber-core (13.0.3)
+      cucumber-gherkin (>= 27, < 28)
+      cucumber-messages (>= 20, < 23)
+      cucumber-tag-expressions (> 5, < 7)
+    cucumber-cucumber-expressions (17.1.0)
+      bigdecimal
+    cucumber-gherkin (27.0.0)
+      cucumber-messages (>= 19.1.4, < 23)
+    cucumber-html-formatter (21.15.1)
+      cucumber-messages (> 19, < 28)
+    cucumber-messages (22.0.0)
+    cucumber-tag-expressions (6.1.2)
     diff-lcs (1.5.1)
     docile (1.4.1)
+    ffi (1.17.4)
+    ffi (1.17.4-arm64-darwin)
+    ffi (1.17.4-x86_64-linux-gnu)
     google-protobuf (4.34.0)
       bigdecimal
       rake (~> 13.3)
@@ -35,6 +64,10 @@ GEM
     json (2.21.2)
     language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
+    logger (1.7.0)
+    memoist3 (1.0.0)
+    mini_mime (1.1.5)
+    multi_test (1.1.0)
     openfeature-sdk (0.3.1)
     parallel (1.27.0)
     parser (3.3.10.2)
@@ -95,6 +128,9 @@ GEM
     standard-performance (1.9.0)
       lint_roller (~> 1.1)
       rubocop-performance (~> 1.26.0)
+    sys-uname (1.5.1)
+      ffi (~> 1.1)
+      memoist3 (~> 1.0.0)
     unicode-display_width (3.2.0)
       unicode-emoji (~> 4.1)
     unicode-emoji (4.2.0)
@@ -106,6 +142,8 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  cucumber (~> 9.2)
+  logger
   openfeature-flagd-provider!
   rake (~> 13.0)
   rspec (~> 3.12.0)

providers/openfeature-flagd-provider/Rakefile

@@ -2,7 +2,12 @@
 
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
+require "cucumber/rake/task"
 
 RSpec::Core::RakeTask.new(:spec)
 
+# End-to-end suite: runs the shared flagd-testbed gherkin features against a testbed container.
+# Requires Docker; not part of the default task.
+Cucumber::Rake::Task.new(:e2e)
+
 task default: :spec

providers/openfeature-flagd-provider/cucumber.yml

@@ -0,0 +1 @@
+default: --require features --publish-quiet --tags "not @deprecated and not @no-default-variant and not @fractional-v1 and not @fractional-v3" test-harness/gherkin/evaluation.feature test-harness/gherkin/targeting.feature

providers/openfeature-flagd-provider/features/step_definitions/flagd_steps.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "json"
+
+module StepHelpers
+  def cast(type, value)
+    case type
+    when "Boolean" then value == "true"
+    when "Integer" then value.to_i
+    when "Float" then value.to_f
+    when "Object" then JSON.parse(value)
+    else value
+    end
+  end
+
+  # flagd returns object values as a protobuf Struct; normalise to a Hash for comparison.
+  def unwrap(value)
+    value.respond_to?(:to_h) ? value.to_h : value
+  end
+end
+World(StepHelpers)
+
+Given(/^an option "[^"]*" of type "[^"]*" with value "[^"]*"$/) do
+  # no-op: caching/streaming options aren't applicable here
+end
+
+Given(/^a stable flagd provider$/) do
+  @client = OpenFeature::Flagd::Provider.build_client
+  @context = {}
+end
+
+Given(/^a (\w+)-flag with key "([^"]*)" and a default value "([^"]*)"$/) do |type, key, default|
+  @type = type
+  @key = key
+  @default = default
+end
+
+Given(/^a context containing a key "([^"]*)", with type "([^"]*)" and with value "([^"]*)"$/) do |key, type, value|
+  @context[key] = cast(type, value)
+end
+
+Given(/^a context containing a targeting key with value "([^"]*)"$/) do |value|
+  # the provider maps the "targeting_key" field to flagd's targetingKey
+  @context["targeting_key"] = value
+end
+
+Given(/^a context containing a nested property with outer key "([^"]*)" and inner key "([^"]*)", with value "([^"]*)"$/) do |outer, inner, value|
+  (@context[outer] ||= {})[inner] = value
+end
+
+When(/^the flag was evaluated with details$/) do
+  ctx = @context.empty? ? nil : OpenFeature::SDK::EvaluationContext.new(**@context.transform_keys(&:to_sym))
+
+  @details =
+    case @type
+    when "Boolean" then @client.fetch_boolean_value(flag_key: @key, default_value: cast("Boolean", @default), evaluation_context: ctx)
+    when "String" then @client.fetch_string_value(flag_key: @key, default_value: @default, evaluation_context: ctx)
+    when "Integer" then @client.fetch_integer_value(flag_key: @key, default_value: @default.to_i, evaluation_context: ctx)
+    when "Float" then @client.fetch_float_value(flag_key: @key, default_value: @default.to_f, evaluation_context: ctx)
+    when "Object" then @client.fetch_object_value(flag_key: @key, default_value: cast("Object", @default), evaluation_context: ctx)
+    else raise "unsupported flag type: #{@type}"
+    end
+end
+
+Then(/^the resolved details value should be "([^"]*)"$/) do |value|
+  actual = (@type == "Object") ? unwrap(@details[:value]) : @details[:value]
+  expect(actual).to eq(cast(@type, value))
+end
+
+Then(/^the reason should be "([^"]*)"$/) do |reason|
+  expect(@details[:reason]).to eq(reason)
+end
+
+Then(/^the error-code should be "([^"]*)"$/) do |code|
+  if code.empty?
+    expect(@details[:error_code]).to be_nil
+  else
+    expect(@details[:error_code]).to eq(code)
+  end
+end

providers/openfeature-flagd-provider/features/support/env.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "rspec/expectations"
+require "open_feature/sdk"
+require "openfeature/flagd/provider"
+
+World(RSpec::Matchers)
+
+# Boots the shared flagd testbed container and drives its launchpad so flagd is serving the
+# standard flag set before the suite runs. Uses the Docker CLI directly (reliable on CI).
+module Testbed
+  VERSION = File.read(File.expand_path("../../test-harness/version.txt", __dir__)).strip
+  IMAGE = "ghcr.io/open-feature/flagd-testbed:v#{VERSION}"
+  NAME = "flagd-e2e-ruby-#{Process.pid}"
+
+  module_function
+
+  def start
+    sh("docker rm -f #{NAME}")
+    raise "failed to start #{IMAGE} (is docker available?)" unless
+      sh("docker run -d --name #{NAME} -p 8013:8013 -p 8014:8014 -p 8080:8080 #{IMAGE}")
+
+    # launchpad starts flagd with the default flag set
+    wait_for { http_code(:post, "http://localhost:8080/start") }
+    wait_for { http_code(:get, "http://localhost:8014/healthz") == "200" } ||
+      raise("flagd testbed did not become healthy in time")
+  end
+
+  def stop
+    sh("docker rm -f #{NAME}")
+  end
+
+  def sh(cmd)
+    system(cmd, out: File::NULL, err: File::NULL)
+  end
+
+  def http_code(verb, url)
+    uri = URI(url)
+    res = if verb == :post
+      Net::HTTP.post(uri, "")
+    else
+      Net::HTTP.get_response(uri)
+    end
+    res.code
+  rescue
+    nil
+  end
+
+  def wait_for(timeout: 60)
+    deadline = Time.now + timeout
+    loop do
+      return true if yield
+      return false if Time.now > deadline
+
+      sleep 0.5
+    end
+  end
+end
+
+# register teardown before startup so a container is cleaned up even if start raises
+at_exit { Testbed.stop }
+Testbed.start

providers/openfeature-flagd-provider/openfeature-flagd-provider.gemspec

@@ -35,6 +35,8 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.12.0"
+  spec.add_development_dependency "cucumber", "~> 9.2"
+  spec.add_development_dependency "logger"
   spec.add_development_dependency "standard"
   spec.add_development_dependency "rubocop"
   spec.add_development_dependency "simplecov", "~> 0.22"