Merge pull request #30 from LegionIO/fix/vault-lease-token-cascade-revocation

fix vault lease cascade revocation — handle non-renewable tokens and reissue on reauth

Author: Esity

CHANGELOG.md

@@ -2,6 +2,19 @@
 
 ## [Unreleased]
 
+## [1.5.9] - 2026-04-10
+
+### Fixed
+- Vault lease cascade revocation: all three service credentials (RabbitMQ, PostgreSQL, Redis) died at exactly 2 hours when the Vault Kerberos auth token expired — Vault cascade-revokes all child leases when the parent token dies, regardless of individual lease TTLs (closes #29)
+- `TokenRenewer` now detects non-renewable tokens (`renewable=false`) and skips `renew_self` (which always fails for non-renewable tokens), going straight to `reauth_kerberos` before the token expires
+- `TokenRenewer#reauth_kerberos` now triggers `LeaseManager.reissue_all` after obtaining a new token, re-issuing all active leases under the new token so they are not orphaned when the old token expires
+- `LeaseManager#push_to_settings` symbol/string key mismatch: `resolve_secrets!` registers refs with string keys (`"rabbitmq"`) via `lease://` URI parsing, but `cache_lease` stores leases with symbol keys (`:rabbitmq` from `Legion::JSON.load`) — now tries both key types
+- `LeaseManager#trigger_reconnect` for `:postgresql` — uses surgical Sequel pool `disconnect` + `test_connection` instead of `Data.shutdown + Data.setup` which tore down unrelated connections (Apollo SQLite, Local cache)
+- `LeaseManager#trigger_reconnect` for `:redis` — uses `Cache.restart` (the actual method) instead of `Cache.reconnect` (which does not exist)
+
+### Added
+- `LeaseManager#reissue_all` — re-issues all active leases under the current vault client token; called by `TokenRenewer` after successful Kerberos re-authentication to prevent cascade revocation of orphaned leases
+
 ## [1.5.8] - 2026-04-09
 
 ### Added

lib/legion/crypt/lease_manager.rb

@@ -86,7 +86,9 @@ def register_ref(name, key, path)
 
       def push_to_settings(name)
         refs, data = @state_mutex.synchronize do
-          [@refs[name]&.dup, @lease_cache[name]&.dup]
+          r = @refs[name] || @refs[name.to_s] || @refs[name.to_sym]
+          d = @lease_cache[name] || @lease_cache[name.to_s] || @lease_cache[name.to_sym]
+          [r&.dup, d&.dup]
         end
         return if refs.nil? || refs.empty?
         return unless data
@@ -109,6 +111,19 @@ def vault_sys
         sys
       end
 
+      def reissue_all
+        log.info('LeaseManager: reissue_all — re-issuing all active leases under new token')
+        lease_names = @state_mutex.synchronize { @active_leases.keys.dup }
+
+        lease_names.each do |name|
+          lease = @state_mutex.synchronize { @active_leases[name]&.dup }
+          next unless lease && lease[:path]
+
+          reissue_lease(name)
+        end
+        log.info('LeaseManager: reissue_all complete')
+      end
+
       def register_dynamic_lease(name:, path:, response:, settings_refs:)
         register_at_exit_hook
 
@@ -451,14 +466,19 @@ def trigger_reconnect(name)
           Legion::Transport::Connection.force_reconnect
           log.info("LeaseManager: triggered transport reconnect after '#{name}' reissue")
         when :postgresql
-          return unless defined?(Legion::Data) && Legion::Data.respond_to?(:reconnect)
+          return unless defined?(Legion::Data::Connection) && Legion::Data::Connection.sequel
 
-          Legion::Data.reconnect
-          log.info("LeaseManager: triggered data reconnect after '#{name}' reissue")
+          Legion::Data::Connection.sequel.disconnect
+          Legion::Data::Connection.sequel.test_connection
+          log.info("LeaseManager: triggered data pool reconnect after '#{name}' reissue")
         when :redis
-          return unless defined?(Legion::Cache) && Legion::Cache.respond_to?(:reconnect)
+          return unless defined?(Legion::Cache)
 
-          Legion::Cache.reconnect
+          if Legion::Cache.respond_to?(:restart)
+            Legion::Cache.restart
+          elsif Legion::Cache.respond_to?(:reconnect)
+            Legion::Cache.reconnect
+          end
           log.info("LeaseManager: triggered cache reconnect after '#{name}' reissue")
         end
       rescue StandardError => e

lib/legion/crypt/token_renewer.rb

@@ -72,7 +72,9 @@ def reauth_kerberos
         @config[:renewable]      = result[:renewable]
         @config[:connected]      = true
         @vault_client.token      = result[:token]
-        log.info("TokenRenewer[#{@cluster_name}]: re-authenticated via Kerberos")
+        log.info("TokenRenewer[#{@cluster_name}]: re-authenticated via Kerberos, ttl=#{result[:lease_duration]}s")
+
+        reissue_all_leases
         true
       rescue StandardError => e
         handle_exception(e, level: :warn, operation: 'crypt.token_renewer.reauth_kerberos', cluster_name: @cluster_name)
@@ -104,10 +106,18 @@ def renewal_loop
         interruptible_sleep(sleep_duration)
 
         until @stop
-          if renew_token || reauth_kerberos
-            on_renewal_success
+          if @config[:renewable]
+            if renew_token || reauth_kerberos
+              on_renewal_success
+            else
+              on_renewal_failure
+            end
           else
-            on_renewal_failure
+            if reauth_kerberos # rubocop:disable Style/IfInsideElse
+              on_renewal_success
+            else
+              on_renewal_failure
+            end
           end
         end
       rescue StandardError => e
@@ -128,6 +138,15 @@ def on_renewal_failure
         interruptible_sleep(delay)
       end
 
+      def reissue_all_leases
+        return unless defined?(Legion::Crypt::LeaseManager)
+
+        Legion::Crypt::LeaseManager.instance.reissue_all
+      rescue StandardError => e
+        handle_exception(e, level: :warn, operation: 'crypt.token_renewer.reissue_all_leases', cluster_name: @cluster_name)
+        log.warn("TokenRenewer[#{@cluster_name}]: failed to reissue leases after reauth: #{e.message}")
+      end
+
       def interruptible_sleep(seconds)
         deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + seconds
         loop do
@@ -150,7 +169,7 @@ def stop_thread_and_revoke
         else
           @thread = nil
           revoke_token
-          log.debug("TokenRenewer[#{@cluster_name}]: token renewal thread stopped")
+          log.info("TokenRenewer[#{@cluster_name}]: token renewal thread stopped")
         end
       end
 

lib/legion/crypt/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Crypt
-    VERSION = '1.5.8'
+    VERSION = '1.5.9'
   end
 end

spec/legion/lease_manager_spec.rb

@@ -611,29 +611,28 @@
     end
 
     context 'when name is :postgresql' do
-      it 'calls Data.reconnect when available' do
-        data_mod = double('Legion::Data')
-        stub_const('Legion::Data', data_mod)
-        allow(data_mod).to receive(:respond_to?).with(:reconnect).and_return(true)
-        expect(data_mod).to receive(:reconnect)
+      it 'disconnects and reconnects the Sequel pool' do
+        sequel_db = double('Sequel::Database')
+        connection_mod = double('Legion::Data::Connection', sequel: sequel_db)
+        stub_const('Legion::Data::Connection', connection_mod)
+        expect(sequel_db).to receive(:disconnect)
+        expect(sequel_db).to receive(:test_connection)
         manager.send(:trigger_reconnect, :postgresql)
       end
 
-      it 'skips when Data does not respond to reconnect' do
-        data_mod = double('Legion::Data')
-        stub_const('Legion::Data', data_mod)
-        allow(data_mod).to receive(:respond_to?).with(:reconnect).and_return(false)
-        expect(data_mod).not_to receive(:reconnect)
-        manager.send(:trigger_reconnect, :postgresql)
+      it 'skips when Data::Connection.sequel is nil' do
+        connection_mod = double('Legion::Data::Connection', sequel: nil)
+        stub_const('Legion::Data::Connection', connection_mod)
+        expect { manager.send(:trigger_reconnect, :postgresql) }.not_to raise_error
       end
     end
 
     context 'when name is :redis' do
-      it 'calls Cache.reconnect when available' do
+      it 'calls Cache.restart when available' do
         cache_mod = double('Legion::Cache')
         stub_const('Legion::Cache', cache_mod)
-        allow(cache_mod).to receive(:respond_to?).with(:reconnect).and_return(true)
-        expect(cache_mod).to receive(:reconnect)
+        allow(cache_mod).to receive(:respond_to?).with(:restart).and_return(true)
+        expect(cache_mod).to receive(:restart)
         manager.send(:trigger_reconnect, :redis)
       end
     end