Improve validator

Author: JoshuaMart

config/settings.yml

@@ -11,6 +11,10 @@ http:
   proxy: null  # Optional: "http://proxy.example.com:8080"
   timeout: 30  # seconds
 
+# Validation settings
+validation:
+  allow_private_suffixes: false  # https://github.com/weppos/publicsuffix-ruby/blob/main/data/list.txt
+
 # API server configuration
 api:
   port: 4567

lib/scopes_extractor/config.rb

@@ -80,6 +80,14 @@ def history_retention_days
         load[:history_retention_days] || 30
       end
 
+      def validation
+        load[:validation] || {}
+      end
+
+      def allow_private_suffixes?
+        validation[:allow_private_suffixes] == true
+      end
+
       def discord
         load[:discord] || {}
       end

lib/scopes_extractor/validator.rb

@@ -70,7 +70,7 @@ def self.valid_wildcard_usage?(val)
 
       # 5. Domain part must be a valid registrable domain (not just a public suffix like *.co.uk)
       begin
-        parsed = PublicSuffix.parse(domain_part)
+        parsed = PublicSuffix.parse(domain_part, ignore_private: Config.allow_private_suffixes?)
         return false if parsed.sld.nil?
       rescue PublicSuffix::DomainInvalid, PublicSuffix::DomainNotAllowed
         return false

spec/scopes_extractor/validator_spec.rb

@@ -59,6 +59,27 @@
         expect(described_class.valid_web_target?('*.test.com.au', 'web')).to be true
       end
 
+      it 'rejects wildcards on private suffixes by default' do
+        expect(described_class.valid_web_target?('*.digitaloceanspaces.com', 'web')).to be false
+        expect(described_class.valid_web_target?('*.githubusercontent.com', 'web')).to be false
+      end
+
+      context 'when allow_private_suffixes is enabled' do
+        before do
+          allow(ScopesExtractor::Config).to receive(:allow_private_suffixes?).and_return(true)
+        end
+
+        it 'accepts wildcards on private suffixes' do
+          expect(described_class.valid_web_target?('*.digitaloceanspaces.com', 'web')).to be true
+          expect(described_class.valid_web_target?('*.githubusercontent.com', 'web')).to be true
+        end
+
+        it 'still rejects public suffixes' do
+          expect(described_class.valid_web_target?('*.co.uk', 'web')).to be false
+          expect(described_class.valid_web_target?('*.com.au', 'web')).to be false
+        end
+      end
+
       it 'rejects descriptions in parentheses' do
         expect(described_class.valid_web_target?('*.example.com (description text)', 'web')).to be false
       end