Handle invalid domains more gracefully

src/findcdn/cdnEngine/detectCDN/cdn_check.py

@@ -19,35 +19,15 @@
 # Internal Libraries
 from .cdn_config import COMMON, CDNs, CDNs_rev
 from .cdn_err import NoIPaddress
+from findcdn.findcdn_err import InvalidDomain
+from .domain import Domain
+import validators
+
 
 # Global variables
 LIFETIME = 10
 
 
-class Domain:
-    """Domain class allows for storage of metadata on domain."""
-
-    def __init__(
-        self,
-        url: str,
-        ip: List[str] = [],
-        cnames: List[str] = [],
-        cdns: List[str] = [],
-        cdns_by_name: List[str] = [],
-        namsrvs: List[str] = [],
-        headers: List[str] = [],
-        whois_data: List[str] = [],
-    ):
-        """Initialize object to store metadata on domain in url."""
-        self.url = url
-        self.ip = ip
-        self.cnames = cnames
-        self.cdns = cdns
-        self.cdns_by_name = cdns_by_name
-        self.namesrvs = namsrvs
-        self.headers = headers
-        self.whois_data = whois_data
-        self.cdn_present = False
 
 
 class cdnCheck:
@@ -259,6 +239,15 @@ def all_checks(
         interactive: bool = False,
     ) -> int:
         """Option to run everything in this library then digest."""
+
+
+        # check if domain is valid before scanning. this should not happen
+        # since main checks for invalid domains, but a sanity check is good
+        # as we will fail if we get a bad domain at this point
+        if validators.domain(dom.url) is not True:
+            raise InvalidDomain(dom.url)
+
+
         # Obtain each attributes data
         self.ip(dom)
         self.cname(dom, timeout)

src/findcdn/cdnEngine/detectCDN/domain.py

@@ -0,0 +1,26 @@
+from typing import List
+class Domain:
+    """Domain class allows for storage of metadata on domain."""
+
+    def __init__(
+        self,
+        url: str,
+        ip: List[str] = [],
+        cnames: List[str] = [],
+        cdns: List[str] = [],
+        cdns_by_name: List[str] = [],
+        namsrvs: List[str] = [],
+        headers: List[str] = [],
+        whois_data: List[str] = [],
+    ):
+        """Initialize object to store metadata on domain in url."""
+        self.url = url
+        self.ip = ip
+        self.cnames = cnames
+        self.cdns = cdns
+        self.cdns_by_name = cdns_by_name
+        self.namesrvs = namsrvs
+        self.headers = headers
+        self.whois_data = whois_data        
+        self.errors = []
+        self.cdn_present = False

src/findcdn/findcdn.py

@@ -78,22 +78,33 @@ def main(
     if len(domain_list) <= 0:
         raise NoDomains("error")
 
+    invalid_domains = []
+    valid_domains = []
     # Validate domains in list
     for item in domain_list:
         if validators.domain(item) is not True:
-            raise InvalidDomain(item)
-
-    # Show the validated domains if in verbose mode
-    if verbose:
-        print("%d Domains Validated" % len(domain_list))
+            #add item to skipped domains for later return to user
+            invalid_domains.append(item)            
+            #remove this list from our main list to check
+        else:
+            valid_domains.append(item)            
+
+
+
+    print(f"{len(valid_domains)} Domains Validated.")
+    if(len(invalid_domains) > 0):
+        print(f"{len(invalid_domains)} domain(s) rejected, listed below: ")
+        for dom in invalid_domains:
+            print(f"\t{dom}")
+
 
     # Define domain dict and counter for json
     domain_dict = {}
     CDN_count = 0
 
     # Check domain list
     processed_list, cnt = run_checks(
-        domain_list,
+        valid_domains,
         threads,
         timeout,
         user_agent,
@@ -121,6 +132,7 @@ def main(
     json_dict["date"] = datetime.datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
     json_dict["cdn_count"] = str(CDN_count)
     json_dict["domains"] = domain_dict  # type: ignore
+    json_dict["invalid_domains"] = invalid_domains
     json_dump = json.dumps(json_dict, indent=4, sort_keys=False)
 
     # Show the dump to stdout if verbose or interactive
@@ -230,10 +242,7 @@ def interactive() -> None:
         sys.exit(1)
     except FileWriteError as fwe:
         print(fwe.message)
-        sys.exit(2)
-    except InvalidDomain as invdom:
-        print(invdom.message)
-        sys.exit(3)
+        sys.exit(2)    
     except NoDomains as nd:
         print(nd.message)
         sys.exit(4)

src/findcdn/findcdn_err.py

@@ -21,11 +21,11 @@ def __init__(self, outFile):
 
 
 class InvalidDomain(Exception):
-    """Raise when an invalid domain is inputted in findcnd.main()."""
+    """Raise when an invalid domain is inputted."""
 
     def __init__(self, item):
         """Instantiate super class with passed message with passed in item."""
-        self.message = item + " is not a valid domain in findcdn.main()"
+        self.message = item + " is not a valid domain."
         super().__init__(self.message)
 
 

tests/test_detectCDN.py

@@ -1,11 +1,15 @@
 #!/usr/bin/env pytest -vs
 """Tests for detectCDN."""
-
+import json, pytest
 # Third-Party Libraries
 import dns.resolver
 
+
+
 # cisagov Libraries
+from findcdn import main
 from findcdn.cdnEngine.detectCDN import Domain, cdnCheck
+from findcdn.findcdn_err import InvalidDomain
 
 # Globals
 TIMEOUT = 10
@@ -138,6 +142,42 @@ def test_broken_whois():
     assert return_code != 0, "This fake site should return a non 0 code."
 
 
+def test_cdncheck_invalid_domain():
+    """Invidual domain check should throw an InvalidDomain error"""
+    dom_in = Domain("*.foo.bar",list(), list(), list(), list(), list(), list(), list())
+    check = cdnCheck()
+    with pytest.raises(InvalidDomain):
+        check.all_checks(dom_in, timeout=TIMEOUT, agent=USER_AGENT)
+
+def test_cdncheck_valid_domain():
+    """Invidual domain check should NOT throw an InvalidDomain error"""
+    dom_in = Domain("cisa.gov",list(), list(), list(), list(), list(), list(), list())
+    check = cdnCheck()    
+
+
+def test_main_catches_invalid():
+    """main() should catch and flag invalid domains in output, and NOT throw an exception"""
+    domain_list = ["*.foo.bar"]
+    json_dump = main(domain_list=domain_list)
+    assert json_dump is not None
+
+    json_obj = json.loads(json_dump)
+    assert "invalid_domains" in json_obj
+    assert len(json_obj["invalid_domains"]) > 0
+    assert json_obj["invalid_domains"][0] == "*.foo.bar"
+
+def test_main_with_valid_domain_shows_no_invalid():
+    """main() should catch and flag invalid domains in output, and NOT throw an exception"""
+    domain_list = ["cisa.gov"]
+    json_dump = main(domain_list=domain_list)
+    assert json_dump is not None
+
+    json_obj = json.loads(json_dump)
+    assert "invalid_domains" in json_obj
+    assert len(json_obj["invalid_domains"]) == 0
+
+
+
 def test_all_checks():
     """Run all checks."""
     dom_in = Domain("login.gov", list(), list(), list(), list(), list(), list(), list())