Merge pull request #492 from target/ScanURLValidation

Adding Validation to ScanURL

Author: skalupa

src/python/strelka/scanners/scan_url.py

@@ -1,5 +1,7 @@
 import re
 
+import validators
+
 from strelka import strelka
 
 
@@ -52,9 +54,19 @@ def scan(self, data, file, options, expire_at):
             urls = set(url_regex.findall(normalized_data))
             for url in urls:
                 # Strip leading and trailing punctuation characters from the URL.
-                clean_url = url.strip(b"!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~").decode()
-                if clean_url not in self.event["urls"]:
-                    self.event["urls"].append(clean_url)
+                strip_trailing_url = url.strip(
+                    b"!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
+                ).decode()
+
+                # Check to see if there are nonURL chars stil in URL:
+                nonurl_regex_pattern = r'[\^&\(\)+\[\]{}\|"]'
+                split_uls = re.split(nonurl_regex_pattern, strip_trailing_url)
+                for split_result in split_uls:
+                    if (
+                        validators.url(split_result)
+                        and split_result not in self.event["urls"]
+                    ):
+                        self.event["urls"].append(split_result)
 
         except Exception as e:
             self.flags.append(f"scanner_error: {e}")

src/python/strelka/tests/test_scan_url.py

@@ -18,7 +18,6 @@ def test_scan_url_text(mocker):
         "flags": [],
         "urls": unordered(
             [
-                "example.com",
                 "http://foobar.example.com",
                 "https://barfoo.example.com",
                 "ftp://barfoo.example.com",