Merge pull request MikeMeliz#13 from the-siegfried/update-docstrings

update docstrings

Author: MikeMeliz

modules/checker.py

@@ -11,11 +11,11 @@
 
 
 def url_canon(website, verbose):
-    """
+    """ URL normalisation/canonicalization
 
-    :param website: String -
-    :param verbose: Boolean -
-    :return: String 'website' -
+    :param website: String - URL of website.
+    :param verbose: Boolean - Verbose logging switch.
+    :return: String 'website' - normalised result.
     """
     if not website.startswith("http"):
         if not website.startswith("www."):
@@ -29,17 +29,18 @@ def url_canon(website, verbose):
 
 
 def extract_domain(url, remove_http=True):
-    """
+    """ Parses the provided 'url' to provide only the netloc or
+    scheme + netloc parts of the provided url.
 
-    :param url: String -
-    :param remove_http: Boolean -
-    :return: String 'domain_name' -
+    :param url: String - Url to parse.
+    :param remove_http: Boolean
+    :return: String 'domain_name' - Resulting parsed Url
     """
     uri = urlparse(url)
     if remove_http:
         domain_name = f"{uri.netloc}"
     else:
-        domain_name = f"{uri.netloc}://{uri.netloc}"
+        domain_name = f"{uri.scheme}://{uri.netloc}"
     return domain_name
 
 

modules/crawler.py

@@ -22,7 +22,7 @@ def __init__(self, website, c_depth, c_pause, out_path, logs, verbose):
     def excludes(self, link):
         """ Excludes links that are not required.
 
-        :param link:
+        :param link: String
         :return: Boolean
         """
         # BUG: For NoneType Exceptions, got to find a solution here
@@ -57,8 +57,8 @@ def excludes(self, link):
     def canonical(self, link):
         """ Canonization of the link.
 
-        :param link:
-        :return:
+        :param link: String
+        :return: String 'final_link': parsed canonical url.
         """
         # Already formatted
         if link.startswith(self.website):

torcrawl.py

@@ -176,7 +176,7 @@ def main():
     website = ''
     out_path = ''
 
-    # Canon/ion of website and create path for output
+    # Canonicalization of web url and create path for output.
     if len(args.url) > 0:
         website = url_canon(args.url, args.verbose)
         if args.folder is not None: