Fixes #226 and Fixes #230. IAM Definition refactor. Speed improvements

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.9.0 (Unreleased)
+* Speed improvements: The IAM definition is now a dictionary instead of a list.
+* Fixed issue where elasticloadbalancing v1 was showing up in query results but v2 was not. Fixes #226
+* Supports "required" as an additional key for privilege (Fixes #230)
+
 ## 0.8.8 (2020-09-15)
 * Fixes issue with querying condition keys (#225)
 * Adds get_region_from_arn back for our friends at Netflix :)

policy_sentry/querying/actions.py

@@ -378,22 +378,24 @@ def remove_actions_that_are_not_wildcard_arn_only(actions_list):
     return results
 
 
-def get_privilege_info(service, action):
+def get_privilege_info(service_prefix, action):
     """
     Given a service, like `s3` and an action name, like `ListBucket`, return info about that action.
 
     Arguments:
-        service: The service prefix, like `s3`
+        service_prefix: The service prefix, like `s3`
         action: An action name, like `ListBucket`
 
     Returns:
         List: The info from the docs about that action, along with some of the info from the docs
     """
-    for service_info in iam_definition:
-        if service_info["prefix"] == service:
-            for privilege_info in service_info["privileges"]:
-                if privilege_info["privilege"] == action:
-                    privilege_info["service_resources"] = service_info["resources"]
-                    privilege_info["service_conditions"] = service_info["conditions"]
-                    return privilege_info
-    raise Exception("Unknown action {}:{}".format(service, action))
+    if service_prefix in iam_definition:
+        for privilege_info in iam_definition[service_prefix]["privileges"]:
+            if privilege_info["privilege"] == action:
+                privilege_info["service_resources"] = iam_definition[service_prefix]["resources"]
+                privilege_info["service_conditions"] = iam_definition[service_prefix]["conditions"]
+                return privilege_info
+    # for service_info in iam_definition:
+    #     if service_info["prefix"] == service_prefix:
+    # If it is not found at all, raise an exception
+    raise Exception("Unknown action {}:{}".format(service_prefix, action))

policy_sentry/querying/all.py

@@ -1,7 +1,7 @@
 """IAM Database queries that are not specific to either the Actions, ARNs, or Condition Keys tables."""
 import logging
 import functools
-from policy_sentry.shared.iam_data import iam_definition
+from policy_sentry.shared.iam_data import iam_definition, get_service_prefix_data
 
 logger = logging.getLogger(__name__)
 
@@ -17,8 +17,8 @@ def get_all_service_prefixes():
     Returns:
         List: A list of all AWS service prefixes present in the table.
     """
-    results = [d["prefix"] for d in iam_definition]
-    results = list(set(results))
+    # results = [d["prefix"] for d in iam_definition]
+    results = list(set(iam_definition.keys()))
     results.sort()
     return results
 
@@ -34,15 +34,17 @@ def get_all_actions(lowercase=False):
     """
     all_actions = set()
 
-    for service_info in iam_definition:
-        for privilege_info in service_info["privileges"]:
+    all_service_prefixes = get_all_service_prefixes()
+    for service_prefix in all_service_prefixes:
+        service_prefix_data = get_service_prefix_data(service_prefix)
+        for privilege_info in service_prefix_data["privileges"]:
             if lowercase:
                 all_actions.add(
-                    f"{service_info['prefix']}:{privilege_info['privilege'].lower()}"
+                    f"{service_prefix_data['prefix']}:{privilege_info['privilege'].lower()}"
                 )
             else:
                 all_actions.add(
-                    f"{service_info['prefix']}:{privilege_info['privilege']}"
+                    f"{service_prefix_data['prefix']}:{privilege_info['privilege']}"
                 )
 
     # results = list(set(results))

policy_sentry/querying/arns.py

@@ -4,7 +4,7 @@
 """
 import logging
 import functools
-from policy_sentry.shared.iam_data import iam_definition, get_service_prefix_data
+from policy_sentry.shared.iam_data import get_service_prefix_data
 from policy_sentry.util.arns import does_arn_match, get_service_from_arn
 
 logger = logging.getLogger(__name__)
@@ -21,16 +21,15 @@ def get_arn_data(service_prefix, resource_type_name):
         Dictionary: Metadata about an ARN type
     """
     results = []
-    for service_data in iam_definition:
-        if service_data["prefix"] == service_prefix:
-            for resource in service_data["resources"]:
-                if resource["resource"].lower() == resource_type_name.lower():
-                    output = {
-                        "resource_type_name": resource["resource"],
-                        "raw_arn": resource["arn"],
-                        "condition_keys": resource["condition_keys"],
-                    }
-                    results.append(output)
+    service_prefix_data = get_service_prefix_data(service_prefix)
+    for resource in service_prefix_data["resources"]:
+        if resource["resource"].lower() == resource_type_name.lower():
+            output = {
+                "resource_type_name": resource["resource"],
+                "raw_arn": resource["arn"],
+                "condition_keys": resource["condition_keys"],
+            }
+            results.append(output)
     return results
 
 
@@ -45,10 +44,9 @@ def get_raw_arns_for_service(service_prefix):
         List: A list of raw ARNs
     """
     results = []
-    for service_data in iam_definition:
-        if service_data["prefix"] == service_prefix:
-            for resource in service_data["resources"]:
-                results.append(resource["arn"])
+    service_prefix_data = get_service_prefix_data(service_prefix)
+    for resource in service_prefix_data["resources"]:
+        results.append(resource["arn"])
     return results
 
 
@@ -63,10 +61,9 @@ def get_arn_types_for_service(service_prefix):
         List: A list of ARN types, like `bucket` or `object`
     """
     results = {}
-    for service_data in iam_definition:
-        if service_data["prefix"] == service_prefix:
-            for resource in service_data["resources"]:
-                results[resource["resource"]] = resource["arn"]
+    service_prefix_data = get_service_prefix_data(service_prefix)
+    for resource in service_prefix_data["resources"]:
+        results[resource["resource"]] = resource["arn"]
     return results
 
 

policy_sentry/shared/awsdocs.py

@@ -148,7 +148,7 @@ def create_database(destination_directory, access_level_overrides_file):
         parents=True, exist_ok=True
     )
 
-    schema = []
+    schema = {}
 
     # for filename in ['list_amazonathena.partial.html']:
     file_list = []
@@ -171,8 +171,10 @@ def create_database(destination_directory, access_level_overrides_file):
             # Get service name
             title = main_content.find("h1", class_="topictitle").text
             title = re.sub(
-                ".*Actions, Resources, and Condition Keys for *", "", str(title)
+                ".*Actions, Resources, and Condition Keys for *", "", str(title),
+                flags=re.IGNORECASE
             )
+
             title = title.replace("</h1>", "")
             service_name = chomp(title)
 
@@ -290,9 +292,16 @@ def create_database(destination_directory, access_level_overrides_file):
                                     action_element
                                 ) in dependent_actions_element.find_all("p"):
                                     dependent_actions.append(chomp(action_element.text))
+                            if "*" in resource_type:
+                                required = True
+                                resource_type = resource_type.strip("*")
+                            else:
+                                required = False
+
                             resource_types.append(
                                 {
                                     "resource_type": resource_type,
+                                    "required": required,
                                     "condition_keys": condition_keys,
                                     "dependent_actions": dependent_actions,
                                 }
@@ -320,7 +329,7 @@ def create_database(destination_directory, access_level_overrides_file):
             # Get resource table
             for table in tables:
                 header_cells = [chomp(str(x)) for x in table.find_all("th")]
-                if "<th> Resource Types </th>" not in header_cells:
+                if "<th> Resource Types </th>".lower() not in (cell.lower() for cell in header_cells):
                     continue
 
                 rows = table.find_all("tr")
@@ -346,15 +355,19 @@ def create_database(destination_directory, access_level_overrides_file):
                         conditions.append(chomp(condition.text))
 
                     service_schema["resources"].append(
-                        {"resource": resource, "arn": arn, "condition_keys": conditions}
+                        {
+                            "resource": resource,
+                            "arn": arn,
+                            "condition_keys": conditions
+                        }
                     )
 
             # Get condition keys table
             for table in tables:
-                if "<th> Condition Keys </th>" not in [
-                    chomp(str(x)) for x in table.find_all("th")
-                ] or "<th> Type </th>" not in [
-                    chomp(str(x)) for x in table.find_all("th")
+                if "<th> Condition Keys </th>".lower() not in [
+                    chomp(str(x)).lower() for x in table.find_all("th")
+                ] or "<th> Type </th>".lower() not in [
+                    chomp(str(x)).lower() for x in table.find_all("th")
                 ]:
                     continue
 
@@ -384,9 +397,13 @@ def create_database(destination_directory, access_level_overrides_file):
                             "type": value_type,
                         }
                     )
-            schema.append(service_schema)
+            this_service_schema = {
+                service_prefix: service_schema
+            }
+            schema.update(this_service_schema)
+            # schema.append(service_schema)
 
-    schema.sort(key=lambda x: x["prefix"])
+    # schema.sort(key=lambda x: x["prefix"])
     iam_definition_file = os.path.join(destination_directory, "iam-definition.json")
     with open(iam_definition_file, "w") as file:
         json.dump(schema, file, indent=4)