add pre-commit and pyupgrade to CI

Author: gruebel

.github/workflows/bump-version.yml

@@ -24,7 +24,7 @@ jobs:
           new_tag=$(echo $latest_tag | awk -F. -v a="$1" -v b="$2" -v c="$3" '{printf("%d.%d.%d", $1+a, $2+b , $3+1)}')
           echo "new tag: $new_tag"
 
-          printf "# pylint: disable=missing-module-docstring\n__version__ = '$new_tag'""" > $version_file
+          printf "# pylint: disable=missing-module-docstring\n__version__ = \"$new_tag\"\n""" > $version_file
 
           git commit -m "Bump to ${new_tag}"  $version_file || echo "No changes to commit"
           git push origin

.github/workflows/ci.yml

@@ -13,7 +13,17 @@ permissions:
   contents: read
 
 jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac  # v4.0.0
+      - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1  # v4.7.0
+        with:
+          python-version: '3.8' # needed for 'pyupgrade'
+      - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507  # v3.0.0
+
   ci:
+    needs: pre-commit
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
@@ -42,6 +52,7 @@ jobs:
       - run: invoke build.uninstall-package
 
   python-version:
+    needs: pre-commit
     if: github.event_name == 'pull_request'
     runs-on: ubuntu-latest
     timeout-minutes: 15

.pre-commit-config.yaml

@@ -1,6 +1,12 @@
-- repo: git://github.com/antonbabenko/pre-commit-terraform
-  rev: v1.44.0 # Get the latest from: https://github.com/antonbabenko/pre-commit-terraform/releases
-  hooks:
-    - id: terraform_fmt
-    - id: terraform_docs
-#      args: ['--sort-by-required', '--no-providers']
+repos:
+  - repo: https://github.com/antonbabenko/pre-commit-terraform
+    rev: v1.83.3
+    hooks:
+      - id: terraform_fmt
+#      - id: terraform_docs
+#        args: ['--sort-by-required', '--no-providers']
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.11.0
+    hooks:
+    - id: pyupgrade
+      args: ["--py37-plus"]

.pylintrc

@@ -89,6 +89,7 @@ disable=raw-checker-failed,
         f-string-without-interpolation,
         logging-fstring-interpolation,
         unused-variable,
+        broad-exception-raised,
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

policy_sentry/bin/version.py

@@ -1,2 +1,2 @@
 # pylint: disable=missing-module-docstring
-__version__ = '0.12.10'
+__version__ = "0.12.10"

policy_sentry/command/create_template.py

@@ -14,7 +14,7 @@
 
 
 @click.command(
-    context_settings=dict(max_content_width=160),
+    context_settings={"max_content_width": 160},
     short_help="Create write-policy YML template files",
 )
 @click.option(

policy_sentry/command/query.py

@@ -157,7 +157,8 @@ def query_action_table(
             print_dict(output=output, fmt=fmt)
         else:
             print("All services in the database:\n")
-            output = all_services  # type:ignore[assignment]  # it is a set here, which is ok
+            # it is a set here, which is ok
+            output = all_services  # type:ignore[assignment]
             print_list(output=output, fmt=fmt)
     elif name is None and access_level and not resource_type:
         print(
@@ -250,7 +251,9 @@ def arn_table(
     query_arn_table(name, service, list_arn_types, fmt)
 
 
-def query_arn_table(name: str, service: str, list_arn_types: bool, fmt: str) -> list[str] | dict[str, str]:
+def query_arn_table(
+    name: str, service: str, list_arn_types: bool, fmt: str
+) -> list[str] | dict[str, str]:
     """Query the ARN Table from the Policy Sentry database. Use this one when leveraging Policy Sentry as a library."""
     if os.path.exists(LOCAL_DATASTORE_FILE_PATH):
         logger.info(
@@ -309,7 +312,9 @@ def condition_table(name: str, service: str, fmt: str, verbose: str | None) -> N
     query_condition_table(name, service, fmt)
 
 
-def query_condition_table(name: str, service: str, fmt: str = "json") -> list[str] | dict[str, str]:
+def query_condition_table(
+    name: str, service: str, fmt: str = "json"
+) -> list[str] | dict[str, str]:
     """Query the condition table from the Policy Sentry database.
     Use this one when leveraging Policy Sentry as a library."""
     if os.path.exists(LOCAL_DATASTORE_FILE_PATH):

policy_sentry/querying/conditions.py

@@ -26,7 +26,7 @@ def get_condition_keys_for_service(service_prefix: str) -> list[str]:
         List: A list of condition keys
     """
     service_prefix_data = get_service_prefix_data(service_prefix)
-    results = [condition for condition in service_prefix_data["conditions"]]
+    results = list(service_prefix_data["conditions"])
     return results
 
 

policy_sentry/shared/awsdocs.py

@@ -183,7 +183,7 @@ def create_database(
         if not filename.startswith("list_"):
             continue
 
-        with open(os.path.join(BUNDLED_HTML_DIRECTORY_PATH, filename), "r") as f:
+        with open(os.path.join(BUNDLED_HTML_DIRECTORY_PATH, filename)) as f:
             soup = BeautifulSoup(f.read(), "html.parser")
             main_content = soup.find(id="main-content")
             if not isinstance(main_content, Tag):
@@ -357,7 +357,7 @@ def create_database(
                             cells = row.find_all("td")
 
                     if "[permission only]" in priv:
-                        priv = priv.split(" ")[0]
+                        priv = priv.split(" ", maxsplit=1)[0]
 
                     privilege_schema = {
                         "privilege": priv,

policy_sentry/shared/constants.py

@@ -29,8 +29,10 @@
 # Check for the existence of the local datastore first.
 if os.path.exists(LOCAL_DATASTORE_FILE_PATH):
     # If it exists, leverage that datastore instead of the one bundled with the python package
-    logger.info(f"Leveraging the local IAM definition at the path: {LOCAL_DATASTORE_FILE_PATH} "
-                f"To leverage the bundled definition instead, remove the folder $HOME/.policy_sentry/")
+    logger.info(
+        f"Leveraging the local IAM definition at the path: {LOCAL_DATASTORE_FILE_PATH} "
+        f"To leverage the bundled definition instead, remove the folder $HOME/.policy_sentry/"
+    )
     DATASTORE_FILE_PATH = LOCAL_DATASTORE_FILE_PATH
 else:
     # Otherwise, leverage the datastore inside the python package
@@ -39,7 +41,7 @@
 
 # Overrides
 if "CUSTOM_ACCESS_OVERRIDES_FILE" in os.environ:
-    CUSTOM_ACCESS_OVERRIDES_FILE=os.environ['CUSTOM_ACCESS_OVERRIDES_FILE']
+    CUSTOM_ACCESS_OVERRIDES_FILE = os.environ["CUSTOM_ACCESS_OVERRIDES_FILE"]
     BUNDLED_ACCESS_OVERRIDES_FILE = os.path.join(
         os.path.abspath(os.path.dirname(__file__)), CUSTOM_ACCESS_OVERRIDES_FILE
     )

policy_sentry/shared/iam_data.py

@@ -22,6 +22,10 @@
 
 @functools.lru_cache(maxsize=1)
 def get_iam_definition_schema_version() -> str:
+    """
+    Returns the schema version of the IAM datastore
+    """
+
     return cast(
         "str",
         iam_definition.get(

policy_sentry/util/file.py

@@ -19,7 +19,7 @@ def read_yaml_file(filename: str | Path) -> dict[str, Any]:
     :param filename: name of the yaml file
     :return: dictionary of YAML file contents
     """
-    with open(filename, "r") as yaml_file:
+    with open(filename) as yaml_file:
         try:
             cfg = cast("dict[str, Any]", yaml.safe_load(yaml_file))
         except yaml.YAMLError as exc:

requirements-dev.txt

@@ -1,5 +1,5 @@
 # CI
-pre-commit==3.4.0
+pre-commit==2.21.0
 # Unit Testing
 pytest==7.4.0
 pylint==2.17.5

terraform_module/ps-template/main.tf

@@ -34,8 +34,8 @@ resource "local_file" "template" {
 }
 
 resource "random_string" "random_template_name" {
-  length           = 24
-  special          = false
+  length  = 24
+  special = false
 }
 
 data "external" "policy" {

test/writing/test_sid_group_crud.py

@@ -16,7 +16,7 @@
     )
 )
 
-with open(crud_with_override_template, "r") as yaml_file:
+with open(crud_with_override_template) as yaml_file:
     crud_with_override_template_cfg = yaml.safe_load(yaml_file)
 
 
@@ -446,7 +446,7 @@ def test_exclude_actions_from_crud_output(self):
                 "crud-with-exclude-actions.yml",
             )
         )
-        with open(crud_with_exclude_actions, "r") as this_yaml_file:
+        with open(crud_with_exclude_actions) as this_yaml_file:
             crud_with_exclude_actions_cfg = yaml.safe_load(this_yaml_file)
         sid_group.process_template(crud_with_exclude_actions_cfg)
         result = sid_group.get_rendered_policy(crud_with_exclude_actions_cfg)
@@ -557,7 +557,7 @@ def test_exclude_actions_empty_sid_from_crud_output(self):
             )
         )
 
-        with open(crud_with_exclude_actions_empty_sid, "r") as this_yaml_file:
+        with open(crud_with_exclude_actions_empty_sid) as this_yaml_file:
             crud_with_exclude_actions_empty_sid_cfg = yaml.safe_load(this_yaml_file)
         # crud_with_exclude_actions_empty_sid_cfg = {
         #     "mode": "crud",