diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 000000000..ab3e3a8b1 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,50 @@ +codecov: + notify: + after_n_builds: 21 # Number of test matrix+lint jobs uploading coverage + wait_for_ci: false + + require_ci_to_pass: false + # notsecret # repo-scoped, upload-only, stability in fork PRs + token: >- + 7316089b-55fe-4646-b640-78d84b79d109 + +comment: + require_changes: true + +coverage: + range: 100..100 + status: + patch: + default: + target: 100% + pytest: + target: 100% + flags: + - pytest + typing: + flags: + - MyPy + project: + default: + target: 95% + lib: + flags: + - pytest + paths: + - src/ + target: 100% + tests: + flags: + - pytest + paths: + - tests/ + target: 100% + typing: + flags: + - MyPy + target: 90% + +github_checks: + # Annotations are deprecated in Codecov because they are misleading. + # Ref: https://github.com/codecov/codecov-action/issues/1710 + annotations: false diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 000000000..28d9564e4 --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,19 @@ +# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json +reviews: + # Collapse main AI comment by default, as it takes too much space when + # expanded. It also is needless for subsequent rounds of PR review, mostly + # only for the first one + collapse_walkthrough: true + # Move AI-generated summary from PR description to main AI comment. It + # hallucinates sometimes, especially with PRs that change code linting rules + high_level_summary_in_walkthrough: true + # Disable false-positive cross links to issues + related_issues: false + # Disable false-positive cross links to PRs + related_prs: false + # Disable useless Poem generation + poem: false + + auto_review: + # Enable AI review for Draft PRs + drafts: true diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..ffab40193 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,34 @@ +[html] +show_contexts = true +skip_covered = false + +[paths] +_site-packages-to-src-mapping = + src + */src + *\src + */lib/pypy*/site-packages + */lib/python*/site-packages + *\Lib\site-packages + +[report] +skip_covered = true +skip_empty = true +show_missing = true +exclude_also = + ^\s*@pytest\.mark\.xfail + ^\s*\.\.\.\s*(#.*)?$ + +[run] +branch = true +cover_pylib = false +# https://coverage.rtfd.io/en/latest/contexts.html#dynamic-contexts +# dynamic_context = test_function # conflicts with `pytest-cov` if set here +parallel = true +plugins = + covdefaults +relative_files = true +source = + . +source_pkgs = + pre_commit_terraform diff --git a/.dockerignore b/.dockerignore index a78e675d2..e57569568 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,4 @@ !.dockerignore !Dockerfile !tools/entrypoint.sh +!tools/install/*.sh diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..98e9a7dbc --- /dev/null +++ b/.flake8 @@ -0,0 +1,108 @@ +[flake8] + +# Print the total number of errors: +count = true + +# Don't even try to analyze these: +extend-exclude = + # GitHub configs + .github, + # Cache files of MyPy + .mypy_cache, + # Cache files of pytest + .pytest_cache, + # Countless third-party libs in venvs + .tox, + # Occasional virtualenv dir + .venv, + # VS Code + .vscode, + # Metadata of `pip wheel` cmd is autogenerated + pip-wheel-metadata, + +# IMPORTANT: avoid using ignore option, always use extend-ignore instead +# Completely and unconditionally ignore the following errors: +extend-ignore = + # Legitimate cases, no need to "fix" these violations: + # D202: No blank lines allowed after function docstring, conflicts with `ruff format` + D202, + # E203: whitespace before ':', conflicts with `ruff format` + E203, + # E501: "line too long", its function is replaced by `flake8-length` + E501, + # W505: "doc line too long", its function is replaced by `flake8-length` + W505, + # I: flake8-isort is drunk + we have isort integrated into pre-commit + I, + # WPS305: "Found f string" -- nothing bad about this + WPS305, + # WPS322: "Found incorrect multi-line string" -- false-positives with + # attribute docstrings. Ref: + # https://github.com/wemake-services/wemake-python-styleguide/issues/3056 + WPS322, + # WPS326: "Found implicit string concatenation" -- nothing bad about this + WPS326, + # WPS428: "Found statement that has no effect" -- false-positives with + # attribute docstrings. Ref: + # https://github.com/wemake-services/wemake-python-styleguide/issues/3056 + WPS428, + # WPS462: "Wrong multiline string usage" -- false-positives with + # attribute docstrings. Ref: + # https://github.com/wemake-services/wemake-python-styleguide/issues/3056 + WPS462, + # WPS300: "Forbid imports relative to the current folder" -- we use relative imports + WPS300, + +# https://wemake-python-styleguide.readthedocs.io/en/latest/pages/usage/formatter.html +format = wemake + +# Let's not overcomplicate the code: +max-complexity = 10 + +# Accessibility/large fonts and PEP8 friendly. +# This is being flexibly extended through the `flake8-length`: +max-line-length = 79 + +# Allow certain violations in certain files: +# Please keep both sections of this list sorted, as it will be easier for others to find and add entries in the future +per-file-ignores = + # The following ignores have been researched and should be considered permanent + # each should be preceded with an explanation of each of the error codes + # If other ignores are added for a specific file in the section following this, + # these will need to be added to that line as well. + + tests/pytest/_cli_test.py: + # WPS431: "Forbid nested classes" -- this is a legitimate use case for tests + WPS431, + # WPS226: "Forbid the overuse of string literals" -- this is a legitimate use case for tests + WPS226, + # WPS115: "Require snake_case for naming class attributes" -- testing legitimate case, ignored in main code + WPS115, + # We will not spend time on fixing complexity in deprecated hook + src/pre_commit_terraform/terraform_docs_replace.py: WPS232 + +# Count the number of occurrences of each error/warning code and print a report: +statistics = true + +# ## Plugin-provided settings: ## + +# flake8-eradicate +# E800: +eradicate-whitelist-extend = isort:\s+\w+ + +# flake8-pytest-style +# PT001: +pytest-fixture-no-parentheses = true +# PT006: +pytest-parametrize-names-type = tuple +# PT007: +pytest-parametrize-values-type = tuple +pytest-parametrize-values-row-type = tuple +# PT023: +pytest-mark-no-parentheses = true + +# wemake-python-styleguide +# WPS410: "Forbid some module-level variables" -- __all__ is a legitimate use case +allowed-module-metadata = __all__ + +show-source = true diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 000000000..db412838b --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,70 @@ +# `git blame` master ignore list. +# +# This file contains a list of git hashes of revisions to be ignored +# by `git blame`. These revisions are considered "unimportant" in +# that they are unlikely to be what you are interested in when blaming. +# They are typically expected to be formatting-only changes. +# +# It can be used for `git blame` using `--ignore-revs-file` or by +# setting `blame.ignoreRevsFile` in the `git config`[1]. +# +# Ignore these commits when reporting with blame. Calling +# +# git blame --ignore-revs-file .git-blame-ignore-revs +# +# will tell `git blame` to ignore changes made by these revisions when +# assigning blame, as if the change never happened. +# +# You can enable this as a default for your local repository by +# running +# +# git config blame.ignoreRevsFile .git-blame-ignore-revs +# +# This will probably be automatically picked by your IDE +# (VSCode+GitLens and JetBrains products are confirmed to do this). +# +# Important: if you are switching to a branch without this file, +# `git blame` will fail with an error. +# +# GitHub also excludes the commits listed below from its "Blame" +# views[2][3]. +# +# [1]: https://git-scm.com/docs/git-blame#Documentation/git-blame.txt-blameignoreRevsFile +# [2]: https://github.blog/changelog/2022-03-24-ignore-commits-in-the-blame-view-beta/ +# [3]: https://docs.github.com/en/repositories/working-with-files/using-files/viewing-a-file#ignore-commits-in-the-blame-view +# +# Guidelines: +# - Only large (generally automated) reformatting or renaming PRs +# should be added to this list. Do not put things here just because +# you feel they are trivial or unimportant. If in doubt, do not put +# it on this list. +# - When adding a single revision, use inline comment to link relevant +# issue/PR. Alternatively, paste the commit title instead. +# Example: +# d4a8b7307acc2dc8a8833ccfa65426ad28b3ffc9 # https://github.com/sanitizers/octomachinery/issues/1 +# - When adding multiple revisions (like a bulk of work over many +# commits), organize them in blocks. Precede each such block with a +# comment starting with the word "START", followed by a link to the +# relevant issue or PR. Add a similar comment after the last block +# line but use the word "END", followed by the same link. +# Alternatively, add or augment the link with a text motivation and +# description of work performed in each commit. +# After each individual commit in the block, add an inline comment +# with the commit title line. +# Example: +# # START https://github.com/sanitizers/octomachinery/issues/1 +# 6f0bd2d8a1e6cd2e794cd39976e9756e0c85ac66 # Bulk-replace smile emojis with unicorns +# d53974df11dbc22cbea9dc7dcbc9896c25979a27 # Replace double with single quotes +# ... +# # END https://github.com/sanitizers/octomachinery/issues/1 +# - Only put full 40-character hashes on this list (not short hashes +# or any other revision reference). +# - Append to the bottom of the file, regardless of the chronological +# order of the revisions. Revisions within blocks should be in +# chronological order from oldest to newest. +# - Because you must use a hash, you need to append to this list in a +# follow-up PR to the actual reformatting PR that you are trying to +# ignore. This approach helps avoid issues with arbitrary rebases +# and squashes while the pull request is in progress. + +23928fbf8511697c915c3231977ee254bd3fa0c2 # chore(linters): Apply ruff-format diff --git a/.git_archival.txt b/.git_archival.txt new file mode 100644 index 000000000..3e26627d4 --- /dev/null +++ b/.git_archival.txt @@ -0,0 +1,3 @@ +node: $Format:%H$ +node-date: $Format:%cI$ +describe-name: $Format:%(describe:tags=true)$ diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..00a7b00c9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +.git_archival.txt export-subst diff --git a/.github/.container-structure-test-config.yaml b/.github/.container-structure-test-config.yaml index 60107655e..0425a71eb 100644 --- a/.github/.container-structure-test-config.yaml +++ b/.github/.container-structure-test-config.yaml @@ -1,91 +1,142 @@ -schemaVersion: '2.0.0' +schemaVersion: 2.0.0 + commandTests: - - name: "git" - command: "git" - args: ["--version"] - expectedOutput: ["^git version 2\\.[0-9]+\\.[0-9]+\\n$"] - - - name: "pre-commit" - command: "pre-commit" - args: ["-V"] - expectedOutput: ["^pre-commit ([0-9]+\\.){2}[0-9]+\\n$"] - - - name: "terraform" - command: "terraform" - args: ["-version"] - expectedOutput: ["^Terraform v([0-9]+\\.){2}[0-9]+\\non linux_amd64\\n$"] - - - name: "checkov" - command: "checkov" - args: ["--version"] - expectedOutput: ["^([0-9]+\\.){2}[0-9]+\\n$"] - - - name: "infracost" - command: "infracost" - args: ["--version"] - expectedOutput: ["^Infracost v([0-9]+\\.){2}[0-9]+\\n$"] - - - name: "terraform-docs" - command: "terraform-docs" - args: ["--version"] - expectedOutput: ["^terraform-docs version v([0-9]+\\.){2}[0-9]+ [a-z0-9]+ linux/amd64\\n$"] - - - name: "terragrunt" - command: "terragrunt" - args: ["--version"] - expectedOutput: ["^terragrunt version v([0-9]+\\.){2}[0-9]+\\n$"] - - - name: "terrascan" - command: "terrascan" - args: [ "version" ] - expectedOutput: [ "^version: v([0-9]+\\.){2}[0-9]+\\n$" ] - - - name: "tflint" - command: "tflint" - args: [ "--version" ] - expectedOutput: [ "TFLint version ([0-9]+\\.){2}[0-9]+\\n" ] - - - name: "tfsec" - command: "tfsec" - args: [ "--version" ] - expectedOutput: [ "([0-9]+\\.){2}[0-9]+\\n$" ] - - - name: "trivy" - command: "trivy" - args: [ "--version" ] - expectedOutput: [ "Version: ([0-9]+\\.){2}[0-9]+\\n" ] - - - name: "tfupdate" - command: "tfupdate" - args: [ "--version" ] - expectedOutput: [ "([0-9]+\\.){2}[0-9]+\\n$" ] - - - name: "hcledit" - command: "hcledit" - args: [ "version" ] - expectedOutput: [ "([0-9]+\\.){2}[0-9]+\\n$" ] - - - name: "entrypoint.sh" - envVars: - - key: "USERID" - value: "1000:1000" - command: "/entrypoint.sh" - args: [ "-V" ] - expectedError: ["^ERROR: uid:gid 1000:1000 lacks permissions to //\\n$"] - exitCode: 1 - - - name: "su-exec" - command: "su-exec" - expectedOutput: ["^Usage: su-exec user-spec command \\[args\\]\\n$"] - - - name: "ssh" - command: "ssh" - args: [ "-V" ] - expectedError: ["^OpenSSH_9\\.[0-9]+"] +- name: git + command: git + args: + - --version + expectedOutput: + - ^git version 2\.[0-9]+\.[0-9]+\n$ + +- name: pre-commit + command: pre-commit + args: + - -V + expectedOutput: + - ^pre-commit ([0-9]+\.){2}[0-9]+\n$ + +- name: gcc + command: gcc + args: + - --version + expectedOutput: + - ^gcc \(Alpine 12\. + +- name: checkov + command: checkov + args: + - --version + expectedOutput: + - ^([0-9]+\.){2}[0-9]+\n$ + +- name: infracost + command: infracost + args: + - --version + expectedOutput: + - ^Infracost v([0-9]+\.){2}[0-9]+ + +- name: opentofu + command: tofu + args: + - --version + expectedOutput: + - ^OpenTofu v([0-9]+\.){2}[0-9]+\n + +- name: terraform + command: terraform + args: + - --version + expectedOutput: + - ^Terraform v([0-9]+\.){2}[0-9]+\n + +- name: terraform-docs + command: terraform-docs + args: + - --version + expectedOutput: + - ^terraform-docs version v([0-9]+\.){2}[0-9]+ [a-z0-9]+ + +- name: terragrunt + command: terragrunt + args: + - --version + expectedOutput: + - ^terragrunt version v([0-9]+\.){2}[0-9]+\n$ + +- name: terrascan + command: terrascan + args: + - version + expectedOutput: + - >- + ^version: v([0-9]+\.){2}[0-9]+\n$ + +- name: tflint + command: tflint + args: + - --version + expectedOutput: + - TFLint version ([0-9]+\.){2}[0-9]+\n + +- name: tfsec + command: tfsec + args: + - --version + expectedOutput: + - ([0-9]+\.){2}[0-9]+\n$ + +- name: trivy + command: trivy + args: + - --version + expectedOutput: + - >- + Version: ([0-9]+\.){2}[0-9]+\n + +- name: tfupdate + command: tfupdate + args: + - --version + expectedOutput: + - ([0-9]+\.){2}[0-9]+\n$ + +- name: hcledit + command: hcledit + args: + - version + expectedOutput: + - ([0-9]+\.){2}[0-9]+\n$ + +- name: entrypoint.sh + envVars: + - key: USERID + value: 1000:1000 + command: /entrypoint.sh + args: + - -V + expectedError: + - >- + ^ERROR: uid:gid 1000:1000 lacks permissions to //\n$ + exitCode: 1 + +- name: su-exec + command: su-exec + expectedOutput: + - >- + ^Usage: su-exec user-spec command \[args\]\n$ + +- name: ssh + command: ssh + args: + - -V + expectedError: + - ^OpenSSH_9\.[0-9]+ fileExistenceTests: - - name: 'terrascan init' - path: '/root/.terrascan/pkg/policies/opa/rego/github/github_repository/privateRepoEnabled.rego' - shouldExist: true - uid: 0 - gid: 0 +- name: terrascan init + path: >- + /root/.terrascan/pkg/policies/opa/rego/github/github_repository/privateRepoEnabled.rego + shouldExist: true + uid: 0 + gid: 0 diff --git a/.github/.dive-ci.yaml b/.github/.dive-ci.yaml index 3c526cd5e..84e22312a 100644 --- a/.github/.dive-ci.yaml +++ b/.github/.dive-ci.yaml @@ -3,11 +3,13 @@ rules: # Expressed as a ratio between 0-1. lowestEfficiency: 0.981 - # If the amount of wasted space is at least X or larger than X, mark as failed. + # If the amount of wasted space is at least X or larger than X, mark + # as failed. # Expressed in B, KB, MB, and GB. highestWastedBytes: 32MB - # If the amount of wasted space makes up for X% or more of the image, mark as failed. + # If the amount of wasted space makes up for X% or more of the image, + # mark as failed. # Note: the base image layer is NOT included in the total image size. # Expressed as a ratio between 0-1; fails if the threshold is met or crossed. highestUserWastedPercent: 0.036 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e8de3562f..c58606bd4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @maxymvlasov @yermulnik +* @maxymvlasov @yermulnik @antonbabenko diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 388bf3751..e40a40a1f 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,22 +1,24 @@ # Notes for contributors -1. Python hooks are supported now too. All you have to do is: - 1. add a line to the `console_scripts` array in `entry_points` in `setup.py` - 2. Put your python script in the `pre_commit_hooks` folder - -Enjoy the clean, valid, and documented code! - +* [Configure `git blame` to ignore formatting commits](#configure-git-blame-to-ignore-formatting-commits) * [Run and debug hooks locally](#run-and-debug-hooks-locally) * [Run hook performance test](#run-hook-performance-test) * [Run via BASH](#run-via-bash) * [Run via Docker](#run-via-docker) * [Check results](#check-results) * [Cleanup](#cleanup) +* [Required tools and plugins to simplify review process](#required-tools-and-plugins-to-simplify-review-process) * [Add new hook](#add-new-hook) * [Before write code](#before-write-code) * [Prepare basic documentation](#prepare-basic-documentation) * [Add code](#add-code) * [Finish with the documentation](#finish-with-the-documentation) +* [Contributing to Python code](#contributing-to-python-code) +* [Run tests in your fork](#run-tests-in-your-fork) + +## Configure `git blame` to ignore formatting commits + +This project uses `.git-blame-ignore-revs` to exclude formatting-related commits from `git blame` history. To configure your local `git blame` to ignore these commits, refer to the [.git-blame-ignore-revs](/.git-blame-ignore-revs) file for details. ## Run and debug hooks locally @@ -98,6 +100,13 @@ Results will be located at `./test/results` dir. sudo rm -rf tests/results ``` +## Required tools and plugins to simplify review process + +1. [editorconfig.org](https://editorconfig.org/) (preinstalled in some IDE) +2. [pre-commit](https://pre-commit.com/#install) +3. (Optional) If you use VS Code - feel free to install all recommended extensions + + ## Add new hook You can use [this PR](https://github.com/antonbabenko/pre-commit-terraform/pull/252) as an example. @@ -106,6 +115,8 @@ You can use [this PR](https://github.com/antonbabenko/pre-commit-terraform/pull/ 1. Try to figure out future hook usage. 2. Confirm the concept with [Anton Babenko](https://github.com/antonbabenko). +3. Install [required tools and plugins](#required-tools-and-plugins-to-simplify-review-process) + ### Prepare basic documentation @@ -113,6 +124,9 @@ You can use [this PR](https://github.com/antonbabenko/pre-commit-terraform/pull/ ### Add code +> [!TIP] +> Here is a screencast of [how to add new dependency in `tools/install/`](https://github.com/antonbabenko/pre-commit-terraform/assets/11096782/8fc461e9-f163-4592-9497-4a18fa89c0e8) - used in Dockerfile + 1. Based on prev. block, add hook dependencies installation to [Dockerfile](../Dockerfile). Check that works: * `docker build -t pre-commit --build-arg INSTALL_ALL=true .` @@ -139,5 +153,46 @@ You can use [this PR](https://github.com/antonbabenko/pre-commit-terraform/pull/ ### Finish with the documentation -1. Add hook description to [Available Hooks](../README.md#available-hooks). +1. Add the hook description to [Available Hooks](../README.md#available-hooks). 2. Create and populate a new hook section in [Hooks usage notes and examples](../README.md#hooks-usage-notes-and-examples). + +## Contributing to Python code + +1. [Install `tox`](https://tox.wiki/en/stable/installation.html) +2. To run tests, run: + + ```bash + tox -qq + ``` + + The easiest way to find out what parts of the code base are left uncovered, is to copy-paste and run the `python3 ...` command that will open the HTML report, so you can inspect it visually. + +3. Before committing any changes (if you do not have `pre-commit` installed locally), run: + + ```bash + tox r -qq -e pre-commit + ``` + + Make sure that all checks pass. + +4. (Optional): If you want to limit the checks to MyPy only, you can run: + + ```bash + tox r -qq -e pre-commit -- mypy --all-files + ``` + + Then copy-paste and run the `python3 ...` commands to inspect the strictest MyPy coverage reports visually. + +5. (Optional): You can find all available `tox` environments by running: + + ```bash + tox list + ``` + +## Run tests in your fork + +Go to your fork's `Actions` tab and click the big green button. + +![Enable workflows](/assets/contributing/enable_actions_in_fork.png) + +Now you can verify that the tests pass before submitting your PR. diff --git a/.github/ISSUE_TEMPLATE/bug_report_docker.md b/.github/ISSUE_TEMPLATE/bug_report_docker.md index a47e30657..0847388be 100644 --- a/.github/ISSUE_TEMPLATE/bug_report_docker.md +++ b/.github/ISSUE_TEMPLATE/bug_report_docker.md @@ -74,7 +74,7 @@ INSERT_OUTPUT_HERE
file content -```bash +```yaml INSERT_FILE_CONTENT_HERE ``` diff --git a/.github/ISSUE_TEMPLATE/bug_report_local_install.md b/.github/ISSUE_TEMPLATE/bug_report_local_install.md index 63a1ce915..3bef0dee0 100644 --- a/.github/ISSUE_TEMPLATE/bug_report_local_install.md +++ b/.github/ISSUE_TEMPLATE/bug_report_local_install.md @@ -78,17 +78,20 @@ Linux DESKTOP-C7315EF 5.4.72-microsoft-standard-WSL2 #1 SMP Wed Oct 28 23:40:43 bash << EOF bash --version | head -n 1 2>/dev/null || echo "bash SKIPPED" pre-commit --version 2>/dev/null || echo "pre-commit SKIPPED" +tofu --version | head -n 1 2>/dev/null || echo "opentofu SKIPPED" terraform --version | head -n 1 2>/dev/null || echo "terraform SKIPPED" python --version 2>/dev/null || echo "python SKIPPED" python3 --version 2>/dev/null || echo "python3 SKIPPED" -echo -n "checkov " && checkov --version 2>/dev/null || echo "checkov SKIPPED" +echo -n "checkov " && checkov --version 2>/dev/null || echo "SKIPPED" +infracost --version 2>/dev/null || echo "infracost SKIPPED" terraform-docs --version 2>/dev/null || echo "terraform-docs SKIPPED" terragrunt --version 2>/dev/null || echo "terragrunt SKIPPED" -echo -n "terrascan " && terrascan version 2>/dev/null || echo "terrascan SKIPPED" +echo -n "terrascan " && terrascan version 2>/dev/null || echo "SKIPPED" tflint --version 2>/dev/null || echo "tflint SKIPPED" -echo -n "tfsec " && tfsec --version 2>/dev/null || echo "tfsec SKIPPED" -echo -n "tfupdate " && tfupdate --version 2>/dev/null || echo "tfupdate SKIPPED" -echo -n "hcledit " && hcledit version 2>/dev/null || echo "hcledit SKIPPED" +echo -n "tfsec " && tfsec --version 2>/dev/null || echo "SKIPPED" +echo -n "trivy " && trivy --version 2>/dev/null || echo "SKIPPED" +echo -n "tfupdate " && tfupdate --version 2>/dev/null || echo "SKIPPED" +echo -n "hcledit " && hcledit version 2>/dev/null || echo "SKIPPED" EOF --> @@ -102,7 +105,7 @@ INSERT_TOOLS_VERSIONS_HERE
file content -```bash +```yaml INSERT_FILE_CONTENT_HERE ``` diff --git a/.github/SECURITY.md b/.github/SECURITY.md new file mode 100644 index 000000000..61c87d55d --- /dev/null +++ b/.github/SECURITY.md @@ -0,0 +1,22 @@ +# Reporting a Vulnerability + +If you believe you have discovered a potential security vulnerability in this project, please report it securely. **Do not create a public GitHub issue for any security concerns.** + +## How to Report + +Send an email with a detailed description of the vulnerability, including any evidence of the disclosure, the impact, and any timelines related to the issue to: [anton@antonbabenko.com](mailto:anton@antonbabenko.com) + +## Vulnerability Disclosure Process + +- **Confidential Disclosure:** All vulnerability reports will be kept confidential until a fix is developed and verified. +- **Assessment and Response:** We aim to acknowledge any valid report within 15 business days. +- **Timelines:** After verification, we plan to have a coordinated disclosure within 60 days, though this may vary depending on the complexity of the fix. +- **Communication:** We will work directly with the vulnerability reporter to clarify details, answer questions, and discuss potential mitigations. +- **Updates:** We may provide periodic updates on the progress of the remediation of the reported vulnerability. + +## Guidelines + +- **Vulnerability Definition:** A vulnerability is any flaw or weakness in this project that can be exploited to compromise system security. +- **Disclosure Expectations:** When you report a vulnerability, please include as much detail as possible to allow us to assess its validity and scope without exposing sensitive information publicly. + +By following this process, you help us improve the security of our project while protecting users and maintainers. We appreciate your efforts to responsibly disclose vulnerabilities. diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 632adfd30..c3cbb66ff 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -1,4 +1,21 @@ { $schema: "https://docs.renovatebot.com/renovate-schema.json", - extends: ["local>SpotOnInc/renovate-config"], + extends: [ + "local>SpotOnInc/renovate-config", + // Automerge patch and minor upgrades if they pass tests. | https://docs.renovatebot.com/presets-default/#automergeminor + ":automergeMinor", + // Require all status checks to pass before any automerging. | https://docs.renovatebot.com/presets-default/#automergerequireallstatuschecks + ":automergeRequireAllStatusChecks", + // Automerge digest upgrades if they pass tests. | https://docs.renovatebot.com/presets-default/#automergedigest + ":automergeDigest", + // Raise a PR first before any automerging. | https://docs.renovatebot.com/presets-default/#automergepr + ":automergePr", + ], + // To make happy 'Validate PR title' GHA + commitMessageLowerCase: "never", + // Disable auto-rebase on every commit to avoid reaching Github limits on macos runners + rebaseWhen: "conflicted", + "pre-commit": { + enabled: false, // Use pre-commit.ci freeze instead + }, } diff --git a/.github/workflows/build-image-test.yaml b/.github/workflows/build-image-test.yaml index f83e11ebf..9c6065d49 100644 --- a/.github/workflows/build-image-test.yaml +++ b/.github/workflows/build-image-test.yaml @@ -1,77 +1,128 @@ -name: "Build Dockerfile if changed and run smoke tests" +name: Build Dockerfile if changed and run smoke tests -on: [pull_request] +on: + merge_group: + pull_request: + +permissions: + contents: read env: IMAGE_TAG: pr-test jobs: build: - runs-on: ubuntu-latest + permissions: + # for MaxymVlasov/dive-action to write comments to PRs + pull-requests: write + + strategy: + matrix: + arch: + - amd64 + - arm64 + include: + - os-name: Ubuntu x64 + os: ubuntu-latest + arch: amd64 + + - os-name: Ubuntu ARM + os: ubuntu-24.04-arm + arch: arm64 + + name: ${{ matrix.os-name }} + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Get changed Docker related files + id: changed-files-specific + uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5 + with: + files: | + .dockerignore + .github/workflows/build-image-test.yaml + Dockerfile + tools/entrypoint.sh + tools/install/*.sh + + - name: Set IMAGE environment variable + if: steps.changed-files-specific.outputs.any_changed == 'true' + # Lowercase the org/repo name to allow for workflow to run in forks, + # which owners have uppercase letters in username + run: >- + echo "IMAGE=ghcr.io/${GITHUB_REPOSITORY@L}:${{ env.IMAGE_TAG }}" + >> $GITHUB_ENV - - name: Get changed Dockerfile - id: changed-files-specific - uses: tj-actions/changed-files@2c85495a7bb72f2734cb5181e29b2ee5e08e61f7 # v13.1 - with: - files: | - Dockerfile - .dockerignore - tools/entrypoint.sh - .github/workflows/build-image-test.yaml + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + if: steps.changed-files-specific.outputs.any_changed == 'true' - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + - name: Build if Dockerfile changed + if: steps.changed-files-specific.outputs.any_changed == 'true' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + build-args: | + INSTALL_ALL=true + push: false + load: true + tags: ${{ env.IMAGE }} + # Fix multi-platform: https://github.com/docker/buildx/issues/1533 + provenance: false + secrets: | + "github_token=${{ secrets.GITHUB_TOKEN }}" - - name: Build if Dockerfile changed - if: steps.changed-files-specific.outputs.any_changed == 'true' - uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 - with: - context: . - build-args: | - INSTALL_ALL=true - platforms: linux/amd64 # Only one allowed here, see https://github.com/docker/buildx/issues/59#issuecomment-1433097926 - push: false - load: true - tags: | - ghcr.io/${{ github.repository }}:${{ env.IMAGE_TAG }} - # Fix multi-platform: https://github.com/docker/buildx/issues/1533 - provenance: false - secrets: | - "github_token=${{ secrets.GITHUB_TOKEN }}" + - name: Setup Container Structure Tests + if: steps.changed-files-specific.outputs.any_changed == 'true' + env: + # yamllint disable-line rule:line-length + # renovate: datasource=github-releases depName=container-structure-test lookupName=GoogleContainerTools/container-structure-test + CST_VERSION: 1.19.3 + CST_REPO: github.com/GoogleContainerTools/container-structure-test + run: >- + curl -L "https://${{ env.CST_REPO }}/releases/download/v${{ + env.CST_VERSION }}/container-structure-test-linux-${{ matrix.arch }}" + > container-structure-test + && chmod +x container-structure-test + && mkdir -p $HOME/bin/ + && mv container-structure-test $HOME/bin/ + && echo $HOME/bin/ >> $GITHUB_PATH - - name: Run structure tests - if: steps.changed-files-specific.outputs.any_changed == 'true' - uses: plexsystems/container-structure-test-action@c0a028aa96e8e82ae35be556040340cbb3e280ca # v0.3.0 - with: - image: ghcr.io/${{ github.repository }}:${{ env.IMAGE_TAG }} - config: .github/.container-structure-test-config.yaml + - name: Run structure tests + if: steps.changed-files-specific.outputs.any_changed == 'true' + run: >- + container-structure-test test + --config ${{ github.workspace + }}/.github/.container-structure-test-config.yaml + --image ${{ env.IMAGE }} - - name: Dive - check image for waste files - if: steps.changed-files-specific.outputs.any_changed == 'true' - uses: MaxymVlasov/dive-action@0035999cae50d4ef657ac94be84f01812aa192a5 # v0.1.0 - with: - image: ghcr.io/${{ github.repository }}:${{ env.IMAGE_TAG }} - config-file: ${{ github.workspace }}/.github/.dive-ci.yaml - github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Dive - check image for waste files + if: steps.changed-files-specific.outputs.any_changed == 'true' + uses: MaxymVlasov/dive-action@43dafd0015826beaca5110157c9262c5dc10672a # v1.4.0 + with: + image: ${{ env.IMAGE }} + config-file: ${{ github.workspace }}/.github/.dive-ci.yaml + github-token: ${{ secrets.GITHUB_TOKEN }} - # Can't build both platforms and use --load at the same time - # https://github.com/docker/buildx/issues/59#issuecomment-1433097926 - - name: Build Multi-arch docker-image - if: steps.changed-files-specific.outputs.any_changed == 'true' - uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 - with: - context: . - build-args: | - INSTALL_ALL=true - platforms: linux/amd64,linux/arm64 - push: false - tags: | - ghcr.io/${{ github.repository }}:${{ env.IMAGE_TAG }} - # Fix multi-platform: https://github.com/docker/buildx/issues/1533 - provenance: false - secrets: | - "github_token=${{ secrets.GITHUB_TOKEN }}" + # Can't build both platforms and use --load at the same time + # https://github.com/docker/buildx/issues/59#issuecomment-1433097926 + - name: Build Multi-arch docker-image + if: >- + steps.changed-files-specific.outputs.any_changed == 'true' + && matrix.os == 'ubuntu-latest' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + build-args: | + INSTALL_ALL=true + platforms: linux/amd64,linux/arm64 + push: false + tags: ${{ env.IMAGE }} + # Fix multi-platform: https://github.com/docker/buildx/issues/1533 + provenance: false + secrets: | + "github_token=${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 3f79fb205..ae6e407d0 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -4,62 +4,82 @@ on: workflow_dispatch: release: types: - - created + - created schedule: - - cron: '00 00 * * *' + - cron: 00 00 * * * + +permissions: + contents: read jobs: docker: + permissions: + # for docker/build-push-action to publish docker image + packages: write + runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 - - name: Login to GitHub Container Registry - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Set tag for image - run: | - echo IMAGE_TAG=$([ ${{ github.ref_type }} == 'tag' ] && echo ${{ github.ref_name }} || echo 'latest') >> $GITHUB_ENV + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + - name: Login to GitHub Container Registry + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set tag for image + env: + REF_TYPE: ${{ github.ref_type }} + REF_NAME: ${{ github.ref_name }} + run: >- + echo IMAGE_TAG=$( + [ $REF_TYPE == 'tag' ] + && echo $REF_NAME + || echo 'latest' + ) >> $GITHUB_ENV - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + - name: Set IMAGE_REPO environment variable + # Lowercase the org/repo name to allow for workflow to run in forks, + # which owners have uppercase letters in username + run: >- + echo "IMAGE_REPO=ghcr.io/${GITHUB_REPOSITORY@L}" >> $GITHUB_ENV + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - - name: Build and Push release - if: github.event_name != 'schedule' - uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 - with: - context: . - build-args: | - INSTALL_ALL=true - platforms: linux/amd64,linux/arm64 - push: true - tags: | - ghcr.io/${{ github.repository }}:${{ env.IMAGE_TAG }} - ghcr.io/${{ github.repository }}:latest - # Fix multi-platform: https://github.com/docker/buildx/issues/1533 - provenance: false - secrets: | - "github_token=${{ secrets.GITHUB_TOKEN }}" + - name: Build and Push release + if: github.event_name != 'schedule' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + build-args: | + INSTALL_ALL=true + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ env.IMAGE_REPO }}:${{ env.IMAGE_TAG }} + ${{ env.IMAGE_REPO }}:latest + # Fix multi-platform: https://github.com/docker/buildx/issues/1533 + provenance: false + secrets: | + "github_token=${{ secrets.GITHUB_TOKEN }}" - - name: Build and Push nightly - if: github.event_name == 'schedule' - uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 - with: - context: . - build-args: | - INSTALL_ALL=true - platforms: linux/amd64,linux/arm64 - push: true - tags: | - ghcr.io/${{ github.repository }}:nightly - # Fix multi-platform: https://github.com/docker/buildx/issues/1533 - provenance: false - secrets: | - "github_token=${{ secrets.GITHUB_TOKEN }}" + - name: Build and Push nightly + if: github.event_name == 'schedule' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + build-args: | + INSTALL_ALL=true + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ env.IMAGE_REPO }}:nightly + # Fix multi-platform: https://github.com/docker/buildx/issues/1533 + provenance: false + secrets: | + "github_token=${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml new file mode 100644 index 000000000..872605850 --- /dev/null +++ b/.github/workflows/ci-cd.yml @@ -0,0 +1,541 @@ +name: โˆž + +on: + merge_group: + push: + branches-ignore: + - dependabot/** # Dependabot always creates PRs + - renovate/** # Our Renovate setup always creates PRs + - gh-readonly-queue/** # Temporary merge queue-related GH-made branches + - pre-commit-ci-update-config # pre-commit.ci always creates a PR + pull_request: + workflow_call: # a way to embed the main tests + secrets: + CODECOV_TOKEN: + required: true + +permissions: + contents: read + +concurrency: + group: >- + ${{ + github.workflow + }}-${{ + github.ref_type + }}-${{ + github.event.pull_request.number || github.sha + }} + cancel-in-progress: true + +env: + FORCE_COLOR: 1 # Request colored output from CLI tools supporting it + MYPY_FORCE_COLOR: 1 # MyPy's color enforcement + PIP_DISABLE_PIP_VERSION_CHECK: 1 # Hide "there's a newer pip" message + PIP_NO_PYTHON_VERSION_WARNING: 1 # Hide "this Python is deprecated" message + PIP_NO_WARN_SCRIPT_LOCATION: 1 # Hide "script dir is not in $PATH" message + PRE_COMMIT_COLOR: always + PROJECT_NAME: pre-commit-terraform + PY_COLORS: 1 # Recognized by the `py` package, dependency of `pytest` + PYTHONIOENCODING: utf-8 + PYTHONUTF8: 1 + TOX_PARALLEL_NO_SPINNER: 1 # Disable tox's parallel run spinner animation + # Make tox-wrapped tools see color requests + TOX_TESTENV_PASSENV: >- + FORCE_COLOR + MYPY_FORCE_COLOR + NO_COLOR + PIP_DISABLE_PIP_VERSION_CHECK + PIP_NO_PYTHON_VERSION_WARNING + PIP_NO_WARN_SCRIPT_LOCATION + PRE_COMMIT_COLOR + PY_COLORS + PYTEST_THEME + PYTEST_THEME_MODE + PYTHONIOENCODING + PYTHONLEGACYWINDOWSSTDIO + PYTHONUTF8 + UPSTREAM_REPOSITORY_ID: 69382485 # Repo ID of antonbabenko/pre-commit-terraform + +run-name: >- + ${{ + github.event_name == 'workflow_dispatch' + && format('๐Ÿ“ฆ Releasing v{0}...', github.event.inputs.release-version) + || '' + }} + ${{ + github.event.pull_request.number && '๐Ÿ”€ PR' || '' + }}${{ + !github.event.pull_request.number && '๐ŸŒฑ Commit' || '' + }} + ${{ github.event.pull_request.number || github.sha }} + triggered by: ${{ github.event_name }} of ${{ + github.ref + }} ${{ + github.ref_type + }} + (workflow run ID: ${{ + github.run_id + }}; number: ${{ + github.run_number + }}; attempt: ${{ + github.run_attempt + }}) + +jobs: + pre-setup: + name: โš™๏ธ Pre-set global build settings + + runs-on: ubuntu-latest + + timeout-minutes: 1 + + defaults: + run: + shell: python + + outputs: + # NOTE: These aren't env vars because the `${{ env }}` context is + # NOTE: inaccessible when passing inputs to reusable workflows. + dists-artifact-name: python-package-distributions + dist-version: ${{ steps.scm-version.outputs.dist-version }} + cache-key-files: >- + ${{ steps.calc-cache-key-files.outputs.files-hash-key }} + git-tag: ${{ steps.git-tag.outputs.tag }} + sdist-artifact-name: ${{ steps.artifact-name.outputs.sdist }} + wheel-artifact-name: ${{ steps.artifact-name.outputs.wheel }} + upstream-repository-id: ${{ env.UPSTREAM_REPOSITORY_ID }} + + steps: + - name: Switch to using Python 3.13 by default + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: 3.13 + - name: Check out src from Git + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + persist-credentials: false + - name: >- + Calculate Python interpreter version hash value + for use in the cache key + id: calc-cache-key-py + run: | + from hashlib import sha512 + from os import environ + from pathlib import Path + from sys import version + + FILE_APPEND_MODE = 'a' + + hash = sha512(version.encode()).hexdigest() + + with Path(environ['GITHUB_OUTPUT']).open( + mode=FILE_APPEND_MODE, + ) as outputs_file: + print(f'py-hash-key={hash}', file=outputs_file) + - name: >- + Calculate dependency files' combined hash value + for use in the cache key + id: calc-cache-key-files + run: | + from os import environ + from pathlib import Path + + FILE_APPEND_MODE = 'a' + + with Path(environ['GITHUB_OUTPUT']).open( + mode=FILE_APPEND_MODE, + ) as outputs_file: + print( + "files-hash-key=${{ + hashFiles( + 'tox.ini', + 'pyproject.toml', + '.pre-commit-config.yaml', + 'pytest.ini', + 'dependencies/**/*' + ) + }}", + file=outputs_file, + ) + - name: Get pip cache dir + id: pip-cache-dir + run: >- + echo "dir=$(python -m pip cache dir)" >> "${GITHUB_OUTPUT}" + shell: bash + - name: Set up pip cache + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: >- + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key }}-${{ + steps.calc-cache-key-files.outputs.files-hash-key }} + restore-keys: | + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key + }}- + ${{ runner.os }}-pip- + ${{ runner.os }}- + - name: Drop Git tags from HEAD for non-release requests + run: >- + git tag --points-at HEAD + | + xargs git tag --delete + shell: bash + - name: Set up versioning prerequisites + run: >- + python -m + pip install + --user + setuptools-scm~=8.2 + shell: bash + - name: Set the current dist version from Git + id: scm-version + run: | + from os import environ + from pathlib import Path + + import setuptools_scm + + FILE_APPEND_MODE = 'a' + + ver = setuptools_scm.get_version() + with Path(environ['GITHUB_OUTPUT']).open( + mode=FILE_APPEND_MODE, + ) as outputs_file: + print(f'dist-version={ver}', file=outputs_file) + print( + f'dist-version-for-filenames={ver.replace("+", "-")}', + file=outputs_file, + ) + - name: Set the target Git tag + id: git-tag + env: + DIST_VERSION: ${{ steps.scm-version.outputs.dist-version }} + run: | + from os import environ + from pathlib import Path + + FILE_APPEND_MODE = 'a' + + with Path(environ['GITHUB_OUTPUT']).open( + mode=FILE_APPEND_MODE, + ) as outputs_file: + print( + f"tag=v{environ['DIST_VERSION'].removeprefix('v')}", + file=outputs_file, + ) + - name: Set the expected dist artifact names + id: artifact-name + env: + DIST_VERSION: ${{ steps.scm-version.outputs.dist-version }} + run: | + from os import environ + from pathlib import Path + + FILE_APPEND_MODE = 'a' + + whl_file_prj_base_name = '${{ env.PROJECT_NAME }}'.replace('-', '_') + sdist_file_prj_base_name = whl_file_prj_base_name.replace('.', '_') + + with Path(environ['GITHUB_OUTPUT']).open( + mode=FILE_APPEND_MODE, + ) as outputs_file: + print( + f"sdist={sdist_file_prj_base_name !s}-{environ['DIST_VERSION']}.tar.gz", + file=outputs_file, + ) + print( + f"wheel={whl_file_prj_base_name !s}-{environ['DIST_VERSION']}-py3-none-any.whl", + file=outputs_file, + ) + + build: + name: ๐Ÿ“ฆ ${{ needs.pre-setup.outputs.git-tag }} + needs: + - pre-setup + # Prevent run 'push' events for the branches in upstream repository as it + # already covered by 'pull_request' event + if: >- + github.repository_id != needs.pre-setup.outputs.upstream-repository-id + || github.event_name != 'push' + || github.ref_name == github.event.repository.default_branch + + + runs-on: ubuntu-latest + + timeout-minutes: 2 + + env: + TOXENV: cleanup-dists,build-dists + SDIST_ARTIFACT_NAME: ${{ needs.pre-setup.outputs.sdist-artifact-name }} + WHEEL_ARTIFACT_NAME: ${{ needs.pre-setup.outputs.wheel-artifact-name }} + outputs: + dists-base64-hash: ${{ steps.dist-hashes.outputs.combined-hash }} + + steps: + - name: Switch to using Python 3.13 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: 3.13 + + - name: Grab the source from Git + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + persist-credentials: false + + - name: >- + Calculate Python interpreter version hash value + for use in the cache key + id: calc-cache-key-py + run: | + from hashlib import sha512 + from os import environ + from pathlib import Path + from sys import version + + FILE_APPEND_MODE = 'a' + + hash = sha512(version.encode()).hexdigest() + + with Path(environ['GITHUB_OUTPUT']).open( + mode=FILE_APPEND_MODE, + ) as outputs_file: + print(f'py-hash-key={hash}', file=outputs_file) + shell: python + - name: Get pip cache dir + id: pip-cache-dir + run: >- + echo "dir=$(python -m pip cache dir)" >> "${GITHUB_OUTPUT}" + - name: Set up pip cache + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: >- + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key }}-${{ + needs.pre-setup.outputs.cache-key-files }} + restore-keys: | + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key + }}- + ${{ runner.os }}-pip- + + - name: Install tox + run: >- + python -Im pip install tox + shell: bash # windows compat + + - name: Pre-populate the tox env + run: >- + python -m + tox + --parallel auto + --parallel-live + --skip-missing-interpreters false + --notest + + - name: Drop Git tags from HEAD for non-tag-create events + run: >- + git tag --points-at HEAD + | + xargs git tag --delete + shell: bash + + - name: Set static timestamp for dist build reproducibility + # ... from the last Git commit since it's immutable + run: >- + echo "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" + >> "${GITHUB_ENV}" + - name: Build dists + run: >- + python -m + tox + --parallel auto + --parallel-live + --skip-missing-interpreters false + --skip-pkg-install + --quiet + - name: Verify that the artifacts with expected names got created + run: >- + ls -1 "dist/${SDIST_ARTIFACT_NAME}" "dist/${WHEEL_ARTIFACT_NAME}" + - name: Generate dist hashes to be used for provenance + id: dist-hashes + run: >- + echo "combined-hash=$( + sha256sum "$SDIST_ARTIFACT_NAME" "$WHEEL_ARTIFACT_NAME" | base64 -w0 + )" >> $GITHUB_OUTPUT + working-directory: dist + - name: Store the distribution packages + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: >- + ${{ needs.pre-setup.outputs.dists-artifact-name }} + # NOTE: Exact expected file names are specified here + # NOTE: as a safety measure โ€” if anything weird ends + # NOTE: up being in this dir or not all dists will be + # NOTE: produced, this will fail the workflow. + path: | + dist/${{ needs.pre-setup.outputs.sdist-artifact-name }} + dist/${{ needs.pre-setup.outputs.wheel-artifact-name }} + retention-days: 30 + + lint: + name: ๐Ÿงน Linters${{ '' }} # Group jobs in sidebar # zizmor: ignore[obfuscation] + needs: + - build + - pre-setup # transitive, for accessing settings + strategy: + matrix: + runner-vm-os: + - ubuntu-latest + python-version: + - 3.13 + toxenv: + - pre-commit + - metadata-validation + environment-variables: + # `no-commit-to-branch` is skipped because it does not make sense + # in the CI, only locally. + # Ref: https://github.com/pre-commit/pre-commit-hooks/issues/1124 + # only affects pre-commit, set for all for simplicity: + - >- + SKIP= + hadolint, + no-commit-to-branch, + shfmt, + tox-run-posargs: + - '' + xfail: + - false + check-name: + - '' + fail-fast: false + uses: ./.github/workflows/reusable-tox.yml + with: + cache-key-files: >- + ${{ needs.pre-setup.outputs.cache-key-files }} + check-name: >- + ${{ matrix.check-name }} + dists-artifact-name: >- + ${{ needs.pre-setup.outputs.dists-artifact-name }} + environment-variables: >- + ${{ matrix.environment-variables }} + python-version: >- + ${{ matrix.python-version }} + runner-vm-os: >- + ${{ matrix.runner-vm-os }} + source-tarball-name: >- + ${{ needs.pre-setup.outputs.sdist-artifact-name }} + timeout-minutes: 3 + toxenv: >- + ${{ matrix.toxenv }} + tox-run-posargs: >- + ${{ matrix.tox-run-posargs }} + upstream-repository-id: >- + ${{ needs.pre-setup.outputs.upstream-repository-id }} + xfail: ${{ fromJSON(matrix.xfail) }} + secrets: + codecov-token: ${{ secrets.CODECOV_TOKEN }} + + tests: + name: ๐Ÿงช Tests${{ '' }} # Group jobs in sidebar # zizmor: ignore[obfuscation] + needs: + - build + - pre-setup # transitive, for accessing settings + strategy: + matrix: + python-version: + # NOTE: The latest and the lowest supported Pythons are prioritized + # NOTE: to improve the responsiveness. It's nice to see the most + # NOTE: important results first. + - 3.13 + - 3.9 + # str + - >- + 3.10 + - 3.12 + - 3.11 + runner-vm-os: + - ubuntu-24.04 + - macos-14 + - macos-13 + - windows-2025 + toxenv: + - pytest + xfail: + - false + + uses: ./.github/workflows/reusable-tox.yml + with: + built-wheel-names: >- + ${{ needs.pre-setup.outputs.wheel-artifact-name }} + cache-key-files: >- + ${{ needs.pre-setup.outputs.cache-key-files }} + dists-artifact-name: >- + ${{ needs.pre-setup.outputs.dists-artifact-name }} + python-version: >- + ${{ matrix.python-version }} + runner-vm-os: >- + ${{ matrix.runner-vm-os }} + source-tarball-name: >- + ${{ needs.pre-setup.outputs.sdist-artifact-name }} + timeout-minutes: 5 + toxenv: >- + ${{ matrix.toxenv }} + tox-run-posargs: >- + --cov-report=xml:.tox/.tmp/.test-results/pytest-${{ + matrix.python-version + }}/cobertura.xml + --junitxml=.tox/.tmp/.test-results/pytest-${{ + matrix.python-version + }}/test.xml + tox-rerun-posargs: >- + -rA + -vvvvv + --lf + --no-cov + --no-fold-skipped + upstream-repository-id: >- + ${{ needs.pre-setup.outputs.upstream-repository-id }} + xfail: ${{ fromJSON(matrix.xfail) }} + secrets: + codecov-token: ${{ secrets.CODECOV_TOKEN }} + + check: # This job does nothing and is only used for the branch protection + + # Separate 'pull_request' check from other checks to avoid confusion in + # GitHub branch protection about which check is required when multiple + # events trigger this workflow. + name: >- + ${{ github.event_name == 'push' && 'check​' || 'check' }} + if: always() + + needs: + - lint + - tests + + runs-on: ubuntu-latest + + timeout-minutes: 1 + + steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 + with: + jobs: ${{ toJSON(needs) }} + # Needed to not fail on skipped 'push' events for the branches in + # upstream repository as they already covered by 'pull_request' event + allowed-skips: >- + ${{ + ( + github.repository_id != needs.pre-setup.outputs.upstream-repository-id + || github.event_name != 'push' + || github.ref_name == github.event.repository.default_branch + ) + && 'lint, tests' + || '' + }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..fa105c1b8 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,81 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: CodeQL + +on: + push: + branches: + - master + merge_group: + pull_request: + schedule: + - cron: 0 0 * * 1 + +permissions: + contents: read + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: + - python + # CodeQL supports [ $supported-codeql-languages ] + # Learn more about CodeQL language support at + # https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in + # a config file. # By default, queries listed here will override any + # specified in a config file. Prefix the list here with "+" to use + # these queries and those in the config file. + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java) + # If this step fails, then you should remove it and run the build + # manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 + + # โ„น๏ธ Command-line programs to run using the OS shell. + # yamllint disable-line rule:line-length + # ๐Ÿ“š See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following + # three lines. Modify them (or add more) to build your code if your + # project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 + with: + category: /language:${{matrix.language}} diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 000000000..b158f07aa --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,29 @@ +# Dependency Review Action +# +# This Action will scan dependency manifest files that change as part of a +# Pull Request, surfacing known-vulnerable versions of the packages declared +# or updated in the PR. +# Once installed, if the workflow run is marked as required, +# PRs introducing known-vulnerable packages will be blocked from merging. +# +# Source repository: https://github.com/actions/dependency-review-action +name: Dependency Review + +on: + merge_group: + pull_request: + +permissions: + contents: read + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Dependency Review + uses: actions/dependency-review-action@da24556b548a50705dd671f47852072ea4c105d9 # v4.7.1 diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index 7a8b64289..8c76ed8ba 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -1,52 +1,63 @@ -name: "Validate PR title" +name: Validate PR title on: - pull_request_target: + pull_request: types: - - opened - - edited - - synchronize + - opened + - edited + - synchronize + +permissions: + contents: read jobs: main: + permissions: + # for amannn/action-semantic-pull-request to analyze PRs + pull-requests: read + # for amannn/action-semantic-pull-request to mark status of analyzed PR + statuses: write + name: Validate PR title runs-on: ubuntu-latest steps: - # Please look up the latest version from - # https://github.com/amannn/action-semantic-pull-request/releases - - uses: amannn/action-semantic-pull-request@e9fabac35e210fea40ca5b14c0da95a099eff26f # v5.4.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - # Configure which types are allowed. - # Default: https://github.com/commitizen/conventional-commit-types - types: | - fix - feat - docs - ci - chore - # Configure that a scope must always be provided. - requireScope: false - # Configure additional validation for the subject based on a regex. - # This example ensures the subject starts with an uppercase character. - subjectPattern: ^[A-Z].+$ - # If `subjectPattern` is configured, you can use this property to override - # the default error message that is shown when the pattern doesn't match. - # The variables `subject` and `title` can be used within the message. - subjectPatternError: | - The subject "{subject}" found in the pull request title "{title}" - didn't match the configured pattern. Please ensure that the subject - starts with an uppercase character. - # For work-in-progress PRs you can typically use draft pull requests - # from Github. However, private repositories on the free plan don't have - # this option and therefore this action allows you to opt-in to using the - # special "[WIP]" prefix to indicate this state. This will avoid the - # validation of the PR title and the pull request checks remain pending. - # Note that a second check will be reported if this is enabled. - wip: true - # When using "Squash and merge" on a PR with only one commit, GitHub - # will suggest using that commit message instead of the PR title for the - # merge commit, and it's easy to commit this by mistake. Enable this option - # to also validate the commit message for one commit PRs. - validateSingleCommit: false + # Please look up the latest version from + # https://github.com/amannn/action-semantic-pull-request/releases + - uses: amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017 # v5.5.3 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + # Configure which types are allowed. + # Default: https://github.com/commitizen/conventional-commit-types + types: | + fix + feat + docs + ci + chore + # Configure that a scope must always be provided. + requireScope: false + # Configure additional validation for the subject based on a regex. + # This example ensures the subject starts with an uppercase character. + subjectPattern: ^[A-Z].+$ + # If `subjectPattern` is configured, you can use this property to + # override the default error message that is shown when the pattern + # doesn't match. The variables `subject` and `title` can be used within + # the message. + subjectPatternError: | + The subject "{subject}" found in the pull request title "{title}" + didn't match the configured pattern. Please ensure that the subject + starts with an uppercase character. + # For work-in-progress PRs you can typically use draft pull requests + # from Github. However, private repositories on the free plan don't + # have this option and therefore this action allows you to opt-in to + # using the special "[WIP]" prefix to indicate this state. This will + # avoid the validation of the PR title and the pull request checks + # remain pending. Note that a second check will be reported if this + # is enabled. + wip: true + # When using "Squash and merge" on a PR with only one commit, GitHub + # will suggest using that commit message instead of the PR title for + # the merge commit, and it's easy to commit this by mistake. Enable + # this option to also validate the commit message for one commit PRs. + validateSingleCommit: false diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 577295042..fbaa235ac 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -1,25 +1,42 @@ name: Common issues check -on: [pull_request] +on: + merge_group: + pull_request: + +permissions: + contents: read jobs: pre-commit: + permissions: + contents: write # for pre-commit/action to push back fixes to PR branch runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - run: | - git fetch --no-tags --prune --depth=1 origin +refs/heads/*:refs/remotes/origin/* + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - run: >- + git fetch --no-tags --prune --depth=1 origin + +refs/heads/*:refs/remotes/origin/* - name: Get changed files id: file_changes + env: + BASE_REF: ${{ github.base_ref }} + SHA: ${{ github.sha }} run: | - export DIFF=$(git diff --name-only origin/${{ github.base_ref }} ${{ github.sha }}) - echo "Diff between ${{ github.base_ref }} and ${{ github.sha }}" + export DIFF=$(git diff --name-only "origin/$BASE_REF" "$SHA") + echo "Diff between $BASE_REF and $SHA" echo "files=$( echo "$DIFF" | xargs echo )" >> $GITHUB_OUTPUT - name: Install shfmt - run: | - curl -L "$(curl -s https://api.github.com/repos/mvdan/sh/releases/latest | grep -o -E -m 1 "https://.+?linux_amd64")" > shfmt \ + run: >- + curl -L "$( + curl -s https://api.github.com/repos/mvdan/sh/releases/latest + | grep -o -E -m 1 "https://.+?linux_amd64" + )" + > shfmt && chmod +x shfmt && sudo mv shfmt /usr/bin/ - name: Install shellcheck @@ -27,30 +44,49 @@ jobs: sudo apt update && sudo apt install shellcheck - name: Install hadolint - run: | - curl -L "$(curl -s https://api.github.com/repos/hadolint/hadolint/releases/latest | grep -o -E -m 1 "https://.+?/hadolint-Linux-x86_64")" > hadolint \ + run: >- + curl -L "$( + curl -s https://api.github.com/repos/hadolint/hadolint/releases/latest + | grep -o -E -m 1 "https://.+?/hadolint-Linux-x86_64" + )" + > hadolint && chmod +x hadolint && sudo mv hadolint /usr/bin/ - # Need to success pre-commit fix push - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + # Needed for pre-commit fix push to succeed + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 + persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} + # Needed to trigger pre-commit workflow on autofix commit. Guide: + # https://web.archive.org/web/20210731173012/https://github.community/t/required-check-is-expected-after-automated-push/187545/ + ssh-key: ${{ secrets.GHA_AUTOFIX_COMMIT_KEY }} # Skip terraform_tflint which interferes to commit pre-commit auto-fixes - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: '3.9' + python-version: '3.13' + - name: Execute pre-commit - uses: pre-commit/action@9b88afc9cd57fd75b655d5c71bd38146d07135fe # v2.0.3 + uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 env: - SKIP: no-commit-to-branch,hadolint + SKIP: no-commit-to-branch with: - token: ${{ secrets.GITHUB_TOKEN }} - extra_args: --color=always --show-diff-on-failure --files ${{ steps.file_changes.outputs.files }} - # Run only skipped checks - - name: Execute pre-commit check that have no auto-fixes - if: always() - uses: pre-commit/action@9b88afc9cd57fd75b655d5c71bd38146d07135fe # v2.0.3 - env: - SKIP: check-added-large-files,check-merge-conflict,check-vcs-permalinks,forbid-new-submodules,no-commit-to-branch,end-of-file-fixer,trailing-whitespace,check-yaml,check-merge-conflict,check-executables-have-shebangs,check-case-conflict,mixed-line-ending,detect-aws-credentials,detect-private-key,shfmt,shellcheck + extra_args: >- + --color=always + --show-diff-on-failure + --files ${{ steps.file_changes.outputs.files}} + + # Needed to trigger pre-commit workflow on autofix commit + - name: Push fixes + if: failure() + uses: EndBug/add-and-commit@a94899bca583c204427a224a7af87c02f9b325d5 # v9.1.4 with: - extra_args: --color=always --show-diff-on-failure --files ${{ steps.file_changes.outputs.files }} + # Determines the way the action fills missing author name and email. + # Three options are available: + # - github_actor -> UserName + # - user_info -> Your Display Name + # - github_actions -> github-actions + # Default: github_actor + default_author: github_actor + # The message for the commit. + # Default: 'Commit from GitHub Actions (name of the workflow)' + message: '[pre-commit] Autofix violations' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 99ed765bd..904486ca6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,31 +4,46 @@ on: workflow_dispatch: push: branches: - - master + - master paths: - - '**/*.py' - - '**/*.sh' - - 'Dockerfile' - - '.pre-commit-hooks.yaml' - # Ignore paths - - '!tests/**' + - '**/*.py' + - '**/*.sh' + - Dockerfile + - .pre-commit-hooks.yaml + # Ignore paths + - '!tests/**' + +permissions: + contents: read + jobs: release: + permissions: + # for cycjimmy/semantic-release-action to create a release + contents: write + # for cycjimmy/semantic-release-action to write comments to issues + issues: write + # for cycjimmy/semantic-release-action to write comments to PRs + pull-requests: write + name: Release runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - persist-credentials: false - fetch-depth: 0 + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + fetch-depth: 0 - - name: Release - uses: cycjimmy/semantic-release-action@61680d0e9b02ff86f5648ade99e01be17f0260a4 # v4.0.0 - with: - semantic_version: 18.0.0 - extra_plugins: | - @semantic-release/changelog@6.0.0 - @semantic-release/git@10.0.0 - env: - GITHUB_TOKEN: ${{ secrets.SEMANTIC_RELEASE_TOKEN }} + - name: Release + uses: cycjimmy/semantic-release-action@0a51e81a6baff2acad3ee88f4121c589c73d0f0e # v4.2.0 + with: + semantic_version: 18.0.0 + extra_plugins: | + @semantic-release/changelog@6.0.0 + @semantic-release/git@10.0.0 + env: + # Custom token for triggering Docker image build GH Workflow on release + # created by cycjimmy/semantic-release-action. Events created by + # workflows with default GITHUB_TOKEN not trigger other GH Workflow. + GITHUB_TOKEN: ${{ secrets.SEMANTIC_RELEASE_TOKEN }} diff --git a/.github/workflows/reusable-tox.yml b/.github/workflows/reusable-tox.yml new file mode 100644 index 000000000..5ee1edba3 --- /dev/null +++ b/.github/workflows/reusable-tox.yml @@ -0,0 +1,439 @@ +name: >- + โŒ + [DO NOT CLICK] + Reusable Tox + +on: + workflow_call: + inputs: + built-wheel-names: + description: >- + A glob for the built distributions in the artifact + to test (is installed into tox env if passed) + required: false + type: string + cache-key-files: + description: Dependency files cache + required: true + type: string + check-name: + description: A custom name for the Checks API-reported status + required: false + type: string + dists-artifact-name: + description: Workflow artifact name containing dists + required: true + type: string + environment-variables: + description: >- + A newline-delimited blob of text with environment variables + to be set using `${GITHUB_ENV}` + required: false + type: string + python-version: + description: Python version to provision in the VM + required: true + type: string + release-requested: + description: Flag whether this is CI run is a release request + default: 'false' + required: false + type: string + runner-vm-os: + description: VM OS to use + default: ubuntu + required: false + type: string + source-tarball-name: + description: Sdist filename wildcard + required: true + type: string + timeout-minutes: + description: Deadline for the job to complete + required: true + type: string + toxenv: + description: Name of the tox environment to use + required: true + type: string + tox-run-posargs: + description: Positional arguments to pass to the regular tox run + required: false + type: string + tox-rerun-posargs: + description: Positional arguments to pass to the re-attempted tox run + required: false + type: string + upstream-repository-id: + description: ID of the upstream GitHub Repository + required: true + type: string + xfail: + description: >- + Whether this job is expected to fail. Controls if the run outcomes + contribute to the failing CI status or not. The job status will be + treated as successful if this is set to `true`. Setting `false` + should be preferred typically. + required: true + type: string + secrets: + codecov-token: + description: Mandatory token for uploading to Codecov + required: true + +permissions: + contents: read + +env: + # Supposedly, pytest or coveragepy use this + COLOR: >- + yes + FORCE_COLOR: 1 # Request colored output from CLI tools supporting it + MYPY_FORCE_COLOR: 1 # MyPy's color enforcement + PIP_DISABLE_PIP_VERSION_CHECK: 1 + PIP_NO_PYTHON_VERSION_WARNING: 1 + PIP_NO_WARN_SCRIPT_LOCATION: 1 + PRE_COMMIT_COLOR: always + PY_COLORS: 1 # Recognized by the `py` package, dependency of `pytest` + PYTHONIOENCODING: utf-8 + PYTHONUTF8: 1 + TOX_PARALLEL_NO_SPINNER: 1 + # Make tox-wrapped tools see color requests + TOX_TESTENV_PASSENV: >- + COLOR + FORCE_COLOR + MYPY_FORCE_COLOR + NO_COLOR + PIP_DISABLE_PIP_VERSION_CHECK + PIP_NO_PYTHON_VERSION_WARNING + PIP_NO_WARN_SCRIPT_LOCATION + PRE_COMMIT_COLOR + PY_COLORS + PYTEST_THEME + PYTEST_THEME_MODE + PYTHONIOENCODING + PYTHONLEGACYWINDOWSSTDIO + PYTHONUTF8 + +jobs: + tox: + name: >- + ${{ + inputs.check-name + && inputs.check-name + || format( + '{0}@๐Ÿ{1}@{2}', + inputs.toxenv, + inputs.python-version, + inputs.runner-vm-os + ) + }} + + runs-on: ${{ inputs.runner-vm-os }} + + timeout-minutes: ${{ fromJSON(inputs.timeout-minutes) }} + + continue-on-error: >- + ${{ + ( + fromJSON(inputs.xfail) || + ( + startsWith(inputs.python-version, '~') + ) || + contains(inputs.python-version, 'alpha') + ) && true || false + }} + + env: + TOXENV: ${{ inputs.toxenv }} + + steps: + - name: Export requested job-global environment variables + if: inputs.environment-variables != '' + env: + INPUT_ENV_VARS: ${{ inputs.environment-variables }} + run: >- + echo "$INPUT_ENV_VARS" >> $GITHUB_ENV + + - name: >- + Switch to using Python v${{ inputs.python-version }} + by default + id: python-install + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ inputs.python-version }} + + # NOTE: `pre-commit --show-diff-on-failure` and `sphinxcontrib-spellcheck` + # NOTE: with Git authors allowlist enabled both depend on the presence of a + # NOTE: Git repository. + - name: Grab the source from Git + if: >- + contains(fromJSON('["pre-commit", "spellcheck-docs"]'), inputs.toxenv) + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + ref: ${{ github.event.inputs.release-committish }} + - name: Retrieve the project source from an sdist inside the GHA artifact + if: >- + !contains(fromJSON('["pre-commit", "spellcheck-docs"]'), inputs.toxenv) + uses: re-actors/checkout-python-sdist@187f55296b0f54d88259aaaf99af32ad3647d3bc # v2.0.0 + with: + source-tarball-name: ${{ inputs.source-tarball-name }} + workflow-artifact-name: ${{ inputs.dists-artifact-name }} + + - name: Cache pre-commit.com virtualenvs + if: inputs.toxenv == 'pre-commit' + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.cache/pre-commit + key: >- + ${{ + runner.os + }}-pre-commit-${{ + hashFiles('.pre-commit-config.yaml') + }} + + - name: Figure out if the interpreter ABI is stable + id: py-abi + run: | + from os import environ + from pathlib import Path + from sys import version_info + + FILE_APPEND_MODE = 'a' + + is_stable_abi = version_info.releaselevel == 'final' + + with Path(environ['GITHUB_OUTPUT']).open( + mode=FILE_APPEND_MODE, + ) as outputs_file: + print( + 'is-stable-abi={is_stable_abi}'. + format(is_stable_abi=str(is_stable_abi).lower()), + file=outputs_file, + ) + shell: python + - name: >- + Calculate Python interpreter version hash value + for use in the cache key + if: fromJSON(steps.py-abi.outputs.is-stable-abi) + id: calc-cache-key-py + run: | + from hashlib import sha512 + from os import environ + from pathlib import Path + from sys import version + + FILE_APPEND_MODE = 'a' + + hash = sha512(version.encode()).hexdigest() + + with Path(environ['GITHUB_OUTPUT']).open( + mode=FILE_APPEND_MODE, + ) as outputs_file: + print(f'py-hash-key={hash}', file=outputs_file) + shell: python + - name: Get pip cache dir + if: fromJSON(steps.py-abi.outputs.is-stable-abi) + id: pip-cache-dir + run: >- + echo "dir=$(python -Im pip cache dir)" >> "${GITHUB_OUTPUT}" + shell: bash + - name: Set up pip cache + if: fromJSON(steps.py-abi.outputs.is-stable-abi) + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: >- + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key }}-${{ + inputs.cache-key-files }} + restore-keys: | + ${{ runner.os }}-pip-${{ + steps.calc-cache-key-py.outputs.py-hash-key + }}- + ${{ runner.os }}-pip- + + - name: Install tox + run: >- + python -Im pip install tox + shell: bash # windows compat + + - name: Make the env clean of non-test files + if: inputs.toxenv == 'metadata-validation' + run: | + shopt -s extglob + rm -rf !tox.ini + shell: bash + - name: Download all the dists + if: >- + contains(fromJSON('["metadata-validation", "pytest"]'), inputs.toxenv) + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: ${{ inputs.dists-artifact-name }} + path: dist/ + + - name: >- + Pre-populate tox envs: `${{ env.TOXENV }}` + shell: bash + env: + INPUT_BUILT_WHEEL_NAMES: ${{ inputs.built-wheel-names }} + run: |- + tox_common_args=( + --parallel auto + --parallel-live + --skip-missing-interpreters false + ) + if [[ $INPUT_BUILT_WHEEL_NAMES ]]; then + python -Im tox "${tox_common_args[@]}" \ + --installpkg "dist/$INPUT_BUILT_WHEEL_NAMES" \ + --notest + else + python -Im tox "${tox_common_args[@]}" \ + --notest + fi + + - name: Initialize pre-commit envs if needed + if: inputs.toxenv == 'pre-commit' + run: >- + python -Im + tox + exec + --skip-pkg-install + --quiet + -- + python -Im pre_commit install-hooks + # Create GHA Job Summary markdown table of the coverage report + # But only for 'pytest' env in 'tox'. + # For details: ../../tox.ini '[testenv:pytest]' 'commands_post' + - name: >- + Run tox envs: `${{ env.TOXENV }}` + id: tox-run + shell: bash + env: + INPUT_TOX_RUN_POSARGS: ${{ inputs.tox-run-posargs }} + run: |- + tox_common_args=( + --parallel auto + --parallel-live + --skip-missing-interpreters false + --skip-pkg-install + --quiet + ) + if [ -n "$INPUT_TOX_RUN_POSARGS" ]; then + python -Im tox "${tox_common_args[@]}" \ + -- "$INPUT_TOX_RUN_POSARGS" + else + python -Im tox "${tox_common_args[@]}" + fi + + # Generate nice SVG image of passed/failed tests in GHA Job Summary + - name: Produce markdown test summary from JUnit + if: >- + !cancelled() + && steps.tox-run.outputs.test-result-files != '' + uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 + with: + paths: >- + ${{ steps.tox-run.outputs.test-result-files }} + - name: Produce markdown test summary from Cobertura XML + # NOTE: MyPy is temporarily excluded because it produces incomplete XML + # NOTE: files that `irongut/CodeCoverageSummary` can't stomach. + # Refs: + # * https://github.com/irongut/CodeCoverageSummary/issues/324 + # * https://github.com/python/mypy/issues/17689 + # FIXME: Revert the exclusion once upstream fixes the bug. + if: >- + !cancelled() + && runner.os == 'Linux' + && steps.tox-run.outputs.cov-report-files != '' + && steps.tox-run.outputs.test-result-files == '' + && steps.tox-run.outputs.codecov-flags != 'MyPy' + uses: irongut/CodeCoverageSummary@51cc3a756ddcd398d447c044c02cb6aa83fdae95 # v1.3.0 + with: + badge: true + filename: >- + ${{ steps.tox-run.outputs.cov-report-files }} + format: markdown + output: both + # Ref: https://github.com/irongut/CodeCoverageSummary/issues/66 + - name: Append coverage results to Job Summary + if: >- + !cancelled() + && runner.os == 'Linux' + && steps.tox-run.outputs.cov-report-files != '' + && steps.tox-run.outputs.test-result-files == '' + && steps.tox-run.outputs.codecov-flags != 'MyPy' + run: >- + cat code-coverage-results.md >> "$GITHUB_STEP_SUMMARY" + - name: Re-run the failing tests with maximum verbosity + if: >- + !cancelled() + && failure() + && inputs.tox-rerun-posargs != '' + # `exit 1` makes sure that the job remains red with flaky runs + env: + INPUT_TOX_RERUN_POSARGS: ${{ inputs.tox-rerun-posargs }} + run: >- + python -Im + tox + --parallel auto + --parallel-live + --skip-missing-interpreters false + -vvvvv + --skip-pkg-install + -- + $INPUT_TOX_RERUN_POSARGS + && exit 1 + shell: bash + - name: Send coverage data to Codecov + if: >- + !cancelled() + && steps.tox-run.outputs.cov-report-files != '' + uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3 + with: + disable_search: true + fail_ci_if_error: >- + ${{ toJSON(inputs.upstream-repository-id == github.repository_id) }} + files: >- + ${{ steps.tox-run.outputs.cov-report-files }} + flags: >- + CI-GHA, + ${{ steps.tox-run.outputs.codecov-flags }}, + OS-${{ + runner.os + }}, + VM-${{ + inputs.runner-vm-os + }}, + Py-${{ + steps.python-install.outputs.python-version + }} + token: ${{ secrets.codecov-token }} + - name: Upload test results to Codecov + if: >- + !cancelled() + && steps.tox-run.outputs.test-result-files != '' + uses: codecov/test-results-action@47f89e9acb64b76debcd5ea40642d25a4adced9f # v1.1.1 + # FIXME There is a bug in action which provokes it to fail during upload + # Related issue: https://github.com/codecov/codecov-action/issues/1794 + continue-on-error: true + with: + disable_search: true + fail_ci_if_error: >- + ${{ toJSON(inputs.upstream-repository-id == github.repository_id) }} + files: >- + ${{ steps.tox-run.outputs.test-result-files }} + flags: >- + CI-GHA, + ${{ steps.tox-run.outputs.codecov-flags }}, + OS-${{ + runner.os + }}, + VM-${{ + inputs.runner-vm-os + }}, + Py-${{ + steps.python-install.outputs.python-version + }} + token: ${{ secrets.codecov-token }} diff --git a/.github/workflows/scheduled-runs.yml b/.github/workflows/scheduled-runs.yml new file mode 100644 index 000000000..ba075e805 --- /dev/null +++ b/.github/workflows/scheduled-runs.yml @@ -0,0 +1,41 @@ +name: ๐Ÿ•” + +on: + pull_request: + paths: # only changes to this workflow itself trigger PR testing + - .github/workflows/scheduled-runs.yml + schedule: + - cron: 3 5 * * * # run daily at 5:03 UTC + workflow_dispatch: # manual trigger + +permissions: + contents: read + +run-name: >- + ๐ŸŒƒ + Nightly run of + ${{ + github.event.pull_request.number && 'PR' || '' + }}${{ + !github.event.pull_request.number && 'Commit' || '' + }} + ${{ github.event.pull_request.number || github.sha }} + triggered by: ${{ github.event_name }} of ${{ + github.ref + }} ${{ + github.ref_type + }} + (workflow run ID: ${{ + github.run_id + }}; number: ${{ + github.run_number + }}; attempt: ${{ + github.run_attempt + }}) + +jobs: + main-ci-cd-pipeline: + name: โˆž Main CI/CD pipeline + uses: ./.github/workflows/ci-cd.yml + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml new file mode 100644 index 000000000..40dec7de9 --- /dev/null +++ b/.github/workflows/scorecards.yml @@ -0,0 +1,79 @@ +# This workflow uses actions that are not certified by GitHub. They are +# provided by a third-party and are governed by separate terms of service, +# privacy policy, and support documentation. + +name: Scorecard supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + schedule: + - cron: 20 7 * * 2 + push: + branches: + - master + +# Declare default permissions as read only. +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed to publish results and get a badge (see publish_results below). + id-token: write + contents: read + actions: read + # To allow GraphQL ListCommits to work + issues: read + pull-requests: read + # To detect SAST tools + checks: read + + steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Run analysis + uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + with: + results_file: results.sarif + results_format: sarif + # (Optional) "write" PAT token. Uncomment the `repo_token` line if: + # - you want to enable the Branch-Protection check on a *public* + # repository, or + # - you are installing Scorecards on a *private* repository + # To create the PAT, follow the steps in + # https://github.com/ossf/scorecard-action#authentication-with-pat. + # repo_token: ${{ secrets.SCORECARD_TOKEN }} + + # Public repositories: + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + # For private repositories: + # - `publish_results` will always be set to `false`, regardless + # of the value entered here. + publish_results: true + + # Upload the results as artifacts (optional). Commenting out will disable + # uploads of run results in SARIF format to the repository Actions tab. + - name: Upload artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # Upload the results to GitHub's code scanning dashboard. + - name: Upload to code-scanning + uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 + with: + sarif_file: results.sarif diff --git a/.github/workflows/stale-actions.yaml b/.github/workflows/stale-actions.yaml index f43e10db8..5957e0ebd 100644 --- a/.github/workflows/stale-actions.yaml +++ b/.github/workflows/stale-actions.yaml @@ -1,32 +1,46 @@ -name: "Mark or close stale issues and PRs" +name: Mark or close stale issues and PRs on: schedule: - - cron: "0 0 * * *" + - cron: 0 0 * * * + +permissions: + contents: read jobs: stale: + permissions: + issues: write # for actions/stale to close stale issues + pull-requests: write # for actions/stale to close stale PRs runs-on: ubuntu-latest steps: - - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0 + - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # Staling issues and PR's days-before-stale: 30 stale-issue-label: stale stale-pr-label: stale - stale-issue-message: | - This issue has been automatically marked as stale because it has been open 30 days - with no activity. Remove stale label or comment or this issue will be closed in 10 days - stale-pr-message: | - This PR has been automatically marked as stale because it has been open 30 days - with no activity. Remove stale label or comment or this PR will be closed in 10 days + stale-issue-message: > + This issue has been automatically marked as stale because it has been + open 30 days with no activity. Remove stale label or comment or this + issue will be closed in 10 days + stale-pr-message: > + This PR has been automatically marked as stale because it has been + open 30 days + + + with no activity. Remove stale label or comment or this PR will be + closed in 10 days # Not stale if have this labels or part of milestone - exempt-issue-labels: bug,wip,on-hold - exempt-pr-labels: bug,wip,on-hold + exempt-issue-labels: bug,wip,on-hold,auto-update + exempt-pr-labels: bug,wip,on-hold exempt-all-milestones: true # Close issue operations - # Label will be automatically removed if the issues are no longer closed nor locked. + # Label will be automatically removed if the issues are no longer + # closed nor locked. days-before-close: 10 delete-branch: true - close-issue-message: This issue was automatically closed because of stale in 10 days - close-pr-message: This PR was automatically closed because of stale in 10 days + close-issue-message: >- + This issue was automatically closed because of stale in 10 days + close-pr-message: >- + This PR was automatically closed because of stale in 10 days diff --git a/.gitignore b/.gitignore index 0bbeada90..10bbad3e8 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,179 @@ +# Generated by https://gitignore.io +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +# End of https://www.toptal.com/developers/gitignore/api/python + tests/results/* diff --git a/.mypy.ini b/.mypy.ini new file mode 100644 index 000000000..a92387a64 --- /dev/null +++ b/.mypy.ini @@ -0,0 +1,69 @@ +[mypy] +python_version = 3.9 +color_output = true +error_summary = true +# IMPORTANT: The file list MUST NOT have a trailing comma after the last entry. +# Ref: https://github.com/python/mypy/issues/11171#issuecomment-2567150548 +files = + src/, + tests/pytest/ + +check_untyped_defs = true + +disallow_any_explicit = true +disallow_any_expr = true +disallow_any_decorated = true +disallow_any_generics = true +disallow_any_unimported = true +disallow_incomplete_defs = true +disallow_subclassing_any = true +disallow_untyped_calls = true +disallow_untyped_decorators = true +disallow_untyped_defs = true + +enable_error_code = + ignore-without-code + +explicit_package_bases = true + +extra_checks = true + +follow_imports = normal + +ignore_missing_imports = false + +local_partial_types = true + +mypy_path = ${MYPY_CONFIG_FILE_DIR}/src:${MYPY_CONFIG_FILE_DIR}/_type_stubs + +namespace_packages = true + +no_implicit_reexport = true + +pretty = true + +show_column_numbers = true +show_error_code_links = true +show_error_codes = true +show_error_context = true +show_error_end = true + +# `strict` will pick up any future strictness-related settings: +strict = true +strict_equality = true +strict_optional = true + +warn_no_return = true +warn_redundant_casts = true +warn_return_any = true +warn_unused_configs = true +warn_unused_ignores = true + +[mypy-tests.*] +# crashes with some decorators like `@pytest.mark.parametrize`: +disallow_any_expr = false +# fails on `@hypothesis.given()`: +disallow_any_decorated = false + +[mypy-tests.pytest.*] +disable_error_code = attr-defined diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 17fc5a6fc..498a147bd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,60 +1,173 @@ +ci: + autoupdate_schedule: quarterly + skip: + - shfmt + - shellcheck + - hadolint + repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 - hooks: - # Git style - - id: check-added-large-files - - id: check-merge-conflict - - id: check-vcs-permalinks - - id: forbid-new-submodules - - id: no-commit-to-branch - - # Common errors - - id: end-of-file-fixer - - id: trailing-whitespace - args: [--markdown-linebreak-ext=md] - exclude: CHANGELOG.md - - id: check-yaml - - id: check-merge-conflict - - id: check-executables-have-shebangs - - # Cross platform - - id: check-case-conflict - - id: mixed-line-ending - args: [--fix=lf] - - # Security - - id: detect-aws-credentials - args: ['--allow-missing-credentials'] - - id: detect-private-key + rev: cef0300fd0fc4d2a87a85fa2093c6b283ea36f4b # frozen: v5.0.0 + hooks: + # Git style + - id: check-added-large-files + - id: check-merge-conflict + - id: check-vcs-permalinks + - id: forbid-new-submodules + - id: no-commit-to-branch + # Common errors + - id: end-of-file-fixer + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + exclude: CHANGELOG.md + - id: check-yaml + - id: check-merge-conflict + - id: check-executables-have-shebangs -- repo: https://github.com/jumanjihouse/pre-commit-hooks - rev: 3.0.0 + # Cross platform + - id: check-case-conflict + - id: mixed-line-ending + args: [--fix=lf] + + # Non-modifying checks: + - id: name-tests-test + files: >- + ^tests/[^_].*\.py$ + + # Security + - id: detect-aws-credentials + args: + - --allow-missing-credentials + - id: detect-private-key + + # Detect hardcoded secrets +- repo: https://github.com/gitleaks/gitleaks + rev: a248f9279b38aeff5bbd4c85cc6f15b64d27e794 # frozen: v8.27.0 hooks: - - id: shfmt - args: ['-l', '-i', '2', '-ci', '-sr', '-w'] - - id: shellcheck + - id: gitleaks -# Dockerfile linter +# Github Action static analysis tool +- repo: https://github.com/woodruffw/zizmor-pre-commit + rev: d2c1833a059c66713cd52c032617766134679a0f # frozen: v1.9.0 + hooks: + - id: zizmor + +# Dockerfile - repo: https://github.com/hadolint/hadolint - rev: v2.12.1-beta - hooks: - - id: hadolint - args: [ - '--ignore', 'DL3027', # Do not use apt - '--ignore', 'DL3007', # Using latest - '--ignore', 'DL4006', # Not related to alpine - '--ignore', 'SC1091', # Useless check - '--ignore', 'SC2015', # Useless check - '--ignore', 'SC3037', # Not related to alpine - '--ignore', 'DL3013', # Pin versions in pip - ] - -# JSON5 Linter + rev: c3dc18df7a501f02a560a2cc7ba3c69a85ca01d3 # frozen: v2.13.1-beta + hooks: + - id: hadolint + +# YAML +- repo: https://github.com/jumanjihouse/pre-commit-hook-yamlfmt + rev: 8d1b9cadaf854cb25bb0b0f5870e1cc66a083d6b # frozen: 0.2.3 + hooks: + - id: yamlfmt + args: + - --mapping=2 + - --sequence=2 + - --offset=0 + - --width=75 + - --implicit_start + +- repo: https://github.com/adrienverge/yamllint.git + rev: 79a6b2b1392eaf49cdd32ac4f14be1a809bbd8f7 # frozen: v1.37.1 + hooks: + - id: yamllint + types: + - file + - yaml + args: + - --strict + +# JSON5 - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.1.0 + rev: f12edd9c7be1c20cfa42420fd0e6df71e42b51ea # frozen: v4.0.0-alpha.8 hooks: - id: prettier # https://prettier.io/docs/en/options.html#parser - files: '.json5$' + files: .json5$ + +# Bash +- repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 38980559e3a605691d6579f96222c30778e5a69e # frozen: 3.0.0 + hooks: + - id: shfmt + args: + - -l + - -i + - '2' + - -ci + - -sr + - -w + - id: shellcheck + +# Python +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: 9aeda5d1f4bbd212c557da1ea78eca9e8c829e19 # frozen: v0.11.13 + hooks: + - id: ruff + args: + - --fix + - id: ruff-format + +- repo: https://github.com/wemake-services/wemake-python-styleguide + rev: 6d4ca2bdc16b3098422a2770728136fc0751b817 # frozen: 1.1.0 + hooks: + - id: wemake-python-styleguide + +- repo: https://github.com/pre-commit/mirrors-mypy.git + rev: 7010b10a09f65cd60a23c207349b539aa36dbec1 # frozen: v1.16.0 + hooks: + - id: mypy + alias: mypy-py313 + name: MyPy, for Python 3.13 + additional_dependencies: + - lxml # dep of `--txt-report`, `--cobertura-xml-report` & `--html-report` + - pytest + - pytest-mock + args: + - --python-version=3.13 + - --any-exprs-report=.tox/.tmp/.test-results/mypy--py-3.13 + - --cobertura-xml-report=.tox/.tmp/.test-results/mypy--py-3.13 + - --html-report=.tox/.tmp/.test-results/mypy--py-3.13 + - --linecount-report=.tox/.tmp/.test-results/mypy--py-3.13 + - --linecoverage-report=.tox/.tmp/.test-results/mypy--py-3.13 + - --lineprecision-report=.tox/.tmp/.test-results/mypy--py-3.13 + - --txt-report=.tox/.tmp/.test-results/mypy--py-3.13 + pass_filenames: false + - id: mypy + alias: mypy-py311 + name: MyPy, for Python 3.11 + additional_dependencies: + - lxml # dep of `--txt-report`, `--cobertura-xml-report` & `--html-report` + - pytest + - pytest-mock + args: + - --python-version=3.11 + - --any-exprs-report=.tox/.tmp/.test-results/mypy--py-3.11 + - --cobertura-xml-report=.tox/.tmp/.test-results/mypy--py-3.11 + - --html-report=.tox/.tmp/.test-results/mypy--py-3.11 + - --linecount-report=.tox/.tmp/.test-results/mypy--py-3.11 + - --linecoverage-report=.tox/.tmp/.test-results/mypy--py-3.11 + - --lineprecision-report=.tox/.tmp/.test-results/mypy--py-3.11 + - --txt-report=.tox/.tmp/.test-results/mypy--py-3.11 + pass_filenames: false + - id: mypy + alias: mypy-py39 + name: MyPy, for Python 3.9 + additional_dependencies: + - lxml # dep of `--txt-report`, `--cobertura-xml-report` & `--html-report` + - pytest + - pytest-mock + args: + - --python-version=3.9 + - --any-exprs-report=.tox/.tmp/.test-results/mypy--py-3.9 + - --cobertura-xml-report=.tox/.tmp/.test-results/mypy--py-3.9 + - --html-report=.tox/.tmp/.test-results/mypy--py-3.9 + - --linecount-report=.tox/.tmp/.test-results/mypy--py-3.9 + - --linecoverage-report=.tox/.tmp/.test-results/mypy--py-3.9 + - --lineprecision-report=.tox/.tmp/.test-results/mypy--py-3.9 + - --txt-report=.tox/.tmp/.test-results/mypy--py-3.9 + pass_filenames: false diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index e8115b487..50607541e 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -4,43 +4,48 @@ entry: hooks/infracost_breakdown.sh language: script require_serial: true - files: \.(tf(vars)?|hcl)$ - exclude: \.terraform\/.*$ + files: \.(tf|tofu|tfvars|hcl)$ + exclude: \.terraform/.*$ - id: terraform_fmt name: Terraform fmt - description: Rewrites all Terraform configuration files to a canonical format. + description: >- + Rewrites all Terraform configuration files to a canonical format. entry: hooks/terraform_fmt.sh language: script - files: (\.tf|\.tfvars)$ - exclude: \.terraform\/.*$ + files: \.(tf|tofu|tfvars)$ + exclude: \.terraform/.*$ - id: terraform_docs name: Terraform docs - description: Inserts input and output documentation into README.md (using terraform-docs). + description: >- + Inserts input and output documentation into README.md + (using terraform-docs). require_serial: true entry: hooks/terraform_docs.sh language: script - files: (\.tf|\.terraform\.lock\.hcl)$ - exclude: \.terraform\/.*$ + files: \.(tf|tofu|terraform\.lock\.hcl)$ + exclude: \.terraform/.*$ - id: terraform_docs_without_aggregate_type_defaults name: Terraform docs (without aggregate type defaults) - description: Inserts input and output documentation into README.md (using terraform-docs). Identical to terraform_docs. + description: >- + Inserts input and output documentation into README.md + (using terraform-docs). Identical to terraform_docs. require_serial: true entry: hooks/terraform_docs.sh language: script - files: (\.tf)$ - exclude: \.terraform\/.*$ + files: \.(tf|tofu)$ + exclude: \.terraform/.*$ - id: terraform_docs_replace name: Terraform docs (overwrite README.md) description: Overwrite content of README.md with terraform-docs. require_serial: true - entry: terraform_docs_replace + entry: python -Im pre_commit_terraform replace-docs language: python - files: (\.tf)$ - exclude: \.terraform\/.*$ + files: \.(tf|tofu)$ + exclude: \.terraform/.*$ - id: terraform_validate name: Terraform validate @@ -48,8 +53,8 @@ require_serial: true entry: hooks/terraform_validate.sh language: script - files: (\.tf|\.tfvars)$ - exclude: \.terraform\/.*$ + files: \.(tf|tofu|tfvars|terraform\.lock\.hcl)$ + exclude: \.terraform/.*$ - id: terraform_providers_lock name: Lock terraform provider versions @@ -58,7 +63,7 @@ entry: hooks/terraform_providers_lock.sh language: script files: (\.terraform\.lock\.hcl)$ - exclude: \.terraform\/.*$ + exclude: \.terraform/.*$ - id: terraform_tflint name: Terraform validate with tflint @@ -66,16 +71,17 @@ require_serial: true entry: hooks/terraform_tflint.sh language: script - files: (\.tf|\.tfvars)$ - exclude: \.terraform\/.*$ + files: \.(tf|tofu|tfvars)$ + exclude: \.terraform/.*$ - id: terragrunt_fmt name: Terragrunt fmt - description: Rewrites all Terragrunt configuration files to a canonical format. + description: >- + Rewrites all Terragrunt configuration files to a canonical format. entry: hooks/terragrunt_fmt.sh language: script files: (\.hcl)$ - exclude: \.terraform\/.*$ + exclude: \.terraform/.*$ - id: terragrunt_validate name: Terragrunt validate @@ -83,22 +89,41 @@ entry: hooks/terragrunt_validate.sh language: script files: (\.hcl)$ - exclude: \.terraform\/.*$ + exclude: \.terraform/.*$ + +- id: terragrunt_validate_inputs + name: Terragrunt validate inputs + description: Validates Terragrunt unused and undefined inputs. + entry: hooks/terragrunt_validate_inputs.sh + language: script + files: (\.hcl)$ + exclude: \.terraform/.*$ + +- id: terragrunt_providers_lock + name: Terragrunt providers lock + description: >- + Updates provider signatures in dependency lock files using terragrunt. + entry: hooks/terragrunt_providers_lock.sh + language: script + files: (terragrunt|\.terraform\.lock)\.hcl$ + exclude: \.(terraform/.*|terragrunt-cache)$ - id: terraform_tfsec name: Terraform validate with tfsec (deprecated, use "terraform_trivy") - description: Static analysis of Terraform templates to spot potential security issues. + description: >- + Static analysis of Terraform templates to spot potential security issues. require_serial: true entry: hooks/terraform_tfsec.sh - files: \.tf(vars)?$ + files: \.(tf|tofu|tfvars)$ language: script - id: terraform_trivy name: Terraform validate with trivy - description: Static analysis of Terraform templates to spot potential security issues. + description: >- + Static analysis of Terraform templates to spot potential security issues. require_serial: true entry: hooks/terraform_trivy.sh - files: \.tf(vars)?$ + files: \.(tf|tofu|tfvars)$ language: script - id: checkov @@ -108,8 +133,8 @@ language: python pass_filenames: false always_run: false - files: \.tf$ - exclude: \.terraform\/.*$ + files: \.(tf|tofu)$ + exclude: \.terraform/.*$ require_serial: true - id: terraform_checkov @@ -118,8 +143,8 @@ entry: hooks/terraform_checkov.sh language: script always_run: false - files: \.tf$ - exclude: \.terraform\/.*$ + files: \.(tf|tofu)$ + exclude: \.terraform/.*$ require_serial: true - id: terraform_wrapper_module_for_each @@ -130,16 +155,16 @@ pass_filenames: false always_run: false require_serial: true - files: \.tf$ - exclude: \.terraform\/.*$ + files: \.(tf|tofu)$ + exclude: \.terraform/.*$ - id: terrascan name: terrascan description: Runs terrascan on Terraform templates. language: script entry: hooks/terrascan.sh - files: \.tf$ - exclude: \.terraform\/.*$ + files: \.(tf|tofu)$ + exclude: \.terraform/.*$ require_serial: true - id: tfupdate @@ -148,6 +173,6 @@ language: script entry: hooks/tfupdate.sh args: - - --args=terraform - files: \.tf$ + - --args=terraform + files: \.(tf|tofu)$ require_serial: true diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 000000000..25c4c5f1f --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,26 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations. + // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp + + // List of extensions which should be recommended for users of this workspace. + "recommendations": [ + // Global + "aaron-bond.better-comments", + "gruntfuggly.todo-tree", + "shardulm94.trailing-spaces", + "glenbuktenica.unicode-substitutions", + "editorconfig.editorconfig", + + // Grammar + "streetsidesoftware.code-spell-checker", + "znck.grammarly", + + // Documentation + "bierner.markdown-preview-github-styles", + "yzhang.markdown-all-in-one", + "DavidAnson.vscode-markdownlint", + + ], + // List of extensions recommended by VS Code that should not be recommended for users of this workspace. + "unwantedRecommendations": [] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..2d002b5a6 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,14 @@ +{ + "editor.guides.bracketPairs": "active", + "markdown.extension.toc.unorderedList.marker": "*", + "markdown.extension.toc.levels": "2..6", + "cSpell.language": "en", + "markdownlint.config": { + "code-block-style": false + }, + "markdown.validate.enabled": true, + "python.analysis.extraPaths": [ + "./src", + "./tests/pytest" + ], +} diff --git a/.yamllint b/.yamllint new file mode 100644 index 000000000..198952e2e --- /dev/null +++ b/.yamllint @@ -0,0 +1,21 @@ +extends: default + +rules: + indentation: + level: error + indent-sequences: false + document-start: + present: false + quoted-strings: + required: only-when-needed + line-length: + max: 100 + truthy: + allowed-values: + - >- + false + - >- + true + # Allow "on" key name in GHA CI/CD workflow definitions + - >- + on diff --git a/CHANGELOG.md b/CHANGELOG.md index bd283b13b..433149570 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,279 @@ All notable changes to this project will be documented in this file. +## [1.99.4](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.99.3...v1.99.4) (2025-06-12) + + +### Bug Fixes + +* **docker:** Drop Mac arm64 build-time hack, needed for `checkov`<3.2.395 ([#907](https://github.com/antonbabenko/pre-commit-terraform/issues/907)) ([3c9ef3d](https://github.com/antonbabenko/pre-commit-terraform/commit/3c9ef3d744011e44642726714521a45e66203eb3)) + +## [1.99.3](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.99.2...v1.99.3) (2025-06-06) + + +### Bug Fixes + +* **`terraform_docs`, `terraform_wrapper_module_for_each`:** Improve `.tofu` files support ([#904](https://github.com/antonbabenko/pre-commit-terraform/issues/904)) ([4f85212](https://github.com/antonbabenko/pre-commit-terraform/commit/4f852124da4d95fc9671138771e959b6c6adf1ee)) + +## [1.99.2](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.99.1...v1.99.2) (2025-06-05) + + +### Bug Fixes + +* make infracost_breakdown.sh compatible with bash 3.2 (macOS) ([#903](https://github.com/antonbabenko/pre-commit-terraform/issues/903)) ([dcb4c36](https://github.com/antonbabenko/pre-commit-terraform/commit/dcb4c3640b9c32db2d1cef5d9d109b56f743a783)) + +## [1.99.1](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.99.0...v1.99.1) (2025-05-29) + + +### Bug Fixes + +* **`terragrunt_*` hooks:** Use new subcommands for terragrunt v0.78.0+ instead of deprecated ones ([#901](https://github.com/antonbabenko/pre-commit-terraform/issues/901)) ([54468bb](https://github.com/antonbabenko/pre-commit-terraform/commit/54468bb79590e155b38b462be44937c4809aa84e)) + +# [1.99.0](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.98.1...v1.99.0) (2025-04-14) + + +### Features + +* Add support for running hooks on `.tofu` files by default ([#875](https://github.com/antonbabenko/pre-commit-terraform/issues/875)) ([fe1f62f](https://github.com/antonbabenko/pre-commit-terraform/commit/fe1f62f3aecadbc22f8ab5e1f8cb02c2821cf5c2)) + +## [1.98.1](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.98.0...v1.98.1) (2025-04-06) + + +### Bug Fixes + +* **WSL:** Fix parallelism support for WSL systems with enabled systemd ([#872](https://github.com/antonbabenko/pre-commit-terraform/issues/872)) ([da2e9a8](https://github.com/antonbabenko/pre-commit-terraform/commit/da2e9a874ac61f94fe0a05e9d952ffb3c7c7639c)) + +# [1.98.0](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.97.4...v1.98.0) (2025-03-25) + + +### Features + +* **docker:** Support execution on repos under `git worktree` ([#845](https://github.com/antonbabenko/pre-commit-terraform/issues/845)) ([e64974e](https://github.com/antonbabenko/pre-commit-terraform/commit/e64974ed7745a3c35882b71be49fc89570cb006e)) + +## [1.97.4](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.97.3...v1.97.4) (2025-02-26) + + +### Bug Fixes + +* **docker image security:** Improve dependency pinning and disable ability to build image from different tag from what specified in Dockefile ([#830](https://github.com/antonbabenko/pre-commit-terraform/issues/830)) ([2c3aa85](https://github.com/antonbabenko/pre-commit-terraform/commit/2c3aa85a2ad4a2d903b2f54ae83ef56ac63146e0)) + +## [1.97.3](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.97.2...v1.97.3) (2025-02-04) + + +### Bug Fixes + +* **`terraform_docs`:** Fix bug introduced in `v1.97.2` ([#801](https://github.com/antonbabenko/pre-commit-terraform/issues/801)) ([64b81f4](https://github.com/antonbabenko/pre-commit-terraform/commit/64b81f449344ed72d180d57ce0a801389c018584)), closes [#796](https://github.com/antonbabenko/pre-commit-terraform/issues/796) + +## [1.97.2](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.97.1...v1.97.2) (2025-02-03) + + +### Bug Fixes + +* **`terraform_docs`:** Allow having whitespaces in path to `.terraform-docs.yaml` config file ([#796](https://github.com/antonbabenko/pre-commit-terraform/issues/796)) ([7d83911](https://github.com/antonbabenko/pre-commit-terraform/commit/7d839114a62b61f2658167356df7e5da1a42ef8f)) + +## [1.97.1](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.97.0...v1.97.1) (2025-02-01) + + +### Bug Fixes + +* Parallelism CPU calculation inside Kubernetes and Docker with limits ([#799](https://github.com/antonbabenko/pre-commit-terraform/issues/799)) ([58a89a1](https://github.com/antonbabenko/pre-commit-terraform/commit/58a89a1cc0760daa515f58da9bb8b167f01044bb)) + +# [1.97.0](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.96.3...v1.97.0) (2025-01-16) + + +### Features + +* **`terraform_docs`:** Add support for custom markers to better support other formats than Markdown ([#752](https://github.com/antonbabenko/pre-commit-terraform/issues/752)) ([cd090b6](https://github.com/antonbabenko/pre-commit-terraform/commit/cd090b69c19869924a468b5a81be63264b679171)) + +## [1.96.3](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.96.2...v1.96.3) (2024-12-24) + + +### Bug Fixes + +* **`terraform_docs`:** Restore multiply `--hook-config` args support. Regression from v1.95.0 ([#731](https://github.com/antonbabenko/pre-commit-terraform/issues/731)) ([87143fb](https://github.com/antonbabenko/pre-commit-terraform/commit/87143fb465503f87d5871b2d579e29b318d2bddf)) + +## [1.96.2](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.96.1...v1.96.2) (2024-10-31) + + +### Bug Fixes + +* **WSL:** Make parallelism work appropriately ([#728](https://github.com/antonbabenko/pre-commit-terraform/issues/728)) ([e87ee43](https://github.com/antonbabenko/pre-commit-terraform/commit/e87ee4371c9f09daac814845df196a65cac28a7a)) + +## [1.96.1](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.96.0...v1.96.1) (2024-09-17) + + +### Bug Fixes + +* **`terraform_docs`:** Fix issue with processing multiply files without `terraform-docs` markers. Issue introduced in v1.95.0 ([#720](https://github.com/antonbabenko/pre-commit-terraform/issues/720)) ([2b1aec8](https://github.com/antonbabenko/pre-commit-terraform/commit/2b1aec86d8a086de4f25b502bdb97345de2eaa27)), closes [#717](https://github.com/antonbabenko/pre-commit-terraform/issues/717) [/github.com/antonbabenko/pre-commit-terraform/blob/869a106a4c8c48f34f58318a830436142e31e10a/hooks/terraform_docs.sh#L216](https://github.com//github.com/antonbabenko/pre-commit-terraform/blob/869a106a4c8c48f34f58318a830436142e31e10a/hooks/terraform_docs.sh/issues/L216) + +# [1.96.0](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.95.0...v1.96.0) (2024-09-16) + + +### Features + +* Expand environment variables in `--args=` which contains lowercase symbols, like `${TF_VAR_lowercase}` ([#719](https://github.com/antonbabenko/pre-commit-terraform/issues/719)) ([bf156b4](https://github.com/antonbabenko/pre-commit-terraform/commit/bf156b40780275db9b8ab5db6d9ef41cecc78861)) + +# [1.95.0](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.94.3...v1.95.0) (2024-09-11) + + +### Features + +* **`terraform_docs`:** Drop support for `terraform-docs` <0.12.0 ([#717](https://github.com/antonbabenko/pre-commit-terraform/issues/717)) ([81e4572](https://github.com/antonbabenko/pre-commit-terraform/commit/81e4572ad4d24fb0066fbfc4626152b6c7d48838)) + +## [1.94.3](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.94.2...v1.94.3) (2024-09-10) + + +### Bug Fixes + +* **`terraform_docs`:** Restore `--hook-config=--add-to-existing-file` default behavior. Regression from 1.94.0. ([#716](https://github.com/antonbabenko/pre-commit-terraform/issues/716)) ([315342e](https://github.com/antonbabenko/pre-commit-terraform/commit/315342e16d8ac8afe67222176e417ea02e415407)) + +## [1.94.2](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.94.1...v1.94.2) (2024-09-09) + + +### Bug Fixes + +* Support custom TF paths which contains spaces ([#714](https://github.com/antonbabenko/pre-commit-terraform/issues/714)) ([2bca410](https://github.com/antonbabenko/pre-commit-terraform/commit/2bca410814fad06f4d9cc9e31123277ae0eed23c)) + +## [1.94.1](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.94.0...v1.94.1) (2024-08-30) + + +### Bug Fixes + +* **`terraform_docs`:** Fix non-GNU sed issues, introduced in v1.93.0, as previous fix doesn't work correctly ([#708](https://github.com/antonbabenko/pre-commit-terraform/issues/708)) ([c986c5e](https://github.com/antonbabenko/pre-commit-terraform/commit/c986c5e3440be4bf5a46c7933bb629227a3cd292)) + +# [1.94.0](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.93.1...v1.94.0) (2024-08-29) + + +### Features + +* **`terraform-docs`:** Add support for `replace` mode for TF 0.12+; Use native saving to file for TF 0.12+. Both requires `terraform-docs` v0.12.0+ which released in 2021. ([#705](https://github.com/antonbabenko/pre-commit-terraform/issues/705)) ([1a1b4a3](https://github.com/antonbabenko/pre-commit-terraform/commit/1a1b4a3181065f221568a9bff86319435a4a87e1)) + +## [1.93.1](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.93.0...v1.93.1) (2024-08-29) + + +### Bug Fixes + +* **`terraform_docs`:** Fix non-GNU `sed` issues, introduced in v1.93.0 ([#704](https://github.com/antonbabenko/pre-commit-terraform/issues/704)) ([3c8734d](https://github.com/antonbabenko/pre-commit-terraform/commit/3c8734dc55e69bcfc70eceff485768a0ee89e811)) + +# [1.93.0](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.92.3...v1.93.0) (2024-08-28) + + +### Features + +* **`terraform_docs`:** Start seamless migration to `terraform-docs` markers ([#701](https://github.com/antonbabenko/pre-commit-terraform/issues/701)) ([d03f44f](https://github.com/antonbabenko/pre-commit-terraform/commit/d03f44facabf31ab7d464468907fb0a5d549e5e7)) + +## [1.92.3](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.92.2...v1.92.3) (2024-08-27) + + +### Bug Fixes + +* **`terraform_docs`:** Suppress redundant warnings pop-ups introduced in v1.92.2 ([#700](https://github.com/antonbabenko/pre-commit-terraform/issues/700)) ([59b2454](https://github.com/antonbabenko/pre-commit-terraform/commit/59b2454e076a9d26ad93d0ca4037746fd7f5962d)) + +## [1.92.2](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.92.1...v1.92.2) (2024-08-16) + + +### Bug Fixes + +* **`terraform_docs`:** Fix issue and prioritize `output.file` setting from `.terraform-docs.yml` config over `--hook-config=--path-to-file=` ([#698](https://github.com/antonbabenko/pre-commit-terraform/issues/698)) ([9d6a22b](https://github.com/antonbabenko/pre-commit-terraform/commit/9d6a22badbd9693a72c2519eb7dde01d10db57b2)) + +## [1.92.1](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.92.0...v1.92.1) (2024-08-01) + + +### Bug Fixes + +* **`terraform_docs`:** Suppress "terraform command not found" error message in case binary does not exist ([#693](https://github.com/antonbabenko/pre-commit-terraform/issues/693)) ([6ff3572](https://github.com/antonbabenko/pre-commit-terraform/commit/6ff3572afb0a70c6fe4c6a0524d1f332a4f8fb6c)) + +# [1.92.0](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.91.0...v1.92.0) (2024-06-19) + + +### Features + +* Add `terragrunt_validate_inputs` hook to check unused and undefined inputs ([#677](https://github.com/antonbabenko/pre-commit-terraform/issues/677)) ([a139b71](https://github.com/antonbabenko/pre-commit-terraform/commit/a139b71bc722ac1d2d5ed89caeb74d66a882bb94)) + +# [1.91.0](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.90.0...v1.91.0) (2024-06-07) + + +### Features + +* Added Terramate as sponsor ([#676](https://github.com/antonbabenko/pre-commit-terraform/issues/676)) ([dae1a48](https://github.com/antonbabenko/pre-commit-terraform/commit/dae1a483b429506863c3c7203932fef4fa74f86a)) + +# [1.90.0](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.89.1...v1.90.0) (2024-05-23) + + +### Features + +* Support set custom TF/OpenTofu binary. | If you use a custom Docker image build, please note that `TERRAFORM_VERSION` now must be provided ([#670](https://github.com/antonbabenko/pre-commit-terraform/issues/670)) ([c7011c0](https://github.com/antonbabenko/pre-commit-terraform/commit/c7011c06b84fc96c9a5f2f4508d5ced83ddd2af0)) + +## [1.89.1](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.89.0...v1.89.1) (2024-04-25) + + +### Bug Fixes + +* **docker:** Prevent all possible "silent errors" during `docker build` ([#644](https://github.com/antonbabenko/pre-commit-terraform/issues/644)) ([0340c8d](https://github.com/antonbabenko/pre-commit-terraform/commit/0340c8d00fe3ba39829b66fd7890d828697a7878)) + +# [1.89.0](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.88.4...v1.89.0) (2024-04-15) + + +### Features + +* Hook terraform_wrapper_module_for_each should use versions.tf from the module if it exists ([#657](https://github.com/antonbabenko/pre-commit-terraform/issues/657)) ([b127601](https://github.com/antonbabenko/pre-commit-terraform/commit/b127601a0b3d5af3dcc9f91a6d74e16f37d66a60)) + +## [1.88.4](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.88.3...v1.88.4) (2024-03-25) + + +### Bug Fixes + +* Improve README and drop quotes from hook env vars ([#651](https://github.com/antonbabenko/pre-commit-terraform/issues/651)) ([daec682](https://github.com/antonbabenko/pre-commit-terraform/commit/daec6823f980ef0e9ac8675ed93b6861fcbe58cc)) + +## [1.88.3](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.88.2...v1.88.3) (2024-03-22) + + +### Bug Fixes + +* **`terraform_providers_lock`:** Require `terraform init` (and `terraform_validate` hook) run when only lockfile changed ([#649](https://github.com/antonbabenko/pre-commit-terraform/issues/649)) ([02c1935](https://github.com/antonbabenko/pre-commit-terraform/commit/02c1935a12c889a029bc0a571410f19eb39bbab1)) + +## [1.88.2](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.88.1...v1.88.2) (2024-03-13) + + +### Bug Fixes + +* **non-linux:** Bash environment variables in arguments not expanded + Add `trace` log level ([#645](https://github.com/antonbabenko/pre-commit-terraform/issues/645)) ([a2a2990](https://github.com/antonbabenko/pre-commit-terraform/commit/a2a2990ca42f93e2c1d61507d8c75e493d29dee6)) + +## [1.88.1](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.88.0...v1.88.1) (2024-03-11) + + +### Bug Fixes + +* **docker:** Checkov installation silently fails on `docker build` in arm64. Workaround till issue will be fixed in `checkov` itself ([#635](https://github.com/antonbabenko/pre-commit-terraform/issues/635)) ([f255b05](https://github.com/antonbabenko/pre-commit-terraform/commit/f255b05feaace02f38822e3b53cf38c38e069115)) + +# [1.88.0](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.87.1...v1.88.0) (2024-02-22) + + +### Features + +* Add `terragrunt_providers_lock` hook ([#632](https://github.com/antonbabenko/pre-commit-terraform/issues/632)) ([77940fd](https://github.com/antonbabenko/pre-commit-terraform/commit/77940fd1fbbe9d3ea70306f396e1d8a13534d51d)) + +## [1.87.1](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.87.0...v1.87.1) (2024-02-19) + + +### Bug Fixes + +* Replace `mapfile` to support Bash 3.2.57 pre-installed in macOS ([#628](https://github.com/antonbabenko/pre-commit-terraform/issues/628)) ([01ab3f0](https://github.com/antonbabenko/pre-commit-terraform/commit/01ab3f0c68abda9f5799647f783c91c3d1fa3a90)) + +# [1.87.0](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.86.1...v1.87.0) (2024-02-17) + + +### Features + +* Add parallelism to major chunk of hooks. Check `Parallelism` section in README ([#620](https://github.com/antonbabenko/pre-commit-terraform/issues/620)) ([6c6eca4](https://github.com/antonbabenko/pre-commit-terraform/commit/6c6eca463a74fa2608cb3de3e03873765d46252f)) + +## [1.86.1](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.86.0...v1.86.1) (2024-02-16) + + +### Bug Fixes + +* `grep: warning: stray \ before /` which pop-up in `grep 3.8` ([#625](https://github.com/antonbabenko/pre-commit-terraform/issues/625)) ([e1a93b2](https://github.com/antonbabenko/pre-commit-terraform/commit/e1a93b26b29eda144fd0f53e3d84a99c07b15070)) + # [1.86.0](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.85.0...v1.86.0) (2023-12-21) diff --git a/Dockerfile b/Dockerfile index fcf33f2a9..4c69f2586 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ -ARG TAG=3.12.0-alpine3.17@sha256:fc34b07ec97a4f288bc17083d288374a803dd59800399c76b977016c9fe5b8f2 -FROM python:${TAG} as builder +FROM python:3.12.0-alpine3.17@sha256:fc34b07ec97a4f288bc17083d288374a803dd59800399c76b977016c9fe5b8f2 AS python_base + +FROM python_base AS builder ARG TARGETOS ARG TARGETARCH @@ -7,39 +8,41 @@ WORKDIR /bin_dir RUN apk add --no-cache \ # Builder deps + bash=~5 \ curl=~8 && \ # Upgrade packages for be able get latest Checkov python3 -m pip install --no-cache-dir --upgrade \ - pip \ - setuptools - -ARG PRE_COMMIT_VERSION=${PRE_COMMIT_VERSION:-latest} -ARG TERRAFORM_VERSION=${TERRAFORM_VERSION:-latest} + pip~=25.0 \ + setuptools~=75.8 -# Install pre-commit -RUN [ ${PRE_COMMIT_VERSION} = "latest" ] && pip3 install --no-cache-dir pre-commit \ - || pip3 install --no-cache-dir pre-commit==${PRE_COMMIT_VERSION} +COPY tools/install/ /install/ -# Install terraform because pre-commit needs it -RUN if [ "${TERRAFORM_VERSION}" = "latest" ]; then \ - TERRAFORM_VERSION="$(curl -s https://api.github.com/repos/hashicorp/terraform/releases/latest | grep tag_name | grep -o -E -m 1 "[0-9.]+")" \ - ; fi && \ - curl -L "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${TARGETOS}_${TARGETARCH}.zip" > terraform.zip && \ - unzip terraform.zip terraform && rm terraform.zip +# +# Install required tools +# +ARG PRE_COMMIT_VERSION=${PRE_COMMIT_VERSION:-latest} +RUN touch /.env && \ + if [ "$PRE_COMMIT_VERSION" = "false" ]; then \ + echo "Vital software can't be skipped" && exit 1; \ + fi +RUN /install/pre-commit.sh # # Install tools # +ARG OPENTOFU_VERSION=${OPENTOFU_VERSION:-false} +ARG TERRAFORM_VERSION=${TERRAFORM_VERSION:-false} + ARG CHECKOV_VERSION=${CHECKOV_VERSION:-false} +ARG HCLEDIT_VERSION=${HCLEDIT_VERSION:-false} ARG INFRACOST_VERSION=${INFRACOST_VERSION:-false} ARG TERRAFORM_DOCS_VERSION=${TERRAFORM_DOCS_VERSION:-false} ARG TERRAGRUNT_VERSION=${TERRAGRUNT_VERSION:-false} ARG TERRASCAN_VERSION=${TERRASCAN_VERSION:-false} ARG TFLINT_VERSION=${TFLINT_VERSION:-false} ARG TFSEC_VERSION=${TFSEC_VERSION:-false} -ARG TRIVY_VERSION=${TRIVY_VERSION:-false} ARG TFUPDATE_VERSION=${TFUPDATE_VERSION:-false} -ARG HCLEDIT_VERSION=${HCLEDIT_VERSION:-false} +ARG TRIVY_VERSION=${TRIVY_VERSION:-false} # Tricky thing to install all tools by set only one arg. @@ -47,134 +50,51 @@ ARG HCLEDIT_VERSION=${HCLEDIT_VERSION:-false} # specified in step below ARG INSTALL_ALL=${INSTALL_ALL:-false} RUN if [ "$INSTALL_ALL" != "false" ]; then \ - echo "export CHECKOV_VERSION=latest" >> /.env && \ - echo "export INFRACOST_VERSION=latest" >> /.env && \ - echo "export TERRAFORM_DOCS_VERSION=latest" >> /.env && \ - echo "export TERRAGRUNT_VERSION=latest" >> /.env && \ - echo "export TERRASCAN_VERSION=latest" >> /.env && \ - echo "export TFLINT_VERSION=latest" >> /.env && \ - echo "export TFSEC_VERSION=latest" >> /.env && \ - echo "export TRIVY_VERSION=latest" >> /.env && \ - echo "export TFUPDATE_VERSION=latest" >> /.env && \ - echo "export HCLEDIT_VERSION=latest" >> /.env \ - ; else \ - touch /.env \ - ; fi - - -# Checkov -RUN . /.env && \ - if [ "$CHECKOV_VERSION" != "false" ]; then \ - ( \ - apk add --no-cache gcc=~12 libffi-dev=~3 musl-dev=~1; \ - [ "$CHECKOV_VERSION" = "latest" ] && pip3 install --no-cache-dir checkov \ - || pip3 install --no-cache-dir checkov==${CHECKOV_VERSION}; \ - apk del gcc libffi-dev musl-dev \ - ) \ - ; fi - -# infracost -RUN . /.env && \ - if [ "$INFRACOST_VERSION" != "false" ]; then \ - ( \ - INFRACOST_RELEASES="https://api.github.com/repos/infracost/infracost/releases" && \ - [ "$INFRACOST_VERSION" = "latest" ] && curl -L "$(curl -s ${INFRACOST_RELEASES}/latest | grep -o -E -m 1 "https://.+?-${TARGETOS}-${TARGETARCH}.tar.gz")" > infracost.tgz \ - || curl -L "$(curl -s ${INFRACOST_RELEASES} | grep -o -E "https://.+?v${INFRACOST_VERSION}/infracost-${TARGETOS}-${TARGETARCH}.tar.gz")" > infracost.tgz \ - ) && tar -xzf infracost.tgz && rm infracost.tgz && mv infracost-${TARGETOS}-${TARGETARCH} infracost \ - ; fi - -# Terraform docs -RUN . /.env && \ - if [ "$TERRAFORM_DOCS_VERSION" != "false" ]; then \ - ( \ - TERRAFORM_DOCS_RELEASES="https://api.github.com/repos/terraform-docs/terraform-docs/releases" && \ - [ "$TERRAFORM_DOCS_VERSION" = "latest" ] && curl -L "$(curl -s ${TERRAFORM_DOCS_RELEASES}/latest | grep -o -E -m 1 "https://.+?-${TARGETOS}-${TARGETARCH}.tar.gz")" > terraform-docs.tgz \ - || curl -L "$(curl -s ${TERRAFORM_DOCS_RELEASES} | grep -o -E "https://.+?v${TERRAFORM_DOCS_VERSION}-${TARGETOS}-${TARGETARCH}.tar.gz")" > terraform-docs.tgz \ - ) && tar -xzf terraform-docs.tgz terraform-docs && rm terraform-docs.tgz && chmod +x terraform-docs \ - ; fi - -# Terragrunt -RUN . /.env \ - && if [ "$TERRAGRUNT_VERSION" != "false" ]; then \ - ( \ - TERRAGRUNT_RELEASES="https://api.github.com/repos/gruntwork-io/terragrunt/releases" && \ - [ "$TERRAGRUNT_VERSION" = "latest" ] && curl -L "$(curl -s ${TERRAGRUNT_RELEASES}/latest | grep -o -E -m 1 "https://.+?/terragrunt_${TARGETOS}_${TARGETARCH}")" > terragrunt \ - || curl -L "$(curl -s ${TERRAGRUNT_RELEASES} | grep -o -E -m 1 "https://.+?v${TERRAGRUNT_VERSION}/terragrunt_${TARGETOS}_${TARGETARCH}")" > terragrunt \ - ) && chmod +x terragrunt \ + echo "OPENTOFU_VERSION=latest" >> /.env && \ + echo "TERRAFORM_VERSION=latest" >> /.env && \ + \ + echo "CHECKOV_VERSION=latest" >> /.env && \ + echo "HCLEDIT_VERSION=latest" >> /.env && \ + echo "INFRACOST_VERSION=latest" >> /.env && \ + echo "TERRAFORM_DOCS_VERSION=latest" >> /.env && \ + echo "TERRAGRUNT_VERSION=latest" >> /.env && \ + echo "TERRASCAN_VERSION=latest" >> /.env && \ + echo "TFLINT_VERSION=latest" >> /.env && \ + echo "TFSEC_VERSION=latest" >> /.env && \ + echo "TFUPDATE_VERSION=latest" >> /.env && \ + echo "TRIVY_VERSION=latest" >> /.env \ ; fi +# Docker `RUN`s shouldn't be consolidated here +# hadolint global ignore=DL3059 +RUN /install/opentofu.sh +RUN /install/terraform.sh -# Terrascan -RUN . /.env && \ - if [ "$TERRASCAN_VERSION" != "false" ]; then \ - if [ "$TARGETARCH" != "amd64" ]; then ARCH="$TARGETARCH"; else ARCH="x86_64"; fi; \ - # Convert the first letter to Uppercase - OS="$(echo ${TARGETOS} | cut -c1 | tr '[:lower:]' '[:upper:]' | xargs echo -n; echo ${TARGETOS} | cut -c2-)"; \ - ( \ - TERRASCAN_RELEASES="https://api.github.com/repos/tenable/terrascan/releases" && \ - [ "$TERRASCAN_VERSION" = "latest" ] && curl -L "$(curl -s ${TERRASCAN_RELEASES}/latest | grep -o -E -m 1 "https://.+?_${OS}_${ARCH}.tar.gz")" > terrascan.tar.gz \ - || curl -L "$(curl -s ${TERRASCAN_RELEASES} | grep -o -E "https://.+?${TERRASCAN_VERSION}_${OS}_${ARCH}.tar.gz")" > terrascan.tar.gz \ - ) && tar -xzf terrascan.tar.gz terrascan && rm terrascan.tar.gz && \ - ./terrascan init \ - ; fi +RUN /install/checkov.sh +RUN /install/hcledit.sh +RUN /install/infracost.sh +RUN /install/terraform-docs.sh +RUN /install/terragrunt.sh +RUN /install/terrascan.sh +RUN /install/tflint.sh +RUN /install/tfsec.sh +RUN /install/tfupdate.sh +RUN /install/trivy.sh -# TFLint -RUN . /.env && \ - if [ "$TFLINT_VERSION" != "false" ]; then \ - ( \ - TFLINT_RELEASES="https://api.github.com/repos/terraform-linters/tflint/releases" && \ - [ "$TFLINT_VERSION" = "latest" ] && curl -L "$(curl -s ${TFLINT_RELEASES}/latest | grep -o -E -m 1 "https://.+?_${TARGETOS}_${TARGETARCH}.zip")" > tflint.zip \ - || curl -L "$(curl -s ${TFLINT_RELEASES} | grep -o -E "https://.+?/v${TFLINT_VERSION}/tflint_${TARGETOS}_${TARGETARCH}.zip")" > tflint.zip \ - ) && unzip tflint.zip && rm tflint.zip \ - ; fi - -# TFSec -RUN . /.env && \ - if [ "$TFSEC_VERSION" != "false" ]; then \ - ( \ - TFSEC_RELEASES="https://api.github.com/repos/aquasecurity/tfsec/releases" && \ - [ "$TFSEC_VERSION" = "latest" ] && curl -L "$(curl -s ${TFSEC_RELEASES}/latest | grep -o -E -m 1 "https://.+?/tfsec-${TARGETOS}-${TARGETARCH}")" > tfsec \ - || curl -L "$(curl -s ${TFSEC_RELEASES} | grep -o -E -m 1 "https://.+?v${TFSEC_VERSION}/tfsec-${TARGETOS}-${TARGETARCH}")" > tfsec \ - ) && chmod +x tfsec \ - ; fi - -# Trivy -RUN . /.env && \ - if [ "$TRIVY_VERSION" != "false" ]; then \ - if [ "$TARGETARCH" != "amd64" ]; then ARCH="$TARGETARCH"; else ARCH="64bit"; fi; \ - ( \ - TRIVY_RELEASES="https://api.github.com/repos/aquasecurity/trivy/releases" && \ - [ "$TRIVY_VERSION" = "latest" ] && curl -L "$(curl -s ${TRIVY_RELEASES}/latest | grep -o -E -i -m 1 "https://.+?/trivy_.+?_${TARGETOS}-${ARCH}.tar.gz")" > trivy.tar.gz \ - || curl -L "$(curl -s ${TRIVY_RELEASES} | grep -o -E -i -m 1 "https://.+?/v${TRIVY_VERSION}/trivy_.+?_${TARGETOS}-${ARCH}.tar.gz")" > trivy.tar.gz \ - ) && tar -xzf trivy.tar.gz trivy && rm trivy.tar.gz \ - ; fi - -# TFUpdate -RUN . /.env && \ - if [ "$TFUPDATE_VERSION" != "false" ]; then \ - ( \ - TFUPDATE_RELEASES="https://api.github.com/repos/minamijoyo/tfupdate/releases" && \ - [ "$TFUPDATE_VERSION" = "latest" ] && curl -L "$(curl -s ${TFUPDATE_RELEASES}/latest | grep -o -E -m 1 "https://.+?_${TARGETOS}_${TARGETARCH}.tar.gz")" > tfupdate.tgz \ - || curl -L "$(curl -s ${TFUPDATE_RELEASES} | grep -o -E -m 1 "https://.+?${TFUPDATE_VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz")" > tfupdate.tgz \ - ) && tar -xzf tfupdate.tgz tfupdate && rm tfupdate.tgz \ - ; fi - -# hcledit -RUN . /.env && \ - if [ "$HCLEDIT_VERSION" != "false" ]; then \ - ( \ - HCLEDIT_RELEASES="https://api.github.com/repos/minamijoyo/hcledit/releases" && \ - [ "$HCLEDIT_VERSION" = "latest" ] && curl -L "$(curl -s ${HCLEDIT_RELEASES}/latest | grep -o -E -m 1 "https://.+?_${TARGETOS}_${TARGETARCH}.tar.gz")" > hcledit.tgz \ - || curl -L "$(curl -s ${HCLEDIT_RELEASES} | grep -o -E -m 1 "https://.+?${HCLEDIT_VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz")" > hcledit.tgz \ - ) && tar -xzf hcledit.tgz hcledit && rm hcledit.tgz \ - ; fi # Checking binaries versions and write it to debug file + +# SC2086 - We do not need to quote "$F" variable, because it's not contain spaces +# DL4006 - Not Applicable for /bin/sh in alpine images. Disable, as recommended by check itself +# hadolint ignore=SC2086,DL4006 RUN . /.env && \ F=tools_versions_info && \ pre-commit --version >> $F && \ - ./terraform --version | head -n 1 >> $F && \ + (if [ "$OPENTOFU_VERSION" != "false" ]; then ./tofu --version | head -n 1 >> $F; else echo "opentofu SKIPPED" >> $F ; fi) && \ + (if [ "$TERRAFORM_VERSION" != "false" ]; then ./terraform --version | head -n 1 >> $F; else echo "terraform SKIPPED" >> $F ; fi) && \ + \ (if [ "$CHECKOV_VERSION" != "false" ]; then echo "checkov $(checkov --version)" >> $F; else echo "checkov SKIPPED" >> $F ; fi) && \ + (if [ "$HCLEDIT_VERSION" != "false" ]; then echo "hcledit $(./hcledit version)" >> $F; else echo "hcledit SKIPPED" >> $F ; fi) && \ (if [ "$INFRACOST_VERSION" != "false" ]; then echo "$(./infracost --version)" >> $F; else echo "infracost SKIPPED" >> $F ; fi) && \ (if [ "$TERRAFORM_DOCS_VERSION" != "false" ]; then ./terraform-docs --version >> $F; else echo "terraform-docs SKIPPED" >> $F ; fi) && \ (if [ "$TERRAGRUNT_VERSION" != "false" ]; then ./terragrunt --version >> $F; else echo "terragrunt SKIPPED" >> $F ; fi) && \ @@ -182,12 +102,12 @@ RUN . /.env && \ (if [ "$TFLINT_VERSION" != "false" ]; then ./tflint --version >> $F; else echo "tflint SKIPPED" >> $F ; fi) && \ (if [ "$TFSEC_VERSION" != "false" ]; then echo "tfsec $(./tfsec --version)" >> $F; else echo "tfsec SKIPPED" >> $F ; fi) && \ (if [ "$TFUPDATE_VERSION" != "false" ]; then echo "tfupdate $(./tfupdate --version)" >> $F; else echo "tfupdate SKIPPED" >> $F ; fi) && \ - (if [ "$HCLEDIT_VERSION" != "false" ]; then echo "hcledit $(./hcledit version)" >> $F; else echo "hcledit SKIPPED" >> $F ; fi) && \ - echo -e "\n\n" && cat $F && echo -e "\n\n" + (if [ "$TRIVY_VERSION" != "false" ]; then echo "trivy $(./trivy --version)" >> $F; else echo "trivy SKIPPED" >> $F ; fi) && \ + printf "\n\n\n" && cat $F && printf "\n\n\n" -FROM python:${TAG} +FROM python_base RUN apk add --no-cache \ # pre-commit deps diff --git a/README.md b/README.md index b4eef7d66..1546f0466 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,41 @@ # Collection of git hooks for Terraform to be used with [pre-commit framework](http://pre-commit.com/) -[![Github tag](https://img.shields.io/github/tag/antonbabenko/pre-commit-terraform.svg)](https://github.com/antonbabenko/pre-commit-terraform/releases) ![maintenance status](https://img.shields.io/maintenance/yes/2023.svg) [![Help Contribute to Open Source](https://www.codetriage.com/antonbabenko/pre-commit-terraform/badges/users.svg)](https://www.codetriage.com/antonbabenko/pre-commit-terraform) +[![Latest Github tag]](https://github.com/antonbabenko/pre-commit-terraform/releases) +![Maintenance status](https://img.shields.io/maintenance/yes/2025.svg) +[![GHA Tests CI/CD Badge]](https://github.com/antonbabenko/pre-commit-terraform/actions/workflows/ci-cd.yml) +[![Codecov pytest Badge]](https://app.codecov.io/gh/antonbabenko/pre-commit-terraform?flags[]=pytest) +[![OpenSSF Scorecard Badge]](https://scorecard.dev/viewer/?uri=github.com/antonbabenko/pre-commit-terraform) +[![OpenSSF Best Practices Badge]](https://www.bestpractices.dev/projects/9963) +[![Codetriage - Help Contribute to Open Source Badge]](https://www.codetriage.com/antonbabenko/pre-commit-terraform) -[![SWUbanner](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner-direct.svg)](https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md) - -Want to contribute? Check [open issues](https://github.com/antonbabenko/pre-commit-terraform/issues?q=label%3A%22good+first+issue%22+is%3Aopen+sort%3Aupdated-desc) and [contributing notes](/.github/CONTRIBUTING.md). - -## Sponsors +[![StandWithUkraine Banner]](https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md) +

pre-commit-terraform logo

-
-env0 +[`pre-commit-terraform`](https://github.com/antonbabenko/pre-commit-terraform) provides a collection of [Git Hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks) for Terraform and related tools and is driven by the [pre-commit framework](https://pre-commit.com). It helps ensure that Terraform, OpenTofu, and Terragrunt configurations are kept in good shape by automatically running various checks and formatting code before committing changes to version control system. This helps maintain code quality and consistency across the project. -Automated provisioning of Terraform workflows and Infrastructure as Code. +It can be run: -
-infracost +* Locally and in CI +* As standalone Git hooks or as a Docker image +* For the entire repository or just for change-related files (e.g., local git stash, last commit, or all changes in a Pull Request) - +Want to contribute? +Check [open issues](https://github.com/antonbabenko/pre-commit-terraform/issues?q=label%3A%22good+first+issue%22+is%3Aopen+sort%3Aupdated-desc) +and [contributing notes](/.github/CONTRIBUTING.md). -Cloud cost estimates for Terraform. +[Latest Github tag]: https://img.shields.io/github/tag/antonbabenko/pre-commit-terraform.svg +[Codetriage - Help Contribute to Open Source Badge]: https://www.codetriage.com/antonbabenko/pre-commit-terraform/badges/users.svg +[GHA Tests CI/CD Badge]: https://github.com/antonbabenko/pre-commit-terraform/actions/workflows/ci-cd.yml/badge.svg?branch=master +[Codecov Pytest Badge]: https://codecov.io/gh/antonbabenko/pre-commit-terraform/branch/master/graph/badge.svg?flag=pytest +[OpenSSF Scorecard Badge]: https://api.scorecard.dev/projects/github.com/antonbabenko/pre-commit-terraform/badge +[OpenSSF Best Practices Badge]: https://www.bestpractices.dev/projects/9963/badge +[StandWithUkraine Banner]: https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner-direct.svg + +## Sponsors -If you are using `pre-commit-terraform` already or want to support its development and [many other open-source projects](https://github.com/antonbabenko/terraform-aws-devops), please become a [GitHub Sponsor](https://github.com/sponsors/antonbabenko)! +If you want to support the development of `pre-commit-terraform` and [many other open-source projects](https://github.com/antonbabenko/terraform-aws-devops), please become a [GitHub Sponsor](https://github.com/sponsors/antonbabenko)! ## Table of content @@ -31,6 +44,7 @@ If you are using `pre-commit-terraform` already or want to support its developme * [Table of content](#table-of-content) * [How to install](#how-to-install) * [1. Install dependencies](#1-install-dependencies) + * [1.1 Custom Terraform binaries and OpenTofu support](#11-custom-terraform-binaries-and-opentofu-support) * [2. Install the pre-commit hook globally](#2-install-the-pre-commit-hook-globally) * [3. Add configs and hooks](#3-add-configs-and-hooks) * [4. Run](#4-run) @@ -40,6 +54,8 @@ If you are using `pre-commit-terraform` already or want to support its developme * [All hooks: Usage of environment variables in `--args`](#all-hooks-usage-of-environment-variables-in---args) * [All hooks: Set env vars inside hook at runtime](#all-hooks-set-env-vars-inside-hook-at-runtime) * [All hooks: Disable color output](#all-hooks-disable-color-output) + * [All hooks: Log levels](#all-hooks-log-levels) + * [Many hooks: Parallelism](#many-hooks-parallelism) * [checkov (deprecated) and terraform\_checkov](#checkov-deprecated-and-terraform_checkov) * [infracost\_breakdown](#infracost_breakdown) * [terraform\_docs](#terraform_docs) @@ -53,9 +69,13 @@ If you are using `pre-commit-terraform` already or want to support its developme * [terraform\_wrapper\_module\_for\_each](#terraform_wrapper_module_for_each) * [terrascan](#terrascan) * [tfupdate](#tfupdate) + * [terragrunt\_providers\_lock](#terragrunt_providers_lock) + * [terragrunt\_validate\_inputs](#terragrunt_validate_inputs) * [Docker Usage](#docker-usage) + * [About Docker image security](#about-docker-image-security) * [File Permissions](#file-permissions) * [Download Terraform modules from private GitHub repositories](#download-terraform-modules-from-private-github-repositories) +* [GitHub Actions](#github-actions) * [Authors](#authors) * [License](#license) * [Additional information for users from Russia and Belarus](#additional-information-for-users-from-russia-and-belarus) @@ -64,31 +84,6 @@ If you are using `pre-commit-terraform` already or want to support its developme ### 1. Install dependencies - - -* [`pre-commit`](https://pre-commit.com/#install), - [`terraform`](https://www.terraform.io/downloads.html), - [`git`](https://git-scm.com/downloads), - POSIX compatible shell, - Internet connection (on first run), - x86_64 or arm64 compatible operation system, - Some hardware where this OS will run, - Electricity for hardware and internet connection, - Some basic physical laws, - Hope that it all will work. -

-* [`checkov`](https://github.com/bridgecrewio/checkov) required for `terraform_checkov` hook. -* [`terraform-docs`](https://github.com/terraform-docs/terraform-docs) required for `terraform_docs` hook. -* [`terragrunt`](https://terragrunt.gruntwork.io/docs/getting-started/install/) required for `terragrunt_validate` hook. -* [`terrascan`](https://github.com/tenable/terrascan) required for `terrascan` hook. -* [`TFLint`](https://github.com/terraform-linters/tflint) required for `terraform_tflint` hook. -* [`TFSec`](https://github.com/liamg/tfsec) required for `terraform_tfsec` hook. -* [`Trivy`](https://github.com/aquasecurity/trivy) required for `terraform_trivy` hook. -* [`infracost`](https://github.com/infracost/infracost) required for `infracost_breakdown` hook. -* [`jq`](https://github.com/stedolan/jq) required for `terraform_validate` with `--retry-once-with-cleanup` flag, and for `infracost_breakdown` hook. -* [`tfupdate`](https://github.com/minamijoyo/tfupdate) required for `tfupdate` hook. -* [`hcledit`](https://github.com/minamijoyo/hcledit) required for `terraform_wrapper_module_for_each` hook. -
Docker
**Pull docker image with all hooks**: @@ -100,9 +95,13 @@ docker pull ghcr.io/antonbabenko/pre-commit-terraform:$TAG All available tags [here](https://github.com/antonbabenko/pre-commit-terraform/pkgs/container/pre-commit-terraform/versions). +Check [About Docker image security](#about-docker-image-security) section to learn more about possible security issues and why you probably want to build and maintain your own image. + + **Build from scratch**: -> **Note**: To build image you need to have [`docker buildx`](https://docs.docker.com/build/install-buildx/) enabled as default builder. +> **IMPORTANT** +> To build image you need to have [`docker buildx`](https://docs.docker.com/build/install-buildx/) enabled as default builder. > Otherwise - provide `TARGETOS` and `TARGETARCH` as additional `--build-arg`'s to `docker build`. When hooks-related `--build-arg`s are not specified, only the latest version of `pre-commit` and `terraform` will be installed. @@ -119,17 +118,18 @@ To install a specific version of individual tools, define it using `--build-arg` ```bash docker build -t pre-commit-terraform \ --build-arg PRE_COMMIT_VERSION=latest \ - --build-arg TERRAFORM_VERSION=latest \ + --build-arg OPENTOFU_VERSION=latest \ + --build-arg TERRAFORM_VERSION=1.5.7 \ --build-arg CHECKOV_VERSION=2.0.405 \ + --build-arg HCLEDIT_VERSION=latest \ --build-arg INFRACOST_VERSION=latest \ --build-arg TERRAFORM_DOCS_VERSION=0.15.0 \ --build-arg TERRAGRUNT_VERSION=latest \ --build-arg TERRASCAN_VERSION=1.10.0 \ --build-arg TFLINT_VERSION=0.31.0 \ --build-arg TFSEC_VERSION=latest \ - --build-arg TRIVY_VERSION=latest \ --build-arg TFUPDATE_VERSION=latest \ - --build-arg HCLEDIT_VERSION=latest \ + --build-arg TRIVY_VERSION=latest \ . ``` @@ -162,7 +162,7 @@ curl -L "$(curl -s https://api.github.com/repos/aquasecurity/tfsec/releases/late curl -L "$(curl -s https://api.github.com/repos/aquasecurity/trivy/releases/latest | grep -o -E -i -m 1 "https://.+?/trivy_.+?_Linux-64bit.tar.gz")" > trivy.tar.gz && tar -xzf trivy.tar.gz trivy && rm trivy.tar.gz && sudo mv trivy /usr/bin curl -L "$(curl -s https://api.github.com/repos/tenable/terrascan/releases/latest | grep -o -E -m 1 "https://.+?_Linux_x86_64.tar.gz")" > terrascan.tar.gz && tar -xzf terrascan.tar.gz terrascan && rm terrascan.tar.gz && sudo mv terrascan /usr/bin/ && terrascan init sudo apt install -y jq && \ -curl -L "$(curl -s https://api.github.com/repos/infracost/infracost/releases/latest | grep -o -E -m 1 "https://.+?-linux-amd64.tar.gz")" > infracost.tgz && tar -xzf infracost.tgz && rm infracost.tgz && sudo mv infracost-linux-amd64 /usr/bin/infracost && infracost register +curl -L "$(curl -s https://api.github.com/repos/infracost/infracost/releases/latest | grep -o -E -m 1 "https://.+?-linux-amd64.tar.gz")" > infracost.tgz && tar -xzf infracost.tgz && rm infracost.tgz && sudo mv infracost-linux-amd64 /usr/bin/infracost && infracost auth login curl -L "$(curl -s https://api.github.com/repos/minamijoyo/tfupdate/releases/latest | grep -o -E -m 1 "https://.+?_linux_amd64.tar.gz")" > tfupdate.tar.gz && tar -xzf tfupdate.tar.gz tfupdate && rm tfupdate.tar.gz && sudo mv tfupdate /usr/bin/ curl -L "$(curl -s https://api.github.com/repos/minamijoyo/hcledit/releases/latest | grep -o -E -m 1 "https://.+?_linux_amd64.tar.gz")" > hcledit.tar.gz && tar -xzf hcledit.tar.gz hcledit && rm hcledit.tar.gz && sudo mv hcledit /usr/bin/ ``` @@ -170,11 +170,11 @@ curl -L "$(curl -s https://api.github.com/repos/minamijoyo/hcledit/releases/late
-
Ubuntu 20.04
+
Ubuntu 20.04+
```bash sudo apt update -sudo apt install -y unzip software-properties-common python3 python3-pip +sudo apt install -y unzip software-properties-common python3 python3-pip python-is-python3 python3 -m pip install --upgrade pip pip3 install --no-cache-dir pre-commit pip3 install --no-cache-dir checkov @@ -184,7 +184,7 @@ curl -L "$(curl -s https://api.github.com/repos/terraform-linters/tflint/release curl -L "$(curl -s https://api.github.com/repos/aquasecurity/tfsec/releases/latest | grep -o -E -m 1 "https://.+?tfsec-linux-amd64")" > tfsec && chmod +x tfsec && sudo mv tfsec /usr/bin/ curl -L "$(curl -s https://api.github.com/repos/aquasecurity/trivy/releases/latest | grep -o -E -i -m 1 "https://.+?/trivy_.+?_Linux-64bit.tar.gz")" > trivy.tar.gz && tar -xzf trivy.tar.gz trivy && rm trivy.tar.gz && sudo mv trivy /usr/bin sudo apt install -y jq && \ -curl -L "$(curl -s https://api.github.com/repos/infracost/infracost/releases/latest | grep -o -E -m 1 "https://.+?-linux-amd64.tar.gz")" > infracost.tgz && tar -xzf infracost.tgz && rm infracost.tgz && sudo mv infracost-linux-amd64 /usr/bin/infracost && infracost register +curl -L "$(curl -s https://api.github.com/repos/infracost/infracost/releases/latest | grep -o -E -m 1 "https://.+?-linux-amd64.tar.gz")" > infracost.tgz && tar -xzf infracost.tgz && rm infracost.tgz && sudo mv infracost-linux-amd64 /usr/bin/infracost && infracost auth login curl -L "$(curl -s https://api.github.com/repos/minamijoyo/tfupdate/releases/latest | grep -o -E -m 1 "https://.+?_linux_amd64.tar.gz")" > tfupdate.tar.gz && tar -xzf tfupdate.tar.gz tfupdate && rm tfupdate.tar.gz && sudo mv tfupdate /usr/bin/ curl -L "$(curl -s https://api.github.com/repos/minamijoyo/hcledit/releases/latest | grep -o -E -m 1 "https://.+?_linux_amd64.tar.gz")" > hcledit.tar.gz && tar -xzf hcledit.tar.gz hcledit && rm hcledit.tar.gz && sudo mv hcledit /usr/bin/ ``` @@ -195,7 +195,8 @@ curl -L "$(curl -s https://api.github.com/repos/minamijoyo/hcledit/releases/late We highly recommend using [WSL/WSL2](https://docs.microsoft.com/en-us/windows/wsl/install) with Ubuntu and following the Ubuntu installation guide. Or use Docker. -> **Note**: We won't be able to help with issues that can't be reproduced in Linux/Mac. +> **IMPORTANT** +> We won't be able to help with issues that can't be reproduced in Linux/Mac. > So, try to find a working solution and send PR before open an issue. Otherwise, you can follow [this gist](https://gist.github.com/etiennejeanneaurevolve/1ed387dc73c5d4cb53ab313049587d09): @@ -211,11 +212,51 @@ E.g. `C:\Users\USERNAME\AppData\Local\Programs\Python\Python39\Lib\site-packages
- +Full list of dependencies and where they are used: + + +* [`pre-commit`](https://pre-commit.com/#install), + [`terraform`](https://www.terraform.io/downloads.html) or [`opentofu`](https://opentofu.org/docs/intro/install/), + [`git`](https://git-scm.com/downloads), + [BASH `3.2.57` or newer](https://www.gnu.org/software/bash/#download), + Internet connection (on first run), + x86_64 or arm64 compatible operating system, + Some hardware where this OS will run, + Electricity for hardware and internet connection, + Some basic physical laws, + Hope that it all will work. +

+* [`checkov`][checkov repo] required for `terraform_checkov` hook +* [`terraform-docs`][terraform-docs repo] 0.12.0+ required for `terraform_docs` hook +* [`terragrunt`][terragrunt repo] required for `terragrunt_validate` and `terragrunt_valid_inputs` hooks +* [`terrascan`][terrascan repo] required for `terrascan` hook +* [`TFLint`][tflint repo] required for `terraform_tflint` hook +* [`TFSec`][tfsec repo] required for `terraform_tfsec` hook +* [`Trivy`][trivy repo] required for `terraform_trivy` hook +* [`infracost`][infracost repo] required for `infracost_breakdown` hook +* [`jq`][jq repo] required for `terraform_validate` with `--retry-once-with-cleanup` flag, and for `infracost_breakdown` hook +* [`tfupdate`][tfupdate repo] required for `tfupdate` hook +* [`hcledit`][hcledit repo] required for `terraform_wrapper_module_for_each` hook + + +#### 1.1 Custom Terraform binaries and OpenTofu support + +It is possible to set custom path to `terraform` binary. +This makes it possible to use [OpenTofu](https://opentofu.org) binary (`tofu`) instead of `terraform`. + +How binary discovery works and how you can redefine it (first matched takes precedence): + +1. Check if per hook configuration `--hook-config=--tf-path=` is set +2. Check if `PCT_TFPATH=` environment variable is set +3. Check if `TERRAGRUNT_TFPATH=` environment variable is set +4. Check if `terraform` binary can be found in the user's `$PATH` +5. Check if `tofu` binary can be found in the user's `$PATH` + ### 2. Install the pre-commit hook globally -> **Note**: not needed if you use the Docker image +> [!NOTE] +> Not needed if you use the Docker image ```bash DIR=~/.git-template @@ -239,6 +280,14 @@ repos: EOF ``` +If this repository was initialized locally via `git init` or `git clone` _before_ +you installed the pre-commit hook globally ([step 2](#2-install-the-pre-commit-hook-globally)), +you will need to run: + +```bash +pre-commit install +``` + ### 4. Run Execute this command to run `pre-commit` on all files in the repository (not only changed files): @@ -249,7 +298,8 @@ pre-commit run -a Or, using Docker ([available tags](https://github.com/antonbabenko/pre-commit-terraform/pkgs/container/pre-commit-terraform/versions)): -> **Note**: This command uses your user id and group id for the docker container to use to access the local files. If the files are owned by another user, update the `USERID` environment variable. See [File Permissions section](#file-permissions) for more information. +> [!TIP] +> This command uses your user id and group id for the docker container to use to access the local files. If the files are owned by another user, update the `USERID` environment variable. See [File Permissions section](#file-permissions) for more information. ```bash TAG=latest @@ -267,26 +317,26 @@ docker run --rm --entrypoint cat ghcr.io/antonbabenko/pre-commit-terraform:$TAG There are several [pre-commit](https://pre-commit.com/) hooks to keep Terraform configurations (both `*.tf` and `*.tfvars`) and Terragrunt configurations (`*.hcl`) in a good shape: - -| Hook name | Description | Dependencies
[Install instructions here](#1-install-dependencies) | -| ------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------ | -| `checkov` and `terraform_checkov` | [checkov](https://github.com/bridgecrewio/checkov) static analysis of terraform templates to spot potential security issues. [Hook notes](#checkov-deprecated-and-terraform_checkov) | `checkov`
Ubuntu deps: `python3`, `python3-pip` | -| `infracost_breakdown` | Check how much your infra costs with [infracost](https://github.com/infracost/infracost). [Hook notes](#infracost_breakdown) | `infracost`, `jq`, [Infracost API key](https://www.infracost.io/docs/#2-get-api-key) | -| `terraform_docs` | Inserts input and output documentation into `README.md`. Recommended. [Hook notes](#terraform_docs) | `terraform-docs` | -| `terraform_docs_replace` | Runs `terraform-docs` and pipes the output directly to README.md. **DEPRECATED**, see [#248](https://github.com/antonbabenko/pre-commit-terraform/issues/248). [Hook notes](#terraform_docs_replace-deprecated) | `python3`, `terraform-docs` | -| `terraform_docs_without_`
`aggregate_type_defaults` | Inserts input and output documentation into `README.md` without aggregate type defaults. Hook notes same as for [terraform_docs](#terraform_docs) | `terraform-docs` | -| `terraform_fmt` | Reformat all Terraform configuration files to a canonical format. [Hook notes](#terraform_fmt) | - | -| `terraform_providers_lock` | Updates provider signatures in [dependency lock files](https://www.terraform.io/docs/cli/commands/providers/lock.html). [Hook notes](#terraform_providers_lock) | - | -| `terraform_tflint` | Validates all Terraform configuration files with [TFLint](https://github.com/terraform-linters/tflint). [Available TFLint rules](https://github.com/terraform-linters/tflint/tree/master/docs/rules#rules). [Hook notes](#terraform_tflint). | `tflint` | -| `terraform_tfsec` | [TFSec](https://github.com/aquasecurity/tfsec) static analysis of terraform templates to spot potential security issues. **DEPRECATED**, use `terraform_trivy`. [Hook notes](#terraform_tfsec-deprecated) | `tfsec` | -| `terraform_trivy` | [Trivy](https://github.com/aquasecurity/trivy) static analysis of terraform templates to spot potential security issues. [Hook notes](#terraform_trivy) | `trivy` | -| `terraform_validate` | Validates all Terraform configuration files. [Hook notes](#terraform_validate) | `jq`, only for `--retry-once-with-cleanup` flag | -| `terragrunt_fmt` | Reformat all [Terragrunt](https://github.com/gruntwork-io/terragrunt) configuration files (`*.hcl`) to a canonical format. | `terragrunt` | -| `terragrunt_validate` | Validates all [Terragrunt](https://github.com/gruntwork-io/terragrunt) configuration files (`*.hcl`) | `terragrunt` | -| `terraform_wrapper_module_for_each` | Generates Terraform wrappers with `for_each` in module. [Hook notes](#terraform_wrapper_module_for_each) | `hcledit` | -| `terrascan` | [terrascan](https://github.com/tenable/terrascan) Detect compliance and security violations. [Hook notes](#terrascan) | `terrascan` | -| `tfupdate` | [tfupdate](https://github.com/minamijoyo/tfupdate) Update version constraints of Terraform core, providers, and modules. [Hook notes](#tfupdate) | `tfupdate` | - +| Hook name | Description | Dependencies
[Install instructions here](#1-install-dependencies) | +| ------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------ | +| `checkov` and `terraform_checkov` | [checkov][checkov repo] static analysis of terraform templates to spot potential security issues. [Hook notes](#checkov-deprecated-and-terraform_checkov) | `checkov`
Ubuntu deps: `python3`, `python3-pip` | +| `infracost_breakdown` | Check how much your infra costs with [infracost][infracost repo]. [Hook notes](#infracost_breakdown) | `infracost`, `jq`, [Infracost API key](https://www.infracost.io/docs/#2-get-api-key) | +| `terraform_docs` | Inserts input and output documentation into `README.md`. [Hook notes](#terraform_docs) | `terraform-docs` | +| `terraform_docs_replace` | Runs `terraform-docs` and pipes the output directly to README.md. **DEPRECATED**, see [#248](https://github.com/antonbabenko/pre-commit-terraform/issues/248). [Hook notes](#terraform_docs_replace-deprecated) | `python3`, `terraform-docs` | +| `terraform_docs_without_`
`aggregate_type_defaults` | Inserts input and output documentation into `README.md` without aggregate type defaults. Hook notes same as for [terraform_docs](#terraform_docs) | `terraform-docs` | +| `terraform_fmt` | Reformat all Terraform configuration files to a canonical format. [Hook notes](#terraform_fmt) | - | +| `terraform_providers_lock` | Updates provider signatures in [dependency lock files](https://www.terraform.io/docs/cli/commands/providers/lock.html). [Hook notes](#terraform_providers_lock) | - | +| `terraform_tflint` | Validates all Terraform configuration files with [TFLint][tflint repo]. [Available TFLint rules](https://github.com/terraform-linters/tflint-ruleset-terraform/blob/main/docs/rules/README.md). [Hook notes](#terraform_tflint). | `tflint` | +| `terraform_tfsec` | [TFSec][tfsec repo] static analysis of terraform templates to spot potential security issues. **DEPRECATED**, use `terraform_trivy`. [Hook notes](#terraform_tfsec-deprecated) | `tfsec` | +| `terraform_trivy` | [Trivy][trivy repo] static analysis of terraform templates to spot potential security issues. [Hook notes](#terraform_trivy) | `trivy` | +| `terraform_validate` | Validates all Terraform configuration files. [Hook notes](#terraform_validate) | `jq`, only for `--retry-once-with-cleanup` flag | +| `terragrunt_fmt` | Reformat all [Terragrunt][terragrunt repo] configuration files (`*.hcl`) to a canonical format. | `terragrunt` | +| `terragrunt_validate` | Validates all [Terragrunt][terragrunt repo] configuration files (`*.hcl`) | `terragrunt` | +| `terragrunt_validate_inputs` | Validates [Terragrunt][terragrunt repo] unused and undefined inputs (`*.hcl`) | | +| `terragrunt_providers_lock` | Generates `.terraform.lock.hcl` files using [Terragrunt][terragrunt repo]. | `terragrunt` | +| `terraform_wrapper_module_for_each` | Generates Terraform wrappers with `for_each` in module. [Hook notes](#terraform_wrapper_module_for_each) | `hcledit` | +| `terrascan` | [terrascan][terrascan repo] Detect compliance and security violations. [Hook notes](#terrascan) | `terrascan` | +| `tfupdate` | [tfupdate][tfupdate repo] Update version constraints of Terraform core, providers, and modules. [Hook notes](#tfupdate) | `tfupdate` | Check the [source file](https://github.com/antonbabenko/pre-commit-terraform/blob/master/.pre-commit-hooks.yaml) to know arguments used for each hook. @@ -302,7 +352,8 @@ Terraform operates on a per-dir basis, while `pre-commit` framework only support You can use environment variables for the `--args` section. -> **Warning**: You _must_ use the `${ENV_VAR}` definition, `$ENV_VAR` will not expand. +> [!IMPORTANT] +> You _must_ use the `${ENV_VAR}` definition, `$ENV_VAR` will not expand. Config example: @@ -310,10 +361,10 @@ Config example: - id: terraform_tflint args: - --args=--config=${CONFIG_NAME}.${CONFIG_EXT} - - --args=--module + - --args=--call-module-type="all" ``` -If for config above set up `export CONFIG_NAME=.tflint; export CONFIG_EXT=hcl` before `pre-commit run`, args will be expanded to `--config=.tflint.hcl --module`. +If for config above set up `export CONFIG_NAME=.tflint; export CONFIG_EXT=hcl` before `pre-commit run`, args will be expanded to `--config=.tflint.hcl --call-module-type="all"`. ### All hooks: Set env vars inside hook at runtime @@ -321,14 +372,18 @@ If for config above set up `export CONFIG_NAME=.tflint; export CONFIG_EXT=hcl` b You can specify environment variables that will be passed to the hook at runtime. +> [!IMPORTANT] +> Variable values are exported _verbatim_: +> - No interpolation or expansion are applied +> - The enclosing double quotes are removed if they are provided + Config example: ```yaml - id: terraform_validate args: - --env-vars=AWS_DEFAULT_REGION="us-west-2" - - --env-vars=AWS_ACCESS_KEY_ID="anaccesskey" - - --env-vars=AWS_SECRET_ACCESS_KEY="asecretkey" + - --env-vars=AWS_PROFILE="my-aws-cli-profile" ``` ### All hooks: Disable color output @@ -341,6 +396,82 @@ To disable color output for all hooks, set `PRE_COMMIT_COLOR=never` var. Eg: PRE_COMMIT_COLOR=never pre-commit run ``` +### All hooks: Log levels + +In case you need to debug hooks, you can set `PCT_LOG=trace`. + +For example: + +```bash +PCT_LOG=trace pre-commit run -a +``` + +Less verbose log levels will be implemented in [#562](https://github.com/antonbabenko/pre-commit-terraform/issues/562). + +### Many hooks: Parallelism + +> All, except deprecated hooks: `checkov`, `terraform_docs_replace` and hooks which can't be paralleled this way: `infracost_breakdown`, `terraform_wrapper_module_for_each`. +> Also, there's a chance that parallelism have no effect on `terragrunt_fmt` and `terragrunt_validate` hooks + +By default, parallelism is set to `number of logical CPUs - 1`. +If you'd like to disable parallelism, set it to `1` + +```yaml +- id: terragrunt_validate + args: + - --hook-config=--parallelism-limit=1 +``` + +In the same way you can set it to any positive integer. + +If you'd like to set parallelism value relative to number of CPU logical cores - provide valid Bash arithmetic expression and use `CPU` as a reference to the number of CPU logical cores + + +```yaml +- id: terraform_providers_lock + args: + - --hook-config=--parallelism-limit=CPU*4 +``` + +> [!TIP] +>
Info useful for parallelism fine-tunning +> +>
+> Tests below were run on repo with 45 Terraform dirs on laptop with 16 CPUs, SSD and 1Gbit/s network. Laptop was slightly used in the process. +> +> Observed results may vary greatly depending on your repo structure, machine characteristics and their usage. +> +> If during fine-tuning you'll find that your results are very different from provided below and you think that this data could help someone else - feel free to send PR. +> +> +> | Hook | Most used resource | Comparison of optimization results / Notes | +> | ------------------------------------------------------------------------------ | ---------------------------------- | --------------------------------------------------------------- | +> | terraform_checkov | CPU heavy | - | +> | terraform_fmt | CPU heavy | - | +> | terraform_providers_lock (3 platforms,
`--mode=always-regenerate-lockfile`) | Network & Disk heavy | `defaults (CPU-1)` - 3m 39s; `CPU*2` - 3m 19s; `CPU*4` - 2m 56s | +> | terraform_tflint | CPU heavy | - | +> | terraform_tfsec | CPU heavy | - | +> | terraform_trivy | CPU moderate | `defaults (CPU-1)` - 32s; `CPU*2` - 30s; `CPU*4` - 31s | +> | terraform_validate (t validate only) | CPU heavy | - | +> | terraform_validate (t init + t validate) | Network & Disk heavy, CPU moderate | `defaults (CPU-1)` - 1m 30s; `CPU*2` - 1m 25s; `CPU*4` - 1m 41s | +> | terragrunt_fmt | CPU heavy | N/A? need more info from TG users | +> | terragrunt_validate | CPU heavy | N/A? need more info from TG users | +> | terrascan | CPU moderate-heavy | `defaults (CPU-1)` - 8s; `CPU*2` - 6s | +> | tfupdate | Disk/Network? | too quick in any settings. More info needed | +> +> +>
+ + + +```yaml +args: + - --hook-config=--parallelism-ci-cpu-cores=N +``` + +If you don't see code above in your `pre-commit-config.yaml` or logs - you don't need it. +`--parallelism-ci-cpu-cores` used only in edge cases and is ignored in other situations. Check out its usage in [hooks/_common.sh](hooks/_common.sh) + ### checkov (deprecated) and terraform_checkov > `checkov` hook is deprecated, please use `terraform_checkov`. @@ -358,15 +489,15 @@ Note that `terraform_checkov` runs recursively during `-d .` usage. That means, Check all available arguments [here](https://www.checkov.io/2.Basics/CLI%20Command%20Reference.html). -For deprecated hook you need to specify each argument separately: + For deprecated hook you need to specify each argument separately: -```yaml -- id: checkov - args: [ - "-d", ".", - "--skip-check", "CKV2_AWS_8", - ] -``` + ```yaml + - id: checkov + args: [ + "-d", ".", + "--skip-check", "CKV2_AWS_8", + ] + ``` 2. When you have multiple directories and want to run `terraform_checkov` in all of them and share a single config file - use the `__GIT_WORKING_DIR__` placeholder. It will be replaced by `terraform_checkov` hooks with the Git working directory (repo root) at run time. For example: @@ -390,7 +521,7 @@ Unlike most other hooks, this hook triggers once if there are any changed files - --args=--path=./env/dev verbose: true # Always show costs ``` - +
Output ```bash @@ -405,7 +536,7 @@ Unlike most other hooks, this hook triggers once if there are any changed files Total Monthly Cost: 86.83 USD Total Monthly Cost (diff): 86.83 USD ``` - +
2. Note that spaces are not allowed in `--args`, so you need to split it, like this: @@ -429,7 +560,7 @@ Unlike most other hooks, this hook triggers once if there are any changed files - --hook-config='.projects[].diff.totalMonthlyCost|tonumber != 10000' - --hook-config='.currency == "USD"' ``` - +
Output ```bash @@ -448,7 +579,7 @@ Unlike most other hooks, this hook triggers once if there are any changed files Total Monthly Cost: 86.83 USD Total Monthly Cost (diff): 86.83 USD ``` - +
* Only one path per one hook (`- id: infracost_breakdown`) is allowed. @@ -470,12 +601,12 @@ Unlike most other hooks, this hook triggers once if there are any changed files ### terraform_docs -1. `terraform_docs` and `terraform_docs_without_aggregate_type_defaults` will insert/update documentation generated by [terraform-docs](https://github.com/terraform-docs/terraform-docs) framed by markers: +1. `terraform_docs` and `terraform_docs_without_aggregate_type_defaults` will insert/update documentation generated by [terraform-docs][terraform-docs repo] framed by markers: ```txt - + - + ``` if they are present in `README.md`. @@ -486,12 +617,13 @@ Unlike most other hooks, this hook triggers once if there are any changed files * create a documentation file * extend existing documentation file by appending markers to the end of the file (see item 1 above) * use different filename for the documentation (default is `README.md`) - * use the same insertion markers as `terraform-docs` by default. It will be default in `v2.0`. - To migrate to `terraform-docs` insertion markers, run in repo root: + * use the same insertion markers as `terraform-docs`. It's default starting from `v1.93`. + To migrate everything to `terraform-docs` insertion markers, run in repo root: ```bash - grep -rl 'BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK' . | xargs sed -i 's/BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK/BEGIN_TF_DOCS/g' - grep -rl 'END OF PRE-COMMIT-TERRAFORM DOCS HOOK' . | xargs sed -i 's/END OF PRE-COMMIT-TERRAFORM DOCS HOOK/END_TF_DOCS/g' + sed --version &> /dev/null && SED_CMD=(sed -i) || SED_CMD=(sed -i '') + grep -rl --null 'BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK' . | xargs -0 "${SED_CMD[@]}" -e 's/BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK/BEGIN_TF_DOCS/' + grep -rl --null 'END OF PRE-COMMIT-TERRAFORM DOCS HOOK' . | xargs -0 "${SED_CMD[@]}" -e 's/END OF PRE-COMMIT-TERRAFORM DOCS HOOK/END_TF_DOCS/' ``` ```yaml @@ -500,10 +632,20 @@ Unlike most other hooks, this hook triggers once if there are any changed files - --hook-config=--path-to-file=README.md # Valid UNIX path. I.e. ../TFDOC.md or docs/README.md etc. - --hook-config=--add-to-existing-file=true # Boolean. true or false - --hook-config=--create-file-if-not-exist=true # Boolean. true or false - - --hook-config=--use-standard-markers=true # Boolean. Defaults in v1.x to false. Set to true for compatibility with terraform-docs + - --hook-config=--use-standard-markers=true # Boolean. Defaults to true (v1.93+), false ( # String. + # Set to use custom marker which helps you with using other formats like asciidoc. + # For Asciidoc this could be "--hook-config=--custom-marker-begin=// BEGIN_TF_DOCS" + - --hook-config=--custom-marker-end= # String. + # Set to use custom marker which helps you with using other formats like asciidoc. + # For Asciidoc this could be "--hook-config=--custom-marker-end=// END_TF_DOCS" + - --hook-config=--custom-doc-header="# " # String. Defaults to "# " + # Set to use custom marker which helps you with using other formats like asciidoc. + # For Asciidoc this could be "--hook-config=--custom-marker-end=\= " ``` -4. You can provide [any configuration available in `terraform-docs`](https://terraform-docs.io/user-guide/configuration/) as an argument to `terraform_doc` hook, for example: +4. If you want to use a terraform-docs config file, you must supply the path to the file, relative to the git repo root path: ```yaml - id: terraform_docs @@ -511,9 +653,18 @@ Unlike most other hooks, this hook triggers once if there are any changed files - --args=--config=.terraform-docs.yml ``` - > **Warning**: Avoid use `recursive.enabled: true` in config file, that can cause unexpected behavior. + > **Warning** + > Avoid use `recursive.enabled: true` in config file, that can cause unexpected behavior. + +5. You can provide [any configuration available in `terraform-docs`](https://terraform-docs.io/user-guide/configuration/) as an argument to `terraform_docs` hook: + + ```yaml + - id: terraform_docs + args: + - --args=--output-mode=replace + ``` -5. If you need some exotic settings, it can be done too. I.e. this one generates HCL files: +6. If you need some exotic settings, it can be done too. I.e. this one generates HCL files: ```yaml - id: terraform_docs @@ -545,7 +696,7 @@ To replicate functionality in `terraform_docs` hook: ```yaml - id: terraform_docs - args: + args: - --args=--config=.terraform-docs.yml ``` @@ -563,12 +714,16 @@ To replicate functionality in `terraform_docs` hook: ### terraform_providers_lock -> **Note**: The hook requires Terraform 0.14 or later. +> [!NOTE] +> The hook requires Terraform 0.14 or later. -> **Note**: The hook can invoke `terraform providers lock` that can be really slow and requires fetching metadata from remote Terraform registries - not all of that metadata is currently being cached by Terraform. +> [!NOTE] +> The hook can invoke `terraform providers lock` that can be really slow and requires fetching metadata from remote Terraform registries - not all of that metadata is currently being cached by Terraform. ->
Note: Read this if you used this hook before v1.80.0 | Planned breaking changes in v2.0 -> We introduced '--mode' flag for this hook. If you'd like to continue using this hook as before, please: +> [!NOTE] +>
Read this if you used this hook before v1.80.0 | Planned breaking changes in v2.0 +>
+> We introduced `--mode` flag for this hook. If you'd like to continue using this hook as before, please: > > * Specify `--hook-config=--mode=always-regenerate-lockfile` in `args:` > * Before `terraform_providers_lock`, add `terraform_validate` hook with `--hook-config=--retry-once-with-cleanup=true` @@ -605,7 +760,8 @@ To replicate functionality in `terraform_docs` hook: * `only-check-is-current-lockfile-cross-platform` with [terraform_validate hook](#terraform_validate) - make up-to-date lockfile by adding/removing providers and only then check that lockfile has all required SHAs. - > **Note**: Next `terraform_validate` flag requires additional dependency to be installed: `jq`. Also, it could run another slow and time consuming command - `terraform init` + > **Important** + > Next `terraform_validate` flag requires additional dependency to be installed: `jq`. Also, it could run another slow and time consuming command - `terraform init` ```yaml - id: terraform_validate @@ -630,8 +786,7 @@ To replicate functionality in `terraform_docs` hook: - --hook-config=--mode=always-regenerate-lockfile ``` - -3. `terraform_providers_lock` supports custom arguments: +2. `terraform_providers_lock` supports custom arguments: ```yaml - id: terraform_providers_lock @@ -640,7 +795,7 @@ To replicate functionality in `terraform_docs` hook: - --args=-platform=darwin_amd64 ``` -4. It may happen that Terraform working directory (`.terraform`) already exists but not in the best condition (eg, not initialized modules, wrong version of Terraform, etc.). To solve this problem, you can find and delete all `.terraform` directories in your repository: +3. It may happen that Terraform working directory (`.terraform`) already exists but not in the best condition (eg, not initialized modules, wrong version of Terraform, etc.). To solve this problem, you can find and delete all `.terraform` directories in your repository: ```bash echo " @@ -654,9 +809,10 @@ To replicate functionality in `terraform_docs` hook: `terraform_providers_lock` hook will try to reinitialize directories before running the `terraform providers lock` command. -5. `terraform_providers_lock` support passing custom arguments to its `terraform init`: +4. `terraform_providers_lock` support passing custom arguments to its `terraform init`: - > **Warning** - DEPRECATION NOTICE: This is available only in `no-mode` mode, which will be removed in v2.0. Please provide this keys to [`terraform_validate`](#terraform_validate) hook, which, to take effect, should be called before `terraform_providers_lock` + > **Warning** + > DEPRECATION NOTICE: This is available only in `no-mode` mode, which will be removed in v2.0. Please provide this keys to [`terraform_validate`](#terraform_validate) hook, which, to take effect, should be called before `terraform_providers_lock` ```yaml - id: terraform_providers_lock @@ -686,12 +842,12 @@ To replicate functionality in `terraform_docs` hook: - --args=--config=__GIT_WORKING_DIR__/.tflint.hcl ``` -3. By default, pre-commit-terraform performs directory switching into the terraform modules for you. If you want to delgate the directory changing to the binary - this will allow tflint to determine the full paths for error/warning messages, rather than just module relative paths. *Note: this requires `tflint>=0.44.0`.* For example: +3. By default, pre-commit-terraform performs directory switching into the terraform modules for you. If you want to delegate the directory changing to the binary - this will allow tflint to determine the full paths for error/warning messages, rather than just module relative paths. *Note: this requires `tflint>=0.44.0`.* For example: ```yaml - id: terraform_tflint - args: - - --hook-config=--delegate-chdir + args: + - --hook-config=--delegate-chdir ``` @@ -791,13 +947,23 @@ To replicate functionality in `terraform_docs` hook: ```yaml - id: terraform_trivy args: - - > - --args=--format json - --skip-dirs="**/.terragrunt-cache" + - --args=--format=json + - --args=--skip-dirs="**/.terraform" + ``` + +4. When you have multiple directories and want to run `trivy` in all of them and share a single config file - use the `__GIT_WORKING_DIR__` placeholder. It will be replaced by `terraform_trivy` hooks with Git working directory (repo root) at run time. For example: + + ```yaml + - id: terraform_trivy + args: + - --args=--ignorefile=__GIT_WORKING_DIR__/.trivyignore ``` ### terraform_validate +> [!IMPORTANT] +> If you use [`TF_PLUGIN_CACHE_DIR`](https://developer.hashicorp.com/terraform/cli/config/config-file#provider-plugin-cache), we recommend enabling `--hook-config=--retry-once-with-cleanup=true` or disabling parallelism (`--hook-config=--parallelism-limit=1`) to avoid [race conditions when `terraform init` writes to it](https://github.com/hashicorp/terraform/issues/31964). + 1. `terraform_validate` supports custom arguments so you can pass supported `-no-color` or `-json` flags: ```yaml @@ -826,9 +992,11 @@ To replicate functionality in `terraform_docs` hook: - --hook-config=--retry-once-with-cleanup=true # Boolean. true or false ``` - > **Note**: The flag requires additional dependency to be installed: `jq`. + > **Important** + > The flag requires additional dependency to be installed: `jq`. - > **Note**: Reinit can be very slow and require downloading data from remote Terraform registries, and not all of that downloaded data or meta-data is currently being cached by Terraform. + > **Note** + > Reinit can be very slow and require downloading data from remote Terraform registries, and not all of that downloaded data or meta-data is currently being cached by Terraform. When `--retry-once-with-cleanup=true`, in each failed directory the cached modules and providers from the `.terraform` directory will be deleted, before retrying once more. To avoid unnecessary deletion of this directory, the cleanup and retry will only happen if Terraform produces any of the following error messages: @@ -838,7 +1006,8 @@ To replicate functionality in `terraform_docs` hook: * "Module not installed" * "Could not load plugin" - **Warning**: When using `--retry-once-with-cleanup=true`, problematic `.terraform/modules/` and `.terraform/providers/` directories will be recursively deleted without prompting for consent. Other files and directories will not be affected, such as the `.terraform/environment` file. + > **Warning** + > When using `--retry-once-with-cleanup=true`, problematic `.terraform/modules/` and `.terraform/providers/` directories will be recursively deleted without prompting for consent. Other files and directories will not be affected, such as the `.terraform/environment` file. **Option 2** @@ -856,7 +1025,8 @@ To replicate functionality in `terraform_docs` hook: `terraform_validate` hook will try to reinitialize them before running the `terraform validate` command. - **Warning**: If you use Terraform workspaces, DO NOT use this option ([details](https://github.com/antonbabenko/pre-commit-terraform/issues/203#issuecomment-918791847)). Consider the first option, or wait for [`force-init`](https://github.com/antonbabenko/pre-commit-terraform/issues/224) option implementation. + > **Caution** + > If you use Terraform workspaces, DO NOT use this option ([details](https://github.com/antonbabenko/pre-commit-terraform/issues/203#issuecomment-918791847)). Consider the first option, or wait for [`force-init`](https://github.com/antonbabenko/pre-commit-terraform/issues/224) option implementation. 4. `terraform_validate` in a repo with Terraform module, written using Terraform 0.15+ and which uses provider `configuration_aliases` ([Provider Aliases Within Modules](https://www.terraform.io/language/modules/develop/providers#provider-aliases-within-modules)), errors out. @@ -896,18 +1066,18 @@ To replicate functionality in `terraform_docs` hook: - repo: local hooks: - id: generate-terraform-providers - name: generate-terraform-providers - require_serial: true - entry: .generate-providers.sh - language: script - files: \.tf(vars)?$ - pass_filenames: false + name: generate-terraform-providers + require_serial: true + entry: .generate-providers.sh + language: script + files: \.tf(vars)?$ + pass_filenames: false - repo: https://github.com/pre-commit/pre-commit-hooks - [...] ``` - > Note: The latter method will leave an "aliased-providers.tf.json" file in your repo. You will either want to automate a way to clean this up or add it to your `.gitignore` or both. + > **Tip** + > The latter method will leave an "aliased-providers.tf.json" file in your repo. You will either want to automate a way to clean this up or add it to your `.gitignore` or both. ### terraform_wrapper_module_for_each @@ -984,8 +1154,63 @@ If the generated name is incorrect, set them by providing the `module-repo-short Check [`tfupdate` usage instructions](https://github.com/minamijoyo/tfupdate#usage) for other available options and usage examples. No need to pass `--recursive .` as it is added automatically. +### terragrunt_providers_lock + +> [!TIP] +> Use this hook only in infrastructure repos managed solely by `terragrunt` and do not mix with [`terraform_providers_lock`](#terraform_providers_lock) to avoid conflicts. + +> [!WARNING] +> Hook _may_ be very slow, because terragrunt invokes `t init` under the hood. + +Hook produces same results as [`terraform_providers_lock`](#terraform_providers_lock), but for terragrunt root modules. + +It invokes `terragrunt providers lock` under the hood and terragrunt [does its' own magic](https://terragrunt.gruntwork.io/docs/features/lock-file-handling/) for handling lock files. + + +```yaml +- id: terragrunt_providers_lock + name: Terragrunt providers lock + args: + - --args=-platform=darwin_arm64 + - --args=-platform=darwin_amd64 + - --args=-platform=linux_amd64 +``` + +### terragrunt_validate_inputs + +Validates Terragrunt unused and undefined inputs. This is useful for keeping +configs clean when module versions change or if configs are copied. + +See the [Terragrunt docs](https://terragrunt.gruntwork.io/docs/reference/cli-options/#validate-inputs) for more details. + +Example: + +```yaml +- id: terragrunt_validate_inputs + name: Terragrunt validate inputs + args: + # Optionally check for unused inputs + - --args=--terragrunt-strict-validate +``` + +> [!NOTE] +> This hook requires authentication to a given account if defined by config to work properly. For example, if you use a third-party tool to store AWS credentials like `aws-vault` you must be authenticated first. +> +> See docs for the [iam_role](https://terragrunt.gruntwork.io/docs/reference/config-blocks-and-attributes/#iam_role) attribute and [--terragrunt-iam-role](https://terragrunt.gruntwork.io/docs/reference/cli-options/#terragrunt-iam-role) flag for more. + ## Docker Usage +### About Docker image security + +Pre-built Docker images contain the latest versions of tools available at the time of their build and remain unchanged afterward. Tags should be immutable whenever possible, and it is highly recommended to pin them using hash sums for security and reproducibility. + +This means that most Docker images will include known CVEs, and the longer an image exists, the more CVEs it may accumulate. This applies even to the latest `vX.Y.Z` tags. +To address this, you can use the `nightly` tag, which rebuilds nightly with the latest versions of all dependencies and latest `pre-commit-terraform` hooks. However, using mutable tags introduces different security concerns. + +Note: Currently, we DO NOT test third-party tools or their dependencies for security vulnerabilities, corruption, or injection (including obfuscated content). If you have ideas for introducing image scans or other security improvements, please open an issue or submit a PR. Some ideas are already tracked in [#835](https://github.com/antonbabenko/pre-commit-terraform/issues/835). + +From a security perspective, the best approach is to manage the Docker image yourself and update its dependencies as needed. This allows you to remove unnecessary dependencies, reducing the number of potential CVEs and improving overall security. + ### File Permissions A mismatch between the Docker container's user and the local repository file ownership can cause permission issues in the repository where `pre-commit` is run. The container runs as the `root` user by default, and uses a `tools/entrypoint.sh` script to assume a user ID and group ID if specified by the environment variable `USERID`. @@ -1030,7 +1255,8 @@ machine github.com login ghp_bl481aBlabl481aBla ``` -> **Note**: The value of `GITHUB_SERVER_HOSTNAME` can also refer to a GitHub Enterprise server (i.e. `github.my-enterprise.com`). +> [!TIP] +> The value of `GITHUB_SERVER_HOSTNAME` can also refer to a GitHub Enterprise server (i.e. `github.my-enterprise.com`). Finally, you can execute `docker run` with an additional volume mount so that the `~/.netrc` is accessible within the container @@ -1041,13 +1267,71 @@ Finally, you can execute `docker run` with an additional volume mount so that th docker run --rm -e "USERID=$(id -u):$(id -g)" -v ~/.netrc:/root/.netrc -v $(pwd):/lint -w /lint ghcr.io/antonbabenko/pre-commit-terraform:latest run -a ``` +## GitHub Actions + +You can use this hook in your GitHub Actions workflow together with [pre-commit](https://pre-commit.com). To easy up +dependency management, you can use the managed [docker image](#docker-usage) within your workflow. Make sure to set the +image tag to the version you want to use. + +In this repository's pre-commit [workflow file](.github/workflows/pre-commit.yaml) we run pre-commit without the container image. + +Here's an example using the container image. It includes caching of pre-commit dependencies and utilizes the pre-commit +command to run checks (Note: Fixes will not be automatically pushed back to your branch, even when possible.): + +```yaml +name: pre-commit-terraform + +on: + pull_request: + +jobs: + pre-commit: + runs-on: ubuntu-latest + container: + image: ghcr.io/antonbabenko/pre-commit-terraform:latest # latest used here for simplicity, not recommended + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - run: | + git config --global --add safe.directory $GITHUB_WORKSPACE + git fetch --no-tags --prune --depth=1 origin +refs/heads/*:refs/remotes/origin/* + + - name: Get changed files + id: file_changes + run: | + export DIFF=$(git diff --name-only origin/${{ github.base_ref }} ${{ github.sha }}) + echo "Diff between ${{ github.base_ref }} and ${{ github.sha }}" + echo "files=$( echo "$DIFF" | xargs echo )" >> $GITHUB_OUTPUT + + - name: fix tar dependency in alpine container image + run: | + apk --no-cache add tar + # check python modules installed versions + python -m pip freeze --local + + - name: Cache pre-commit since we use pre-commit from container + uses: actions/cache@v4 + with: + path: ~/.cache/pre-commit + key: pre-commit-3|${{ hashFiles('.pre-commit-config.yaml') }} + + - name: Execute pre-commit + run: | + pre-commit run --color=always --show-diff-on-failure --files ${{ steps.file_changes.outputs.files }} +``` + ## Authors This repository is managed by [Anton Babenko](https://github.com/antonbabenko) with help from these awesome contributors: - - + Contributors @@ -1059,8 +1343,6 @@ This repository is managed by [Anton Babenko](https://github.com/antonbabenko) w - - ## License MIT licensed. See [LICENSE](LICENSE) for full details. @@ -1070,3 +1352,17 @@ MIT licensed. See [LICENSE](LICENSE) for full details. * Russia has [illegally annexed Crimea in 2014](https://en.wikipedia.org/wiki/Annexation_of_Crimea_by_the_Russian_Federation) and [brought the war in Donbas](https://en.wikipedia.org/wiki/War_in_Donbas) followed by [full-scale invasion of Ukraine in 2022](https://en.wikipedia.org/wiki/2022_Russian_invasion_of_Ukraine). * Russia has brought sorrow and devastations to millions of Ukrainians, killed hundreds of innocent people, damaged thousands of buildings, and forced several million people to flee. * [Putin khuylo!](https://en.wikipedia.org/wiki/Putin_khuylo!) + + + +[checkov repo]: https://github.com/bridgecrewio/checkov +[terraform-docs repo]: https://github.com/terraform-docs/terraform-docs +[terragrunt repo]: https://github.com/gruntwork-io/terragrunt +[terrascan repo]: https://github.com/tenable/terrascan +[tflint repo]: https://github.com/terraform-linters/tflint +[tfsec repo]: https://github.com/aquasecurity/tfsec +[trivy repo]: https://github.com/aquasecurity/trivy +[infracost repo]: https://github.com/infracost/infracost +[jq repo]: https://github.com/stedolan/jq +[tfupdate repo]: https://github.com/minamijoyo/tfupdate +[hcledit repo]: https://github.com/minamijoyo/hcledit diff --git a/assets/contributing/enable_actions_in_fork.png b/assets/contributing/enable_actions_in_fork.png new file mode 100644 index 000000000..d39433bea Binary files /dev/null and b/assets/contributing/enable_actions_in_fork.png differ diff --git a/assets/env0.png b/assets/env0.png deleted file mode 100644 index da3eec1e3..000000000 Binary files a/assets/env0.png and /dev/null differ diff --git a/assets/infracost.png b/assets/infracost.png deleted file mode 100644 index bacbd0470..000000000 Binary files a/assets/infracost.png and /dev/null differ diff --git a/assets/pre-commit-terraform-banner.png b/assets/pre-commit-terraform-banner.png new file mode 100644 index 000000000..09b9b0036 Binary files /dev/null and b/assets/pre-commit-terraform-banner.png differ diff --git a/hatch.toml b/hatch.toml new file mode 100644 index 000000000..1cb850f63 --- /dev/null +++ b/hatch.toml @@ -0,0 +1,23 @@ +[build.targets.sdist] +include = [ + '.codecov.yml', + '.coveragerc', + 'src/', + 'tests/', + 'pytest.ini', + 'tox.ini', +] + +[build.targets.wheel] +packages = [ + 'src/pre_commit_terraform/', +] + +[metadata.hooks.vcs.urls] +# FIXME: Uncomment 'Source Archive' as soon as +# FIXME: https://github.com/ofek/hatch-vcs/issues/80 is fixed. +# 'Source Archive' = 'https://github.com/antonbabenko/pre-commit-terraform/archive/{commit_hash}.tar.gz' +'GitHub: repo' = 'https://github.com/antonbabenko/pre-commit-terraform' + +[version] +source = 'vcs' diff --git a/hooks/__init__.py b/hooks/__init__.py deleted file mode 100644 index aeb6f9b27..000000000 --- a/hooks/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -print( - '`terraform_docs_replace` hook is DEPRECATED.' - 'For migration instructions see https://github.com/antonbabenko/pre-commit-terraform/issues/248#issuecomment-1290829226' -) diff --git a/hooks/_common.sh b/hooks/_common.sh index 31be98100..7281b0244 100644 --- a/hooks/_common.sh +++ b/hooks/_common.sh @@ -1,6 +1,22 @@ #!/usr/bin/env bash set -eo pipefail +if [[ $PCT_LOG == trace ]]; then + + echo "BASH path: '$BASH'" + echo "BASH_VERSION: $BASH_VERSION" + echo "BASHOPTS: $BASHOPTS" + echo "OSTYPE: $OSTYPE" + + # ${FUNCNAME[*]} - function calls in reversed order. Each new function call is appended to the beginning + # ${BASH_SOURCE##*/} - get filename + # $LINENO - get line number + export PS4='\e[2m +trace: ${FUNCNAME[*]} + ${BASH_SOURCE##*/}:$LINENO: \e[0m' + + set -x +fi # Hook ID, based on hook filename. # Hook filename MUST BE same with `- id` in .pre-commit-hooks.yaml file # shellcheck disable=SC2034 # Unused var. @@ -36,7 +52,9 @@ function common::initialize { function common::parse_cmdline { # common global arrays. # Populated via `common::parse_cmdline` and can be used inside hooks' functions - ARGS=() HOOK_CONFIG=() FILES=() + ARGS=() + HOOK_CONFIG=() + FILES=() # Used inside `common::terraform_init` function TF_INIT_ARGS=() # Used inside `common::export_provided_env_vars` function @@ -112,7 +130,7 @@ function common::parse_and_export_env_vars { while true; do # Check if at least 1 env var exists in `$arg` # shellcheck disable=SC2016 # '${' should not be expanded - if [[ "$arg" =~ .*'${'[A-Z_][A-Z0-9_]+?'}'.* ]]; then + if [[ "$arg" =~ '${'[A-Z_][A-Za-z0-9_]*'}' ]]; then # Get `ENV_VAR` from `.*${ENV_VAR}.*` local env_var_name=${arg#*$\{} env_var_name=${env_var_name%%\}*} @@ -123,7 +141,7 @@ function common::parse_and_export_env_vars { # `$arg` will be checked in `if` conditional, `$ARGS` will be used in the next functions. # shellcheck disable=SC2016 # '${' should not be expanded arg=${arg/'${'$env_var_name'}'/$env_var_value} - ARGS[$arg_idx]=$arg + ARGS[arg_idx]=$arg # shellcheck disable=SC2016 # '${' should not be expanded common::colorify "green" 'After ${'"$env_var_name"'} expansion: '"'$arg'\n" continue @@ -170,6 +188,87 @@ function common::is_hook_run_on_whole_repo { fi } +####################################################################### +# Get the number of CPU logical cores available for pre-commit to use +# +# CPU quota should be calculated as `cpu.cfs_quota_us / cpu.cfs_period_us` +# For K8s see: https://docs.kernel.org/scheduler/sched-bwc.html +# For Docker see: https://docs.docker.com/engine/containers/resource_constraints/#configure-the-default-cfs-scheduler +# +# Arguments: +# parallelism_ci_cpu_cores (string) Used in edge cases when number of +# CPU cores can't be derived automatically +# Outputs: +# Returns number of CPU logical cores, rounded down to nearest integer +####################################################################### +function common::get_cpu_num { + local -r parallelism_ci_cpu_cores=$1 + + local cpu_quota cpu_period cpu_num + + local -r wslinterop_path="/proc/sys/fs/binfmt_misc/WSLInterop" + + if [[ -f /sys/fs/cgroup/cpu/cpu.cfs_quota_us && + (! -f "${wslinterop_path}" && ! -f "${wslinterop_path}-late" && ! -f "/run/WSL") ]]; then # WSL has cfs_quota_us, but WSL should be checked as usual Linux host + # Inside K8s pod or DinD in K8s + cpu_quota=$(< /sys/fs/cgroup/cpu/cpu.cfs_quota_us) + cpu_period=$(cat /sys/fs/cgroup/cpu/cpu.cfs_period_us 2> /dev/null || echo "$cpu_quota") + + if [[ $cpu_quota -eq -1 || $cpu_period -lt 1 ]]; then + # K8s no limits or in DinD + if [[ -n $parallelism_ci_cpu_cores ]]; then + if [[ ! $parallelism_ci_cpu_cores =~ ^[[:digit:]]+$ ]]; then + common::colorify "yellow" "--parallelism-ci-cpu-cores set to" \ + "'$parallelism_ci_cpu_cores' which is not a positive integer.\n" \ + "To avoid possible harm, parallelism is disabled.\n" \ + "To re-enable it, change corresponding value in config to positive integer" + + echo 1 + return + fi + + echo "$parallelism_ci_cpu_cores" + return + fi + + common::colorify "yellow" "Unable to derive number of available CPU cores.\n" \ + "Running inside K8s pod without limits or inside DinD without limits propagation.\n" \ + "To avoid possible harm, parallelism is disabled.\n" \ + "To re-enable it, set corresponding limits, or set the following for the current hook:\n" \ + " args:\n" \ + " - --hook-config=--parallelism-ci-cpu-cores=N\n" \ + "where N is the number of CPU cores to allocate to pre-commit." + + echo 1 + return + fi + + cpu_num=$((cpu_quota / cpu_period)) + [[ $cpu_num -lt 1 ]] && echo 1 || echo $cpu_num + return + fi + + if [[ -f /sys/fs/cgroup/cpu.max ]]; then + # Inside Linux (Docker?) container + cpu_quota=$(cut -d' ' -f1 /sys/fs/cgroup/cpu.max) + cpu_period=$(cut -d' ' -f2 /sys/fs/cgroup/cpu.max) + + if [[ $cpu_quota == max || $cpu_period -lt 1 ]]; then + # No limits + nproc 2> /dev/null || echo 1 + return + fi + + cpu_num=$((cpu_quota / cpu_period)) + [[ $cpu_num -lt 1 ]] && echo 1 || echo $cpu_num + return + fi + + # On host machine or any other case + # `nproc` - Linux/FreeBSD/WSL, `sysctl -n hw.ncpu` - macOS/BSD, `echo 1` - fallback + nproc 2> /dev/null || sysctl -n hw.ncpu 2> /dev/null || echo 1 +} + ####################################################################### # Hook execution boilerplate logic which is common to hooks, that run # on per dir basis. @@ -198,6 +297,8 @@ function common::per_dir_hook { # despite there's only one positional ARG left local -a -r files=("$@") + local -r tf_path=$(common::get_tf_binary_path) + # check is (optional) function defined if [ "$(type -t run_hook_on_whole_repo)" == function ] && # check is hook run via `pre-commit run --all` @@ -219,10 +320,16 @@ function common::per_dir_hook { # Lookup hook-config for modifiers that impact common behavior local change_dir_in_unique_part=false + + local parallelism_limit IFS=";" read -r -a configs <<< "${HOOK_CONFIG[*]}" for c in "${configs[@]}"; do IFS="=" read -r -a config <<< "$c" - key=${config[0]} + + # $hook_config receives string like '--foo=bar; --baz=4;' etc. + # It gets split by `;` into array, which we're parsing here ('--foo=bar' ' --baz=4') + # Next line removes leading spaces, to support >1 `--hook-config` args + key="${config[0]## }" value=${config[1]} case $key in @@ -233,32 +340,84 @@ function common::per_dir_hook { change_dir_in_unique_part="delegate_chdir" fi ;; + --parallelism-limit) + # this flag will limit the number of parallel processes + parallelism_limit="$value" + ;; + --parallelism-ci-cpu-cores) + # Used in edge cases when number of CPU cores can't be derived automatically + parallelism_ci_cpu_cores="$value" + ;; esac done + CPU=$(common::get_cpu_num "$parallelism_ci_cpu_cores") + # parallelism_limit can include reference to 'CPU' variable + local parallelism_disabled=false + + if [[ ! $parallelism_limit ]]; then + # Could evaluate to 0 + parallelism_limit=$((CPU - 1)) + elif [[ $parallelism_limit -eq 1 ]]; then + parallelism_disabled=true + else + # Could evaluate to <1 + parallelism_limit=$((parallelism_limit)) + fi + + if [[ $parallelism_limit -lt 1 ]]; then + # Suppress warning for edge cases when only 1 CPU available or + # when `--parallelism-ci-cpu-cores=1` and `--parallelism_limit` unset + if [[ $CPU -ne 1 ]]; then + + common::colorify "yellow" "Observed Parallelism limit '$parallelism_limit'." \ + "To avoid possible harm, parallelism set to '1'" + fi + + parallelism_limit=1 + parallelism_disabled=true + fi + + local pids=() + + # shellcheck disable=SC2207 # More readable way + local -a dir_paths_unique=($(printf '%s\n' "${dir_paths[@]}" | sort -u)) + + local length=${#dir_paths_unique[@]} + local last_index=$((${#dir_paths_unique[@]} - 1)) + + local final_exit_code=0 # preserve errexit status shopt -qo errexit && ERREXIT_IS_SET=true # allow hook to continue if exit_code is greater than 0 set +e - local final_exit_code=0 - - # run hook for each path - for dir_path in $(echo "${dir_paths[*]}" | tr ' ' '\n' | sort -u); do - dir_path="${dir_path//__REPLACED__SPACE__/ }" + # run hook for each path in parallel + for ((i = 0; i < length; i++)); do + dir_path="${dir_paths_unique[$i]//__REPLACED__SPACE__/ }" + { + if [[ $change_dir_in_unique_part == false ]]; then + pushd "$dir_path" > /dev/null + fi - if [[ $change_dir_in_unique_part == false ]]; then - pushd "$dir_path" > /dev/null || continue - fi + per_dir_hook_unique_part "$dir_path" "$change_dir_in_unique_part" "$parallelism_disabled" "$tf_path" "${args[@]}" + } & + pids+=("$!") - per_dir_hook_unique_part "$dir_path" "$change_dir_in_unique_part" "${args[@]}" + if [[ $parallelism_disabled == true ]] || + [[ $i -ne 0 && $((i % parallelism_limit)) -eq 0 ]] || # don't stop on first iteration when parallelism_limit>1 + [[ $i -eq $last_index ]]; then - local exit_code=$? - if [ $exit_code -ne 0 ]; then - final_exit_code=$exit_code - fi + for pid in "${pids[@]}"; do + # Get the exit code from the background process + local exit_code=0 + wait "$pid" || exit_code=$? - if [[ $change_dir_in_unique_part == false ]]; then - popd > /dev/null + if [ $exit_code -ne 0 ]; then + final_exit_code=$exit_code + fi + done + # Reset pids for next iteration + unset pids fi done @@ -291,14 +450,67 @@ function common::colorify { # Params start # local COLOR="${!1}" - local -r TEXT=$2 + shift + local -r TEXT="$*" # Params end # if [ "$PRE_COMMIT_COLOR" = "never" ]; then COLOR=$RESET fi - echo -e "${COLOR}${TEXT}${RESET}" + echo -e "${COLOR}${TEXT}${RESET}" >&2 +} + +####################################################################### +# Get Terraform/OpenTofu binary path +# Allows user to set the path to custom Terraform or OpenTofu binary +# Globals (init and populate): +# HOOK_CONFIG (array) arguments that configure hook behavior +# PCT_TFPATH (string) user defined env var with path to Terraform/OpenTofu binary +# TERRAGRUNT_TFPATH (string) user defined env var with path to Terraform/OpenTofu binary +# Outputs: +# If failed - exit 1 with error message about missing Terraform/OpenTofu binary +####################################################################### +function common::get_tf_binary_path { + local hook_config_tf_path + + for config in "${HOOK_CONFIG[@]}"; do + if [[ $config == --tf-path=* ]]; then + hook_config_tf_path=${config#*=} + hook_config_tf_path=${hook_config_tf_path%;} + break + fi + done + + # direct hook config, has the highest precedence + if [[ $hook_config_tf_path ]]; then + echo "$hook_config_tf_path" + return + + # environment variable + elif [[ $PCT_TFPATH ]]; then + echo "$PCT_TFPATH" + return + + # Maybe there is a similar setting for Terragrunt already + elif [[ $TERRAGRUNT_TFPATH ]]; then + echo "$TERRAGRUNT_TFPATH" + return + + # check if Terraform binary is available + elif command -v terraform &> /dev/null; then + command -v terraform + return + + # finally, check if Tofu binary is available + elif command -v tofu &> /dev/null; then + command -v tofu + return + + else + common::colorify "red" "Neither Terraform nor OpenTofu binary could be found. Please either set the \"--tf-path\" hook configuration argument, or set the \"PCT_TFPATH\" environment variable, or set the \"TERRAGRUNT_TFPATH\" environment variable, or install Terraform or OpenTofu globally." + exit 1 + fi } ####################################################################### @@ -307,8 +519,12 @@ function common::colorify { # command_name (string) command that will tun after successful init # dir_path (string) PATH to dir relative to git repo root. # Can be used in error logging +# parallelism_disabled (bool) if true - skip lock mechanism +# tf_path (string) PATH to Terraform/OpenTofu binary # Globals (init and populate): # TF_INIT_ARGS (array) arguments for `terraform init` command +# TF_PLUGIN_CACHE_DIR (string) user defined env var with name of the directory +# which can't be R/W concurrently # Outputs: # If failed - print out terraform init output ####################################################################### @@ -316,6 +532,8 @@ function common::colorify { function common::terraform_init { local -r command_name=$1 local -r dir_path=$2 + local -r parallelism_disabled=$3 + local -r tf_path=$4 local exit_code=0 local init_output @@ -325,9 +543,32 @@ function common::terraform_init { TF_INIT_ARGS+=("-no-color") fi - if [ ! -d .terraform/modules ] || [ ! -d .terraform/providers ]; then - init_output=$(terraform init -backend=false "${TF_INIT_ARGS[@]}" 2>&1) - exit_code=$? + recreate_modules=$([[ ! -d .terraform/modules ]] && echo true || echo false) + recreate_providers=$([[ ! -d .terraform/providers ]] && echo true || echo false) + + if [[ $recreate_modules == true || $recreate_providers == true ]]; then + # Plugin cache dir can't be written concurrently or read during write + # https://github.com/hashicorp/terraform/issues/31964 + if [[ -z $TF_PLUGIN_CACHE_DIR || $parallelism_disabled == true ]]; then + init_output=$("$tf_path" init -backend=false "${TF_INIT_ARGS[@]}" 2>&1) + exit_code=$? + else + # Locking just doesn't work, and the below works quicker instead. Details: + # https://github.com/hashicorp/terraform/issues/31964#issuecomment-1939869453 + for i in {1..10}; do + init_output=$("$tf_path" init -backend=false "${TF_INIT_ARGS[@]}" 2>&1) + exit_code=$? + + if [ $exit_code -eq 0 ]; then + break + fi + sleep 1 + + common::colorify "green" "Race condition detected. Retrying 'terraform init' command [retry $i]: $dir_path." + [[ $recreate_modules == true ]] && rm -rf .terraform/modules + [[ $recreate_providers == true ]] && rm -rf .terraform/providers + done + fi if [ $exit_code -ne 0 ]; then common::colorify "red" "'terraform init' failed, '$command_name' skipped: $dir_path" @@ -356,7 +597,43 @@ function common::export_provided_env_vars { for var in "${env_vars[@]}"; do var_name="${var%%=*}" var_value="${var#*=}" + # Drop enclosing double quotes + if [[ $var_value =~ ^\" && $var_value =~ \"$ ]]; then + var_value="${var_value#\"}" + var_value="${var_value%\"}" + fi # shellcheck disable=SC2086 export $var_name="$var_value" done } + +####################################################################### +# Check if the installed Terragrunt version is >=0.78.0 or not +# +# This function helps to determine which terragrunt subcomand to use +# based on Terragrunt version +# +# Returns: +# - 0 if version >= 0.78.0 +# - 1 if version < 0.78.0 +# Defaults to 0 if version cannot be determined +####################################################################### +# TODO: Drop after May 2027. Two years to upgrade is more than enough. +function common::terragrunt_version_ge_0.78 { + local terragrunt_version + + # Extract version number (e.g., "terragrunt version v0.80.4" -> "0.80") + terragrunt_version=$(terragrunt --version 2> /dev/null | grep -oE '[0-9]+\.[0-9]+') + # If we can't parse version, default to newer command + [[ ! $terragrunt_version ]] && return 0 + + local major minor + IFS='.' read -r major minor <<< "$terragrunt_version" + + # New subcommands added in v0.78.0 (May 2025) + if [[ $major -gt 0 || ($major -eq 0 && $minor -ge 78) ]]; then + return 0 + else + return 1 + fi +} diff --git a/hooks/infracost_breakdown.sh b/hooks/infracost_breakdown.sh index 551579112..14d34b82f 100755 --- a/hooks/infracost_breakdown.sh +++ b/hooks/infracost_breakdown.sh @@ -2,8 +2,8 @@ set -eo pipefail # globals variables -# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines -readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR # shellcheck source=_common.sh . "$SCRIPT_DIR/_common.sh" @@ -70,19 +70,24 @@ function infracost_breakdown_ { # -h .totalHourlyCost > 0.1 # --hook-config=.currency == "USD" first_char=${check:0:1} - last_char=${check: -1} + last_char=${check:$((${#check} - 1)):1} if [ "$first_char" == "$last_char" ] && { [ "$first_char" == '"' ] || [ "$first_char" == "'" ] }; then - check="${check:1:-1}" + check="${check:1:$((${#check} - 2))}" fi - mapfile -t operations < <(echo "$check" | grep -oE '[!<>=]{1,2}') + # Replace mapfile with while read loop for bash 3.2 compatibility + operations=() + while IFS= read -r line; do + operations+=("$line") + done < <(echo "$check" | grep -oE '[!<>=]{1,2}') + # Get the very last operator, that is used in comparison inside `jq` query. # From the example below we need to pick the `>` which is in between `add` and `1000`, # but not the `!=`, which goes earlier in the `jq` expression # [.projects[].diff.totalMonthlyCost | select (.!=null) | tonumber] | add > 1000 - operation=${operations[-1]} + operation=${operations[$((${#operations[@]} - 1))]} IFS="$operation" read -r -a jq_check <<< "$check" real_value="$(jq "${jq_check[0]}" <<< "$RESULTS")" diff --git a/hooks/terraform_checkov.sh b/hooks/terraform_checkov.sh index 03ad208c6..01e2f2454 100755 --- a/hooks/terraform_checkov.sh +++ b/hooks/terraform_checkov.sh @@ -2,8 +2,8 @@ set -eo pipefail # globals variables -# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines -readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR # shellcheck source=_common.sh . "$SCRIPT_DIR/_common.sh" @@ -34,7 +34,9 @@ function main { # change_dir_in_unique_part (string/false) Modifier which creates # possibilities to use non-common chdir strategies. # Availability depends on hook. +# parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior +# tf_path (string) PATH to Terraform/OpenTofu binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -43,7 +45,11 @@ function per_dir_hook_unique_part { local -r dir_path="$1" # shellcheck disable=SC2034 # Unused var. local -r change_dir_in_unique_part="$2" - shift 2 + # shellcheck disable=SC2034 # Unused var. + local -r parallelism_disabled="$3" + # shellcheck disable=SC2034 # Unused var. + local -r tf_path="$4" + shift 4 local -a -r args=("$@") checkov -d . "${args[@]}" diff --git a/hooks/terraform_docs.sh b/hooks/terraform_docs.sh index c597730b0..02d012f83 100755 --- a/hooks/terraform_docs.sh +++ b/hooks/terraform_docs.sh @@ -2,19 +2,18 @@ set -eo pipefail # globals variables -# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines -readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR # shellcheck source=_common.sh . "$SCRIPT_DIR/_common.sh" -# set up default insertion markers. These will be changed to the markers used by -# terraform-docs if the hook config contains `--use-standard-markers=true` -insertion_marker_begin="" -insertion_marker_end="" +insertion_marker_begin="" +insertion_marker_end="" +doc_header="# " -# these are the standard insertion markers used by terraform-docs -readonly standard_insertion_marker_begin="" -readonly standard_insertion_marker_end="" +# Old markers used by the hook before the introduction of the terraform-docs markers +readonly old_insertion_marker_begin="" +readonly old_insertion_marker_end="" function main { common::initialize "$SCRIPT_DIR" @@ -26,80 +25,48 @@ function main { ARGS[i]=${ARGS[i]/--config=/--config=$(pwd)\/} done # shellcheck disable=SC2153 # False positive - terraform_docs_ "${HOOK_CONFIG[*]}" "${ARGS[*]}" "${FILES[@]}" + terraform_docs "${HOOK_CONFIG[*]}" "${ARGS[*]}" "${FILES[@]}" } ####################################################################### -# Function which prepares hacks for old versions of `terraform` and -# `terraform-docs` that them call `terraform_docs` +# Function to replace old markers with new markers affected files +# Globals: +# insertion_marker_begin - Standard insertion marker at beginning +# insertion_marker_end - Standard insertion marker at the end +# old_insertion_marker_begin - Old insertion marker at beginning +# old_insertion_marker_end - Old insertion marker at the end # Arguments: -# hook_config (string with array) arguments that configure hook behavior -# args (string with array) arguments that configure wrapped tool behavior -# files (array) filenames to check +# file (string) filename to check ####################################################################### -function terraform_docs_ { - local -r hook_config="$1" - local -r args="$2" - shift 2 - local -a -r files=("$@") - - # Get hook settings - IFS=";" read -r -a configs <<< "$hook_config" +function replace_old_markers { + local -r file=$1 - local hack_terraform_docs - hack_terraform_docs=$(terraform version | sed -n 1p | grep -c 0.12) || true - - if [[ ! $(command -v terraform-docs) ]]; then - echo "ERROR: terraform-docs is required by terraform_docs pre-commit hook but is not installed or in the system's PATH." - exit 1 - fi - - local is_old_terraform_docs - is_old_terraform_docs=$(terraform-docs version | grep -o "v0.[1-7]\." | tail -1) || true - - if [[ -z "$is_old_terraform_docs" ]]; then # Using terraform-docs 0.8+ (preferred) - - terraform_docs "0" "${configs[*]}" "$args" "${files[@]}" - - elif [[ "$hack_terraform_docs" == "1" ]]; then # Using awk script because terraform-docs is older than 0.8 and terraform 0.12 is used - - if [[ ! $(command -v awk) ]]; then - echo "ERROR: awk is required for terraform-docs hack to work with Terraform 0.12." - exit 1 - fi - - local tmp_file_awk - tmp_file_awk=$(mktemp "${TMPDIR:-/tmp}/terraform-docs-XXXXXXXXXX") - terraform_docs_awk "$tmp_file_awk" - terraform_docs "$tmp_file_awk" "${configs[*]}" "$args" "${files[@]}" - rm -f "$tmp_file_awk" - - else # Using terraform 0.11 and no awk script is needed for that - - terraform_docs "0" "${configs[*]}" "$args" "${files[@]}" - - fi + # Determine the appropriate sed command based on the operating system (GNU sed or BSD sed) + sed --version &> /dev/null && SED_CMD=(sed -i) || SED_CMD=(sed -i '') + "${SED_CMD[@]}" -e "s/^${old_insertion_marker_begin}$/${insertion_marker_begin//\//\\/}/" "$file" + "${SED_CMD[@]}" -e "s/^${old_insertion_marker_end}$/${insertion_marker_end//\//\\/}/" "$file" } ####################################################################### -# Wrapper around `terraform-docs` tool that check and change/create -# (depends on provided hook_config) terraform documentation in -# markdown format +# Wrapper around `terraform-docs` tool that checks and changes/creates +# (depending on provided hook_config) terraform documentation in +# Markdown # Arguments: -# terraform_docs_awk_file (string) filename where awk hack for old -# `terraform-docs` was written. Needed for TF 0.12+. -# Hack skipped when `terraform_docs_awk_file == "0"` # hook_config (string with array) arguments that configure hook behavior # args (string with array) arguments that configure wrapped tool behavior # files (array) filenames to check ####################################################################### function terraform_docs { - local -r terraform_docs_awk_file="$1" - local -r hook_config="$2" - local args="$3" - shift 3 + local -r hook_config="$1" + local args="$2" + shift 2 local -a -r files=("$@") + if [[ ! $(command -v terraform-docs) ]]; then + echo "ERROR: terraform-docs is required by terraform_docs pre-commit hook but is not installed or in the system's PATH." + exit 1 + fi + local -a paths local index=0 @@ -112,27 +79,32 @@ function terraform_docs { ((index += 1)) done - local -r tmp_file=$(mktemp) - # # Get hook settings # - local text_file="README.md" + local output_file="README.md" + local output_mode="inject" + local use_path_to_file=false local add_to_existing=false local create_if_not_exist=false - local use_standard_markers=false + local use_standard_markers=true + local have_config_flag=false - read -r -a configs <<< "$hook_config" + IFS=";" read -r -a configs <<< "$hook_config" for c in "${configs[@]}"; do IFS="=" read -r -a config <<< "$c" - key=${config[0]} + # $hook_config receives string like '--foo=bar; --baz=4;' etc. + # It gets split by `;` into array, which we're parsing here ('--foo=bar' ' --baz=4') + # Next line removes leading spaces, to support >1 `--hook-config` args + key="${config[0]## }" value=${config[1]} case $key in --path-to-file) - text_file=$value + output_file=$value + use_path_to_file=true ;; --add-to-existing-file) add_to_existing=$value @@ -142,27 +114,75 @@ function terraform_docs { ;; --use-standard-markers) use_standard_markers=$value + common::colorify "yellow" "WARNING: --use-standard-markers is deprecated and will be removed in the future." + common::colorify "yellow" " All needed changes already done by the hook, feel free to remove --use-standard-markers setting from your pre-commit config" + ;; + --custom-marker-begin) + insertion_marker_begin=$value + common::colorify "green" "INFO: --custom-marker-begin is used and the marker is set to \"$value\"." + ;; + --custom-marker-end) + insertion_marker_end=$value + common::colorify "green" "INFO: --custom-marker-end is used and the marker is set to \"$value\"." + ;; + --custom-doc-header) + doc_header=$value + common::colorify "green" "INFO: --custom-doc-header is used and the doc header is set to \"$value\"." ;; esac done - if [ "$use_standard_markers" = true ]; then - # update the insertion markers to those used by terraform-docs - insertion_marker_begin="$standard_insertion_marker_begin" - insertion_marker_end="$standard_insertion_marker_end" + if [[ $use_standard_markers == false ]]; then + # update the insertion markers to those used by pre-commit-terraform before v1.93 + insertion_marker_begin="$old_insertion_marker_begin" + insertion_marker_end="$old_insertion_marker_end" fi # Override formatter if no config file set if [[ "$args" != *"--config"* ]]; then local tf_docs_formatter="md" - # Suppress terraform_docs color else + have_config_flag=true + # Enable extended pattern matching operators + shopt -qp extglob || EXTGLOB_IS_NOT_SET=true && shopt -s extglob + # Trim any args before the `--config` arg value + local config_file=${args##*--config@(+([[:space:]])|=)} + # Trim any trailing spaces and args (if any) + config_file="${config_file%%+([[:space:]])?(--*)}" + # Trim `--config` arg and its value from original args as we will + # pass `--config` separately to allow whitespaces in its value + args=${args/--config@(+([[:space:]])|=)$config_file*([[:space:]])/} + # Restore state of `extglob` if we changed it + [[ $EXTGLOB_IS_NOT_SET ]] && shopt -u extglob + + # Prioritize `.terraform-docs.yml` `output.file` over + # `--hook-config=--path-to-file=` if it set + local config_output_file + # Get latest non-commented `output.file` from `.terraform-docs.yml` + config_output_file=$(grep -A1000 -e '^output:$' "$config_file" 2> /dev/null | grep -E '^[[:space:]]+file:' | tail -n 1) || true + + if [[ $config_output_file ]]; then + # Extract filename from `output.file` line + config_output_file=$(echo "$config_output_file" | awk -F':' '{print $2}' | tr -d '[:space:]"' | tr -d "'") + + if [[ $use_path_to_file == true && "$config_output_file" != "$output_file" ]]; then + common::colorify "yellow" "NOTE: You set both '--hook-config=--path-to-file=$output_file' and 'output.file: $config_output_file' in '$config_file'" + common::colorify "yellow" " 'output.file' from '$config_file' will be used." + fi - local config_file=${args#*--config} - config_file=${config_file#*=} - config_file=${config_file% *} + output_file=$config_output_file + fi + + # Use `.terraform-docs.yml` `output.mode` if it set + local config_output_mode + config_output_mode=$(grep -A1000 -e '^output:$' "$config_file" 2> /dev/null | grep -E '^[[:space:]]+mode:' | tail -n 1) || true + if [[ $config_output_mode ]]; then + # Extract mode from `output.mode` line + output_mode=$(echo "$config_output_mode" | awk -F':' '{print $2}' | tr -d '[:space:]"' | tr -d "'") + fi + # Suppress terraform_docs color local config_file_no_color config_file_no_color="$config_file$(date +%s).yml" @@ -185,71 +205,57 @@ function terraform_docs { # # Create file if it not exist and `--create-if-not-exist=true` provided # - if $create_if_not_exist && [[ ! -f "$text_file" ]]; then + if $create_if_not_exist && [[ ! -f "$output_file" ]]; then dir_have_tf_files="$( - find . -maxdepth 1 -type f | sed 's|.*\.||' | sort -u | grep -oE '^tf$|^tfvars$' || + find . -maxdepth 1 \( -name '*.tf' -o -name '*.tofu' -o -name '*.tfvars' \) -type f || exit 0 )" # if no TF files - skip dir [ ! "$dir_have_tf_files" ] && popd > /dev/null && continue - dir="$(dirname "$text_file")" + dir="$(dirname "$output_file")" mkdir -p "$dir" # Use of insertion markers, where there is no existing README file { - echo -e "# ${PWD##*/}\n" + echo -e "${doc_header}${PWD##*/}\n" echo "$insertion_marker_begin" echo "$insertion_marker_end" - } >> "$text_file" + } >> "$output_file" fi # If file still not exist - skip dir - [[ ! -f "$text_file" ]] && popd > /dev/null && continue + [[ ! -f "$output_file" ]] && popd > /dev/null && continue + + replace_old_markers "$output_file" # - # If `--add-to-existing-file=true` set, check is in file exist "hook markers", - # and if not - append "hook markers" to the end of file. + # If `--add-to-existing-file=false` (default behavior), check if "hook markers" exist in file, + # and, if not, skip execution to avoid addition of terraform-docs section, as + # terraform-docs in 'inject' mode adds markers by default if they are not present # - if $add_to_existing; then - HAVE_MARKER=$(grep -o "$insertion_marker_begin" "$text_file" || exit 0) - - if [ ! "$HAVE_MARKER" ]; then - # Use of insertion markers, where addToExisting=true, with no markers in the existing file - echo "$insertion_marker_begin" >> "$text_file" - echo "$insertion_marker_end" >> "$text_file" - fi + if [[ $add_to_existing == false ]]; then + have_marker=$(grep -o "$insertion_marker_begin" "$output_file") || unset have_marker + [[ ! $have_marker ]] && popd > /dev/null && continue fi - if [[ "$terraform_docs_awk_file" == "0" ]]; then - # shellcheck disable=SC2086 - terraform-docs $tf_docs_formatter $args ./ > "$tmp_file" + # shellcheck disable=SC2206 + # Need to pass $tf_docs_formatter and $args as separate arguments, not as single string + local tfdocs_cmd=( + terraform-docs + --output-mode="$output_mode" + --output-file="$output_file" + $tf_docs_formatter + $args + ) + if [[ $have_config_flag == true ]]; then + "${tfdocs_cmd[@]}" "--config=$config_file" ./ > /dev/null else - # Can't append extension for mktemp, so renaming instead - local tmp_file_docs - tmp_file_docs=$(mktemp "${TMPDIR:-/tmp}/terraform-docs-XXXXXXXXXX") - mv "$tmp_file_docs" "$tmp_file_docs.tf" - local tmp_file_docs_tf - tmp_file_docs_tf="$tmp_file_docs.tf" - - awk -f "$terraform_docs_awk_file" ./*.tf > "$tmp_file_docs_tf" - # shellcheck disable=SC2086 - terraform-docs $tf_docs_formatter $args "$tmp_file_docs_tf" > "$tmp_file" - rm -f "$tmp_file_docs_tf" + "${tfdocs_cmd[@]}" ./ > /dev/null fi - # Use of insertion markers to insert the terraform-docs output between the markers - # Replace content between markers with the placeholder - https://stackoverflow.com/questions/1212799/how-do-i-extract-lines-between-two-line-delimiters-in-perl#1212834 - perl_expression="if (/$insertion_marker_begin/../$insertion_marker_end/) { print \$_ if /$insertion_marker_begin/; print \"I_WANT_TO_BE_REPLACED\\n\$_\" if /$insertion_marker_end/;} else { print \$_ }" - perl -i -ne "$perl_expression" "$text_file" - - # Replace placeholder with the content of the file - perl -i -e 'open(F, "'"$tmp_file"'"); $f = join "", ; while(<>){if (/I_WANT_TO_BE_REPLACED/) {print $f} else {print $_};}' "$text_file" - - rm -f "$tmp_file" - popd > /dev/null done @@ -257,169 +263,4 @@ function terraform_docs { rm -f "$config_file_no_color" } -####################################################################### -# Function which creates file with `awk` hacks for old versions of -# `terraform-docs` -# Arguments: -# output_file (string) filename where hack will be written to -####################################################################### -function terraform_docs_awk { - local -r output_file=$1 - - cat << "EOF" > "$output_file" -# This script converts Terraform 0.12 variables/outputs to something suitable for `terraform-docs` -# As of terraform-docs v0.6.0, HCL2 is not supported. This script is a *dirty hack* to get around it. -# https://github.com/terraform-docs/terraform-docs/ -# https://github.com/terraform-docs/terraform-docs/issues/62 -# Script was originally found here: https://github.com/cloudposse/build-harness/blob/master/bin/terraform-docs.awk -{ - if ( $0 ~ /\{/ ) { - braceCnt++ - } - if ( $0 ~ /\}/ ) { - braceCnt-- - } - # ---------------------------------------------------------------------------------------------- - # variable|output "..." { - # ---------------------------------------------------------------------------------------------- - # [END] variable/output block - if (blockCnt > 0 && blockTypeCnt == 0 && blockDefaultCnt == 0) { - if (braceCnt == 0 && blockCnt > 0) { - blockCnt-- - print $0 - } - } - # [START] variable or output block started - if ($0 ~ /^[[:space:]]*(variable|output)[[:space:]][[:space:]]*"(.*?)"/) { - # Normalize the braceCnt and block (should be 1 now) - braceCnt = 1 - blockCnt = 1 - # [CLOSE] "default" and "type" block - blockDefaultCnt = 0 - blockTypeCnt = 0 - # Print variable|output line - print $0 - } - # ---------------------------------------------------------------------------------------------- - # default = ... - # ---------------------------------------------------------------------------------------------- - # [END] multiline "default" continues/ends - if (blockCnt > 0 && blockTypeCnt == 0 && blockDefaultCnt > 0) { - print $0 - # Count opening blocks - blockDefaultCnt += gsub(/\(/, "") - blockDefaultCnt += gsub(/\[/, "") - blockDefaultCnt += gsub(/\{/, "") - # Count closing blocks - blockDefaultCnt -= gsub(/\)/, "") - blockDefaultCnt -= gsub(/\]/, "") - blockDefaultCnt -= gsub(/\}/, "") - } - # [START] multiline "default" statement started - if (blockCnt > 0 && blockTypeCnt == 0 && blockDefaultCnt == 0) { - if ($0 ~ /^[[:space:]][[:space:]]*(default)[[:space:]][[:space:]]*=/) { - if ($3 ~ "null") { - print " default = \"null\"" - } else { - print $0 - # Count opening blocks - blockDefaultCnt += gsub(/\(/, "") - blockDefaultCnt += gsub(/\[/, "") - blockDefaultCnt += gsub(/\{/, "") - # Count closing blocks - blockDefaultCnt -= gsub(/\)/, "") - blockDefaultCnt -= gsub(/\]/, "") - blockDefaultCnt -= gsub(/\}/, "") - } - } - } - # ---------------------------------------------------------------------------------------------- - # type = ... - # ---------------------------------------------------------------------------------------------- - # [END] multiline "type" continues/ends - if (blockCnt > 0 && blockTypeCnt > 0 && blockDefaultCnt == 0) { - # The following 'print $0' would print multiline type definitions - #print $0 - # Count opening blocks - blockTypeCnt += gsub(/\(/, "") - blockTypeCnt += gsub(/\[/, "") - blockTypeCnt += gsub(/\{/, "") - # Count closing blocks - blockTypeCnt -= gsub(/\)/, "") - blockTypeCnt -= gsub(/\]/, "") - blockTypeCnt -= gsub(/\}/, "") - } - # [START] multiline "type" statement started - if (blockCnt > 0 && blockTypeCnt == 0 && blockDefaultCnt == 0) { - if ($0 ~ /^[[:space:]][[:space:]]*(type)[[:space:]][[:space:]]*=/ ) { - if ($3 ~ "object") { - print " type = \"object\"" - } else { - # Convert multiline stuff into single line - if ($3 ~ /^[[:space:]]*list[[:space:]]*\([[:space:]]*$/) { - type = "list" - } else if ($3 ~ /^[[:space:]]*string[[:space:]]*\([[:space:]]*$/) { - type = "string" - } else if ($3 ~ /^[[:space:]]*map[[:space:]]*\([[:space:]]*$/) { - type = "map" - } else { - type = $3 - } - # legacy quoted types: "string", "list", and "map" - if (type ~ /^[[:space:]]*"(.*?)"[[:space:]]*$/) { - print " type = " type - } else { - print " type = \"" type "\"" - } - } - # Count opening blocks - blockTypeCnt += gsub(/\(/, "") - blockTypeCnt += gsub(/\[/, "") - blockTypeCnt += gsub(/\{/, "") - # Count closing blocks - blockTypeCnt -= gsub(/\)/, "") - blockTypeCnt -= gsub(/\]/, "") - blockTypeCnt -= gsub(/\}/, "") - } - } - # ---------------------------------------------------------------------------------------------- - # description = ... - # ---------------------------------------------------------------------------------------------- - # [PRINT] single line "description" - if (blockCnt > 0 && blockTypeCnt == 0 && blockDefaultCnt == 0) { - if ($0 ~ /^[[:space:]][[:space:]]*description[[:space:]][[:space:]]*=/) { - print $0 - } - } - # ---------------------------------------------------------------------------------------------- - # value = ... - # ---------------------------------------------------------------------------------------------- - ## [PRINT] single line "value" - #if (blockCnt > 0 && blockTypeCnt == 0 && blockDefaultCnt == 0) { - # if ($0 ~ /^[[:space:]][[:space:]]*value[[:space:]][[:space:]]*=/) { - # print $0 - # } - #} - # ---------------------------------------------------------------------------------------------- - # Newlines, comments, everything else - # ---------------------------------------------------------------------------------------------- - #if (blockTypeCnt == 0 && blockDefaultCnt == 0) { - # Comments with '#' - if ($0 ~ /^[[:space:]]*#/) { - print $0 - } - # Comments with '//' - if ($0 ~ /^[[:space:]]*\/\//) { - print $0 - } - # Newlines - if ($0 ~ /^[[:space:]]*$/) { - print $0 - } - #} -} -EOF - -} - [ "${BASH_SOURCE[0]}" != "$0" ] || main "$@" diff --git a/hooks/terraform_docs_replace.py b/hooks/terraform_docs_replace.py deleted file mode 100644 index a9cf6c9bc..000000000 --- a/hooks/terraform_docs_replace.py +++ /dev/null @@ -1,56 +0,0 @@ -import argparse -import os -import subprocess -import sys - - -def main(argv=None): - parser = argparse.ArgumentParser( - description="""Run terraform-docs on a set of files. Follows the standard convention of - pulling the documentation from main.tf in order to replace the entire - README.md file each time.""" - ) - parser.add_argument( - '--dest', dest='dest', default='README.md', - ) - parser.add_argument( - '--sort-inputs-by-required', dest='sort', action='store_true', - help='[deprecated] use --sort-by-required instead', - ) - parser.add_argument( - '--sort-by-required', dest='sort', action='store_true', - ) - parser.add_argument( - '--with-aggregate-type-defaults', dest='aggregate', action='store_true', - help='[deprecated]', - ) - parser.add_argument('filenames', nargs='*', help='Filenames to check.') - args = parser.parse_args(argv) - - dirs = [] - for filename in args.filenames: - if (os.path.realpath(filename) not in dirs and - (filename.endswith(".tf") or filename.endswith(".tfvars"))): - dirs.append(os.path.dirname(filename)) - - retval = 0 - - for dir in dirs: - try: - procArgs = [] - procArgs.append('terraform-docs') - if args.sort: - procArgs.append('--sort-by-required') - procArgs.append('md') - procArgs.append("./{dir}".format(dir=dir)) - procArgs.append('>') - procArgs.append("./{dir}/{dest}".format(dir=dir, dest=args.dest)) - subprocess.check_call(" ".join(procArgs), shell=True) - except subprocess.CalledProcessError as e: - print(e) - retval = 1 - return retval - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/hooks/terraform_fmt.sh b/hooks/terraform_fmt.sh index bb0b327d1..e8f974fd7 100755 --- a/hooks/terraform_fmt.sh +++ b/hooks/terraform_fmt.sh @@ -2,8 +2,8 @@ set -eo pipefail # globals variables -# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines -readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR # shellcheck source=_common.sh . "$SCRIPT_DIR/_common.sh" @@ -31,7 +31,9 @@ function main { # change_dir_in_unique_part (string/false) Modifier which creates # possibilities to use non-common chdir strategies. # Availability depends on hook. +# parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior +# tf_path (string) PATH to Terraform/OpenTofu binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -40,11 +42,14 @@ function per_dir_hook_unique_part { local -r dir_path="$1" # shellcheck disable=SC2034 # Unused var. local -r change_dir_in_unique_part="$2" - shift 2 + # shellcheck disable=SC2034 # Unused var. + local -r parallelism_disabled="$3" + local -r tf_path="$4" + shift 4 local -a -r args=("$@") # pass the arguments to hook - terraform fmt "${args[@]}" + "$tf_path" fmt "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? diff --git a/hooks/terraform_providers_lock.sh b/hooks/terraform_providers_lock.sh index d85a794f8..9a4d1b624 100755 --- a/hooks/terraform_providers_lock.sh +++ b/hooks/terraform_providers_lock.sh @@ -3,8 +3,8 @@ set -eo pipefail # globals variables -# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines -readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR # shellcheck source=_common.sh . "$SCRIPT_DIR/_common.sh" @@ -85,7 +85,9 @@ function lockfile_contains_all_needed_sha { # change_dir_in_unique_part (string/false) Modifier which creates # possibilities to use non-common chdir strategies. # Availability depends on hook. +# parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior +# tf_path (string) PATH to Terraform/OpenTofu binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -93,7 +95,9 @@ function per_dir_hook_unique_part { local -r dir_path="$1" # shellcheck disable=SC2034 # Unused var. local -r change_dir_in_unique_part="$2" - shift 2 + local -r parallelism_disabled="$3" + local -r tf_path="$4" + shift 4 local -a -r args=("$@") local platforms_count=0 @@ -136,7 +140,7 @@ function per_dir_hook_unique_part { common::colorify "yellow" "DEPRECATION NOTICE: We introduced '--mode' flag for this hook. Check migration instructions at https://github.com/antonbabenko/pre-commit-terraform#terraform_providers_lock " - common::terraform_init 'terraform providers lock' "$dir_path" || { + common::terraform_init "$tf_path providers lock" "$dir_path" "$parallelism_disabled" "$tf_path" || { exit_code=$? return $exit_code } @@ -151,7 +155,7 @@ Check migration instructions at https://github.com/antonbabenko/pre-commit-terra #? Don't require `tf init` for providers, but required `tf init` for modules #? Mitigated by `function match_validate_errors` from terraform_validate hook # pass the arguments to hook - terraform providers lock "${args[@]}" + "$tf_path" providers lock "${args[@]}" # return exit code to common::per_dir_hook exit_code=$? diff --git a/hooks/terraform_tflint.sh b/hooks/terraform_tflint.sh index f26201a3c..54bcebe38 100755 --- a/hooks/terraform_tflint.sh +++ b/hooks/terraform_tflint.sh @@ -3,8 +3,8 @@ set -eo pipefail # globals variables -# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines -readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR # shellcheck source=_common.sh . "$SCRIPT_DIR/_common.sh" @@ -44,14 +44,20 @@ function main { # change_dir_in_unique_part (string/false) Modifier which creates # possibilities to use non-common chdir strategies. # Availability depends on hook. +# parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior +# tf_path (string) PATH to Terraform/OpenTofu binary # Outputs: # If failed - print out hook checks status ####################################################################### function per_dir_hook_unique_part { local -r dir_path="$1" local -r change_dir_in_unique_part="$2" - shift 2 + # shellcheck disable=SC2034 # Unused var. + local -r parallelism_disabled="$3" + # shellcheck disable=SC2034 # Unused var. + local -r tf_path="$4" + shift 4 local -a -r args=("$@") if [ "$change_dir_in_unique_part" == "delegate_chdir" ]; then diff --git a/hooks/terraform_tfsec.sh b/hooks/terraform_tfsec.sh index d05ed4241..dddad49a6 100755 --- a/hooks/terraform_tfsec.sh +++ b/hooks/terraform_tfsec.sh @@ -2,8 +2,8 @@ set -eo pipefail # globals variables -# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines -readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR # shellcheck source=_common.sh . "$SCRIPT_DIR/_common.sh" @@ -37,7 +37,9 @@ function main { # change_dir_in_unique_part (string/false) Modifier which creates # possibilities to use non-common chdir strategies. # Availability depends on hook. +# parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior +# tf_path (string) PATH to Terraform/OpenTofu binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -46,7 +48,11 @@ function per_dir_hook_unique_part { local -r dir_path="$1" # shellcheck disable=SC2034 # Unused var. local -r change_dir_in_unique_part="$2" - shift 2 + # shellcheck disable=SC2034 # Unused var. + local -r parallelism_disabled="$3" + # shellcheck disable=SC2034 # Unused var. + local -r tf_path="$4" + shift 4 local -a -r args=("$@") # pass the arguments to hook diff --git a/hooks/terraform_trivy.sh b/hooks/terraform_trivy.sh index dc205601a..fb4a529a0 100755 --- a/hooks/terraform_trivy.sh +++ b/hooks/terraform_trivy.sh @@ -2,8 +2,8 @@ set -eo pipefail # globals variables -# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines -readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR # shellcheck source=_common.sh . "$SCRIPT_DIR/_common.sh" @@ -29,7 +29,9 @@ function main { # change_dir_in_unique_part (string/false) Modifier which creates # possibilities to use non-common chdir strategies. # Availability depends on hook. +# parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior +# tf_path (string) PATH to Terraform/OpenTofu binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -38,7 +40,11 @@ function per_dir_hook_unique_part { local -r dir_path="$1" # shellcheck disable=SC2034 # Unused var. local -r change_dir_in_unique_part="$2" - shift 2 + # shellcheck disable=SC2034 # Unused var. + local -r parallelism_disabled="$3" + # shellcheck disable=SC2034 # Unused var. + local -r tf_path="$4" + shift 4 local -a -r args=("$@") # pass the arguments to hook diff --git a/hooks/terraform_validate.sh b/hooks/terraform_validate.sh index 2f3795a72..ad792b038 100755 --- a/hooks/terraform_validate.sh +++ b/hooks/terraform_validate.sh @@ -2,8 +2,8 @@ set -eo pipefail # globals variables -# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines -readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR # shellcheck source=_common.sh . "$SCRIPT_DIR/_common.sh" @@ -75,7 +75,9 @@ function match_validate_errors { # change_dir_in_unique_part (string/false) Modifier which creates # possibilities to use non-common chdir strategies. # Availability depends on hook. +# parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior +# tf_path (string) PATH to Terraform/OpenTofu binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -83,7 +85,9 @@ function per_dir_hook_unique_part { local -r dir_path="$1" # shellcheck disable=SC2034 # Unused var. local -r change_dir_in_unique_part="$2" - shift 2 + local -r parallelism_disabled="$3" + local -r tf_path="$4" + shift 4 local -a -r args=("$@") local exit_code @@ -114,25 +118,25 @@ function per_dir_hook_unique_part { # First try `terraform validate` with the hope that all deps are # pre-installed. That is needed for cases when `.terraform/modules` # or `.terraform/providers` missed AND that is expected. - terraform validate "${args[@]}" &> /dev/null && { + "$tf_path" validate "${args[@]}" &> /dev/null && { exit_code=$? return $exit_code } # In case `terraform validate` failed to execute # - check is simple `terraform init` will help - common::terraform_init 'terraform validate' "$dir_path" || { + common::terraform_init "$tf_path validate" "$dir_path" "$parallelism_disabled" "$tf_path" || { exit_code=$? return $exit_code } if [ "$retry_once_with_cleanup" != "true" ]; then # terraform validate only - validate_output=$(terraform validate "${args[@]}" 2>&1) + validate_output=$("$tf_path" validate "${args[@]}" 2>&1) exit_code=$? else # terraform validate, plus capture possible errors - validate_output=$(terraform validate -json "${args[@]}" 2>&1) + validate_output=$("$tf_path" validate -json "${args[@]}" 2>&1) exit_code=$? # Match specific validation errors @@ -150,12 +154,12 @@ function per_dir_hook_unique_part { common::colorify "yellow" "Re-validating: $dir_path" - common::terraform_init 'terraform validate' "$dir_path" || { + common::terraform_init "$tf_path validate" "$dir_path" "$parallelism_disabled" "$tf_path" || { exit_code=$? return $exit_code } - validate_output=$(terraform validate "${args[@]}" 2>&1) + validate_output=$("$tf_path" validate "${args[@]}" 2>&1) exit_code=$? fi fi diff --git a/hooks/terraform_wrapper_module_for_each.sh b/hooks/terraform_wrapper_module_for_each.sh index b01fe4601..dc08b791c 100755 --- a/hooks/terraform_wrapper_module_for_each.sh +++ b/hooks/terraform_wrapper_module_for_each.sh @@ -2,8 +2,8 @@ set -eo pipefail # globals variables -# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines -readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR # shellcheck source=_common.sh . "$SCRIPT_DIR/_common.sh" @@ -312,10 +312,10 @@ EOF # Read content of all terraform files # shellcheck disable=SC2207 - all_tf_content=$(find "${full_module_dir}" -name '*.tf' -maxdepth 1 -type f -exec cat {} +) + all_tf_content=$(find "${full_module_dir}" -maxdepth 1 \( -name '*.tf' -o -name '*.tofu' \) -type f -exec cat {} +) if [[ ! $all_tf_content ]]; then - common::colorify "yellow" "Skipping ${full_module_dir} because there are no *.tf files." + common::colorify "yellow" "Skipping ${full_module_dir} because there are no .tf or .tofu files." continue fi @@ -393,7 +393,13 @@ EOF mv "$tmp_file_tf" "${output_dir}/main.tf" echo "$CONTENT_VARIABLES_TF" > "${output_dir}/variables.tf" - echo "$CONTENT_VERSIONS_TF" > "${output_dir}/versions.tf" + + # If the root module has a versions.tf, use that; otherwise, create it + if [[ -f "${full_module_dir}/versions.tf" ]]; then + cp "${full_module_dir}/versions.tf" "${output_dir}/versions.tf" + else + echo "$CONTENT_VERSIONS_TF" > "${output_dir}/versions.tf" + fi echo "$CONTENT_OUTPUTS_TF" > "${output_dir}/outputs.tf" sed -i.bak "s|WRAPPER_OUTPUT_SENSITIVE|${wrapper_output_sensitive}|g" "${output_dir}/outputs.tf" diff --git a/hooks/terragrunt_fmt.sh b/hooks/terragrunt_fmt.sh index 7c78b9233..2d6697ae3 100755 --- a/hooks/terragrunt_fmt.sh +++ b/hooks/terragrunt_fmt.sh @@ -2,8 +2,8 @@ set -eo pipefail # globals variables -# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines -readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR # shellcheck source=_common.sh . "$SCRIPT_DIR/_common.sh" @@ -12,7 +12,13 @@ function main { common::parse_cmdline "$@" common::export_provided_env_vars "${ENV_VARS[@]}" common::parse_and_export_env_vars - # JFYI: terragrunt hclfmt color already suppressed via PRE_COMMIT_COLOR=never + # JFYI: `terragrunt hcl format` color already suppressed via PRE_COMMIT_COLOR=never + + if common::terragrunt_version_ge_0.78; then + local -ra SUBCOMMAND=(hcl format) + else + local -ra SUBCOMMAND=(hclfmt) + fi # shellcheck disable=SC2153 # False positive common::per_dir_hook "$HOOK_ID" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" @@ -27,7 +33,9 @@ function main { # change_dir_in_unique_part (string/false) Modifier which creates # possibilities to use non-common chdir strategies. # Availability depends on hook. +# parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior +# tf_path (string) PATH to Terraform/OpenTofu binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -36,11 +44,15 @@ function per_dir_hook_unique_part { local -r dir_path="$1" # shellcheck disable=SC2034 # Unused var. local -r change_dir_in_unique_part="$2" - shift 2 + # shellcheck disable=SC2034 # Unused var. + local -r parallelism_disabled="$3" + # shellcheck disable=SC2034 # Unused var. + local -r tf_path="$4" + shift 4 local -a -r args=("$@") # pass the arguments to hook - terragrunt hclfmt "${args[@]}" + terragrunt "${SUBCOMMAND[@]}" "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? @@ -57,7 +69,7 @@ function run_hook_on_whole_repo { local -a -r args=("$@") # pass the arguments to hook - terragrunt hclfmt "$(pwd)" "${args[@]}" + terragrunt "${SUBCOMMAND[@]}" "$(pwd)" "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? diff --git a/hooks/terragrunt_providers_lock.sh b/hooks/terragrunt_providers_lock.sh new file mode 100755 index 000000000..21ee668fe --- /dev/null +++ b/hooks/terragrunt_providers_lock.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -eo pipefail + +# globals variables +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR +# shellcheck source=_common.sh +. "$SCRIPT_DIR/_common.sh" + +function main { + common::initialize "$SCRIPT_DIR" + common::parse_cmdline "$@" + common::export_provided_env_vars "${ENV_VARS[@]}" + common::parse_and_export_env_vars + # JFYI: terragrunt providers lock color already suppressed via PRE_COMMIT_COLOR=never + + if common::terragrunt_version_ge_0.78; then + local -ra RUN_ALL_SUBCOMMAND=(run --all providers lock) + else + local -ra RUN_ALL_SUBCOMMAND=(run-all providers lock) + fi + + # shellcheck disable=SC2153 # False positive + common::per_dir_hook "$HOOK_ID" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" +} + +####################################################################### +# Unique part of `common::per_dir_hook`. The function is executed in loop +# on each provided dir path. Run wrapped tool with specified arguments +# Arguments: +# dir_path (string) PATH to dir relative to git repo root. +# Can be used in error logging +# change_dir_in_unique_part (string/false) Modifier which creates +# possibilities to use non-common chdir strategies. +# Availability depends on hook. +# parallelism_disabled (bool) if true - skip lock mechanism +# args (array) arguments that configure wrapped tool behavior +# tf_path (string) PATH to Terraform/OpenTofu binary +# Outputs: +# If failed - print out hook checks status +####################################################################### +function per_dir_hook_unique_part { + # shellcheck disable=SC2034 # Unused var. + local -r dir_path="$1" + # shellcheck disable=SC2034 # Unused var. + local -r change_dir_in_unique_part="$2" + # shellcheck disable=SC2034 # Unused var. + local -r parallelism_disabled="$3" + # shellcheck disable=SC2034 # Unused var. + local -r tf_path="$4" + shift 4 + local -a -r args=("$@") + + # pass the arguments to hook + terragrunt providers lock "${args[@]}" + + # return exit code to common::per_dir_hook + local exit_code=$? + return $exit_code +} + +####################################################################### +# Unique part of `common::per_dir_hook`. The function is executed one time +# in the root git repo +# Arguments: +# args (array) arguments that configure wrapped tool behavior +####################################################################### +function run_hook_on_whole_repo { + local -a -r args=("$@") + + # pass the arguments to hook + terragrunt "${RUN_ALL_SUBCOMMAND[@]}" "${args[@]}" + + # return exit code to common::per_dir_hook + local exit_code=$? + return $exit_code +} + +[ "${BASH_SOURCE[0]}" != "$0" ] || main "$@" diff --git a/hooks/terragrunt_validate.sh b/hooks/terragrunt_validate.sh index 15e203be7..54b7a98d6 100755 --- a/hooks/terragrunt_validate.sh +++ b/hooks/terragrunt_validate.sh @@ -2,8 +2,8 @@ set -eo pipefail # globals variables -# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines -readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR # shellcheck source=_common.sh . "$SCRIPT_DIR/_common.sh" @@ -14,6 +14,12 @@ function main { common::parse_and_export_env_vars # JFYI: terragrunt validate color already suppressed via PRE_COMMIT_COLOR=never + if common::terragrunt_version_ge_0.78; then + local -ra RUN_ALL_SUBCOMMAND=(run --all validate) + else + local -ra RUN_ALL_SUBCOMMAND=(run-all validate) + fi + # shellcheck disable=SC2153 # False positive common::per_dir_hook "$HOOK_ID" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" } @@ -27,7 +33,9 @@ function main { # change_dir_in_unique_part (string/false) Modifier which creates # possibilities to use non-common chdir strategies. # Availability depends on hook. +# parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior +# tf_path (string) PATH to Terraform/OpenTofu binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -36,7 +44,11 @@ function per_dir_hook_unique_part { local -r dir_path="$1" # shellcheck disable=SC2034 # Unused var. local -r change_dir_in_unique_part="$2" - shift 2 + # shellcheck disable=SC2034 # Unused var. + local -r parallelism_disabled="$3" + # shellcheck disable=SC2034 # Unused var. + local -r tf_path="$4" + shift 4 local -a -r args=("$@") # pass the arguments to hook @@ -57,7 +69,7 @@ function run_hook_on_whole_repo { local -a -r args=("$@") # pass the arguments to hook - terragrunt run-all validate "${args[@]}" + terragrunt "${RUN_ALL_SUBCOMMAND[@]}" "${args[@]}" # return exit code to common::per_dir_hook local exit_code=$? diff --git a/hooks/terragrunt_validate_inputs.sh b/hooks/terragrunt_validate_inputs.sh new file mode 100755 index 000000000..39e8e484e --- /dev/null +++ b/hooks/terragrunt_validate_inputs.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -eo pipefail + +# globals variables +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR +# shellcheck source=_common.sh +. "$SCRIPT_DIR/_common.sh" + +function main { + common::initialize "$SCRIPT_DIR" + common::parse_cmdline "$@" + common::export_provided_env_vars "${ENV_VARS[@]}" + common::parse_and_export_env_vars + # JFYI: terragrunt validate color already suppressed via PRE_COMMIT_COLOR=never + + if common::terragrunt_version_ge_0.78; then + local -ra SUBCOMMAND=(hcl validate --inputs) + local -ra RUN_ALL_SUBCOMMAND=(run --all hcl validate --inputs) + else + local -ra SUBCOMMAND=(validate-inputs) + local -ra RUN_ALL_SUBCOMMAND=(run-all validate-inputs) + fi + + # shellcheck disable=SC2153 # False positive + common::per_dir_hook "$HOOK_ID" "${#ARGS[@]}" "${ARGS[@]}" "${FILES[@]}" +} + +####################################################################### +# Unique part of `common::per_dir_hook`. The function is executed in loop +# on each provided dir path. Run wrapped tool with specified arguments +# Arguments: +# dir_path (string) PATH to dir relative to git repo root. +# Can be used in error logging +# change_dir_in_unique_part (string/false) Modifier which creates +# possibilities to use non-common chdir strategies. +# Availability depends on hook. +# parallelism_disabled (bool) if true - skip lock mechanism +# args (array) arguments that configure wrapped tool behavior +# tf_path (string) PATH to Terraform/OpenTofu binary +# Outputs: +# If failed - print out hook checks status +####################################################################### +function per_dir_hook_unique_part { + # shellcheck disable=SC2034 # Unused var. + local -r dir_path="$1" + # shellcheck disable=SC2034 # Unused var. + local -r change_dir_in_unique_part="$2" + # shellcheck disable=SC2034 # Unused var. + local -r parallelism_disabled="$3" + # shellcheck disable=SC2034 # Unused var. + local -r tf_path="$4" + shift 4 + local -a -r args=("$@") + + # pass the arguments to hook + terragrunt "${SUBCOMMAND[@]}" "${args[@]}" + + # return exit code to common::per_dir_hook + local exit_code=$? + return $exit_code +} + +####################################################################### +# Unique part of `common::per_dir_hook`. The function is executed one time +# in the root git repo +# Arguments: +# args (array) arguments that configure wrapped tool behavior +####################################################################### +function run_hook_on_whole_repo { + local -a -r args=("$@") + + # pass the arguments to hook + terragrunt "${RUN_ALL_SUBCOMMAND[@]}" "${args[@]}" + + # return exit code to common::per_dir_hook + local exit_code=$? + return $exit_code +} + +[ "${BASH_SOURCE[0]}" != "$0" ] || main "$@" diff --git a/hooks/terrascan.sh b/hooks/terrascan.sh index ac040b93e..f6586ceae 100755 --- a/hooks/terrascan.sh +++ b/hooks/terrascan.sh @@ -2,8 +2,8 @@ set -eo pipefail # globals variables -# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines -readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR # shellcheck source=_common.sh . "$SCRIPT_DIR/_common.sh" @@ -27,7 +27,9 @@ function main { # change_dir_in_unique_part (string/false) Modifier which creates # possibilities to use non-common chdir strategies. # Availability depends on hook. +# parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior +# tf_path (string) PATH to Terraform/OpenTofu binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -36,7 +38,11 @@ function per_dir_hook_unique_part { local -r dir_path="$1" # shellcheck disable=SC2034 # Unused var. local -r change_dir_in_unique_part="$2" - shift 2 + # shellcheck disable=SC2034 # Unused var. + local -r parallelism_disabled="$3" + # shellcheck disable=SC2034 # Unused var. + local -r tf_path="$4" + shift 4 local -a -r args=("$@") # pass the arguments to hook diff --git a/hooks/tfupdate.sh b/hooks/tfupdate.sh index d9a482917..1d474318b 100755 --- a/hooks/tfupdate.sh +++ b/hooks/tfupdate.sh @@ -2,8 +2,8 @@ set -eo pipefail # globals variables -# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines -readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR # shellcheck source=_common.sh . "$SCRIPT_DIR/_common.sh" @@ -37,7 +37,9 @@ function main { # change_dir_in_unique_part (string/false) Modifier which creates # possibilities to use non-common chdir strategies. # Availability depends on hook. +# parallelism_disabled (bool) if true - skip lock mechanism # args (array) arguments that configure wrapped tool behavior +# tf_path (string) PATH to Terraform/OpenTofu binary # Outputs: # If failed - print out hook checks status ####################################################################### @@ -46,7 +48,11 @@ function per_dir_hook_unique_part { local -r dir_path="$1" # shellcheck disable=SC2034 # Unused var. local -r change_dir_in_unique_part="$2" - shift 2 + # shellcheck disable=SC2034 # Unused var. + local -r parallelism_disabled="$3" + # shellcheck disable=SC2034 # Unused var. + local -r tf_path="$4" + shift 4 local -a -r args=("$@") # pass the arguments to hook diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..afef00e9d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,64 @@ +[build-system] +requires = [ + 'hatchling', + 'hatch-vcs', # setuptools-scm +] +build-backend = 'hatchling.build' + +[dependency-groups] +building = [ + 'build', +] +linting = [ + 'pre-commit', +] +testing = [ + 'covdefaults', # sets up `coveragepy` config boilerplate + 'pytest >= 8', + 'pytest-cov', # integrates `coveragepy` into pytest runs + 'pytest-mock', # provides a `mocker` fixture + 'pytest-xdist', # paralellizes tests through subprocesses +] +upstreaming = [ + 'twine', +] + +[project] +name = 'pre-commit-terraform' +classifiers = [ + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', +] +description = 'Pre-commit hooks for Terraform, OpenTofu, Terragrunt and related tools' +dependencies = [] +dynamic = [ + 'urls', + 'version', +] +requires-python = ">= 3.9" + +[[project.authors]] +name = 'Anton Babenko' +email = 'anton@antonbabenko.com' + +[[project.authors]] +name = 'Contributors' + +[[project.maintainers]] +name = 'Maksym Vlasov' + +[[project.maintainers]] +name = 'George L. Yermulnik' +email = 'yz@yz.kiev.ua' + +[project.readme] +file = 'README.md' +content-type = 'text/markdown' diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..b2bb788d6 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,84 @@ +[pytest] +addopts = + # `pytest-xdist`: + --numprocesses=auto + # NOTE: the plugin is disabled because it's slower with so few tests + --numprocesses=0 + + # Show 10 slowest invocations: + --durations=10 + + # Report all the things == -rxXs: + -ra + + # Show values of the local vars in errors/tracebacks: + --showlocals + + # Autocollect and invoke the doctests from all modules: + # https://docs.pytest.org/en/stable/doctest.html + --doctest-modules + + # Pre-load the `pytest-cov` plugin early: + -p pytest_cov + + # `pytest-cov`: + --cov + --cov-config=.coveragerc + --cov-context=test + --no-cov-on-fail + + # Fail on config parsing warnings: + # --strict-config + + # Fail on non-existing markers: + # * Deprecated since v6.2.0 but may be reintroduced later covering a + # broader scope: + # --strict + # * Exists since v4.5.0 (advised to be used instead of `--strict`): + --strict-markers + +doctest_optionflags = ALLOW_UNICODE ELLIPSIS + +# Marks tests with an empty parameterset as xfail(run=False) +empty_parameter_set_mark = xfail + +faulthandler_timeout = 30 +# Turn all warnings into errors +filterwarnings = + error + +# https://docs.pytest.org/en/stable/usage.html#creating-junitxml-format-files +junit_duration_report = call +# xunit1 contains more metadata than xunit2 so it's better for CI UIs: +junit_family = xunit1 +junit_logging = all +junit_log_passing_tests = true +junit_suite_name = awx_plugins_test_suite + +# A mapping of markers to their descriptions allowed in strict mode: +markers = + +minversion = 6.1.0 + +# Optimize pytest's lookup by restricting potentially deep dir tree scan: +norecursedirs = + build + dependencies + dist + docs + .cache + .eggs + .git + .github + .tox + *.egg + *.egg-info + */*.egg-info + */**/*.egg-info + *.dist-info + */*.dist-info + */**/*.dist-info + +testpaths = tests/pytest/ + +xfail_strict = true diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 000000000..0e57620ea --- /dev/null +++ b/ruff.toml @@ -0,0 +1,41 @@ +# Assume Python 3.9 +target-version = "py39" + +line-length = 79 # To decrease PR diff size + +namespace-packages = ["src/pre_commit_terraform/", "tests/pytest/"] + +[format] +quote-style = "single" + +[lint.flake8-quotes] +inline-quotes = "single" + +[lint.pydocstyle] +convention = "pep257" + +[lint] +select = ["ALL"] +preview = true +external = ["WPS"] # Do not remove noqa for wemake-python-style (WPS) checks +ignore = [ + "CPY001", # Skip copyright notice requirement at top of files +] + +[lint.isort] +# force-single-line = true # To decrease PR diff size +lines-after-imports = 2 + +[lint.flake8-pytest-style] +parametrize-values-type = "tuple" + +[lint.per-file-ignores] +# Exceptions for test files +"tests/**.py" = [ + "S101", # Allow use of `assert` in test files + "PLC2701", # Allow importing internal files needed for testing + "PLR6301", # Allow 'self' parameter in method definitions (required for test stubs) + "ARG002", # Allow unused arguments in instance methods (required for test stubs) + "S404", # Allow importing 'subprocess' module to testing call external tools needed by these hooks + +] diff --git a/setup.py b/setup.py deleted file mode 100644 index 2d88425b9..000000000 --- a/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -from setuptools import find_packages -from setuptools import setup - - -setup( - name='pre-commit-terraform', - description='Pre-commit hooks for terraform_docs_replace', - url='https://github.com/antonbabenko/pre-commit-terraform', - version_format='{tag}+{gitsha}', - - author='Contributors', - - classifiers=[ - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - ], - - packages=find_packages(exclude=('tests*', 'testing*')), - install_requires=[ - 'setuptools-git-version', - ], - entry_points={ - 'console_scripts': [ - 'terraform_docs_replace = hooks.terraform_docs_replace:main', - ], - }, -) diff --git a/src/pre_commit_terraform/README.md b/src/pre_commit_terraform/README.md new file mode 100644 index 000000000..392d4221b --- /dev/null +++ b/src/pre_commit_terraform/README.md @@ -0,0 +1,93 @@ +# Maintainer's manual + +## Structure + +This folder is what's called an [importable package]. It's a top-level folder +that ends up being installed into `site-packages/` of virtualenvs. + +When the Git repository is `pip install`ed, this [import package] becomes +available for use within respective Python interpreter instance. It can be +imported and sub-modules can be imported through the dot-syntax. Additionally, +the modules within can import the neighboring ones using relative imports that +have a leading dot in them. + +It additionally implements a [runpy interface], meaning that its name can +be passed to `python -m` to invoke the CLI. This is the primary method of +integration with the [`pre-commit` framework] and local development/testing. + +The layout allows for having several Python modules wrapping third-party tools, +each having an argument parser and being a subcommand for the main CLI +interface. + +## Control flow + +When `python -m pre_commit_terraform` is executed, it imports `__main__.py`. +Which in turn, performs the initialization of the main argument parser and the +parsers of subcommands, followed by executing the logic defined in dedicated +subcommand modules. + +## Integrating a new subcommand + +1. Create a new module called `subcommand_x.py`. +2. Within that module, define two functions โ€” + `invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType | int` and + `populate_argument_parser(subcommand_parser: ArgumentParser) -> None`. + Additionally, define a module-level constant + `CLI_SUBCOMMAND_NAME: Final[str] = 'subcommand-x'`. +3. Edit [`_cli_subcommands.py`], importing `subcommand_x` as a relative module + and add it into the `SUBCOMMAND_MODULES` list. +4. Edit [`.pre-commit-hooks.yaml`], adding a new hook that invokes + `python -m pre_commit_terraform subcommand-x`. + +## Manual testing + +Usually, having a development virtualenv where you `pip install -e .` is enough +to make it possible to invoke the CLI app. Do so first. Most source code +updates do not require running it again. But sometimes, it's needed. + +Once done, you can run `python -m pre_commit_terraform` and/or +`python -m pre_commit_terraform subcommand-x` to see how it behaves. There's +`--help` and all other typical conventions one would usually expect from a +POSIX-inspired CLI app. + +## DX/UX considerations + +Since it's an app that can be executed outside the [`pre-commit` framework], +it is useful to check out and follow these [CLI guidelines][clig]. + +## Subcommand development + +`populate_argument_parser()` accepts a regular instance of +[`argparse.ArgumentParser`]. Call its methods to extend the CLI arguments that +would be specific for the subcommand you are creating. Those arguments will be +available later, as an argument to the `invoke_cli_app()` function โ€” through an +instance of [`argparse.Namespace`]. For the `CLI_SUBCOMMAND_NAME` constant, +choose `kebab-space-sub-command-style`, it does not need to be `snake_case`. + +Make sure to return a `ReturnCode` instance or an integer from +`invoke_cli_app()`. Returning a non-zero value will result in the CLI app +exiting with a return code typically interpreted as an error while zero means +success. You can `import errno` to use typical POSIX error codes through their +human-readable identifiers. + +Another way to interrupt the CLI app control flow is by raising an instance of +one of the in-app errors. `raise PreCommitTerraformExit` for a successful exit, +but it can be turned into an error outcome via +`raise PreCommitTerraformExit(1)`. +`raise PreCommitTerraformRuntimeError('The world is broken')` to indicate +problems within the runtime. The framework will intercept any exceptions +inheriting `PreCommitTerraformBaseError`, so they won't be presented to the +end-users. + +[`.pre-commit-hooks.yaml`]: ../../.pre-commit-hooks.yaml +[`_cli_parsing.py`]: ./_cli_parsing.py +[`_cli_subcommands.py`]: ./_cli_subcommands.py +[`argparse.ArgumentParser`]: +https://docs.python.org/3/library/argparse.html#argparse.ArgumentParser +[`argparse.Namespace`]: +https://docs.python.org/3/library/argparse.html#argparse.Namespace +[clig]: https://clig.dev +[importable package]: https://docs.python.org/3/tutorial/modules.html#packages +[import package]: https://packaging.python.org/en/latest/glossary/#term-Import-Package +[`pre-commit` framework]: https://pre-commit.com +[runpy interface]: https://docs.python.org/3/library/__main__.html diff --git a/src/pre_commit_terraform/__main__.py b/src/pre_commit_terraform/__main__.py new file mode 100644 index 000000000..00629a7b7 --- /dev/null +++ b/src/pre_commit_terraform/__main__.py @@ -0,0 +1,10 @@ +"""A runpy-style CLI entry-point module.""" + +from sys import argv +from sys import exit as exit_with_return_code + +from ._cli import invoke_cli_app + + +return_code = invoke_cli_app(argv[1:]) +exit_with_return_code(return_code) diff --git a/src/pre_commit_terraform/_cli.py b/src/pre_commit_terraform/_cli.py new file mode 100644 index 000000000..8c99e2b54 --- /dev/null +++ b/src/pre_commit_terraform/_cli.py @@ -0,0 +1,64 @@ +"""Outer CLI layer of the app interface.""" + +import sys +from typing import cast as cast_to + +from ._cli_parsing import initialize_argument_parser +from ._errors import ( + PreCommitTerraformBaseError, + PreCommitTerraformExit, + PreCommitTerraformRuntimeError, +) +from ._structs import ReturnCode +from ._types import CLIAppEntryPointCallableType, ReturnCodeType + + +def invoke_cli_app(cli_args: list[str]) -> ReturnCodeType: + """Run the entry-point of the CLI app. + + Includes initializing parsers of all the sub-apps and + choosing what to execute. + + Returns: + ReturnCodeType: The return code of the app. + + Raises: + PreCommitTerraformExit: If the app is exiting with error. + """ + root_cli_parser = initialize_argument_parser() + parsed_cli_args = root_cli_parser.parse_args(cli_args) + invoke_cli_app = cast_to( + # FIXME: attempt typing per https://stackoverflow.com/a/75666611/595220 # noqa: TD001, TD002, FIX001, E501 All these suppressions caused by "FIXME" comment + 'CLIAppEntryPointCallableType', + parsed_cli_args.invoke_cli_app, + ) + + try: # noqa: WPS225 - too many `except` cases + return invoke_cli_app(parsed_cli_args) + except PreCommitTerraformExit as exit_err: + # T201,WPS421 - FIXME here and below - we will replace 'print' + # with logging later + print(f'App exiting: {exit_err!s}', file=sys.stderr) # noqa: T201,WPS421 + raise + except PreCommitTerraformRuntimeError as unhandled_exc: + print( # noqa: T201,WPS421 + f'App execution took an unexpected turn: {unhandled_exc!s}. ' + 'Exiting...', + file=sys.stderr, + ) + return ReturnCode.ERROR + except PreCommitTerraformBaseError as unhandled_exc: + print( # noqa: T201,WPS421 + f'A surprising exception happened: {unhandled_exc!s}. Exiting...', + file=sys.stderr, + ) + return ReturnCode.ERROR + except KeyboardInterrupt as ctrl_c_exc: + print( # noqa: T201,WPS421 + f'User-initiated interrupt: {ctrl_c_exc!s}. Exiting...', + file=sys.stderr, + ) + return ReturnCode.ERROR + + +__all__ = ('invoke_cli_app',) diff --git a/src/pre_commit_terraform/_cli_parsing.py b/src/pre_commit_terraform/_cli_parsing.py new file mode 100644 index 000000000..fac4da15c --- /dev/null +++ b/src/pre_commit_terraform/_cli_parsing.py @@ -0,0 +1,45 @@ +"""Argument parser initialization logic. + +This defines helpers for setting up both the root parser and the parsers +of all the sub-commands. +""" + +from argparse import ArgumentParser + +from ._cli_subcommands import SUBCOMMAND_MODULES + + +def attach_subcommand_parsers_to(root_cli_parser: ArgumentParser, /) -> None: + """Connect all sub-command parsers to the given one. + + This functions iterates over a mapping of subcommands to their + respective population functions, executing them to augment the + main parser. + """ + subcommand_parsers = root_cli_parser.add_subparsers( + dest='check_name', + help='A check to be performed.', + required=True, + ) + for subcommand_module in SUBCOMMAND_MODULES: + subcommand_parser = subcommand_parsers.add_parser( + subcommand_module.CLI_SUBCOMMAND_NAME, + ) + subcommand_parser.set_defaults( + invoke_cli_app=subcommand_module.invoke_cli_app, + ) + subcommand_module.populate_argument_parser(subcommand_parser) + + +def initialize_argument_parser() -> ArgumentParser: + """Return the root argument parser with sub-commands. + + Returns: + ArgumentParser: The root parser with sub-commands attached. + """ + root_cli_parser = ArgumentParser(prog=f'python -m {__package__!s}') + attach_subcommand_parsers_to(root_cli_parser) + return root_cli_parser + + +__all__ = ('initialize_argument_parser',) diff --git a/src/pre_commit_terraform/_cli_subcommands.py b/src/pre_commit_terraform/_cli_subcommands.py new file mode 100644 index 000000000..ba7a6982d --- /dev/null +++ b/src/pre_commit_terraform/_cli_subcommands.py @@ -0,0 +1,12 @@ +"""A CLI sub-commands organization module.""" + +from . import terraform_docs_replace +from ._types import CLISubcommandModuleProtocol + + +SUBCOMMAND_MODULES: tuple[CLISubcommandModuleProtocol] = ( + terraform_docs_replace, +) + + +__all__ = ('SUBCOMMAND_MODULES',) diff --git a/src/pre_commit_terraform/_errors.py b/src/pre_commit_terraform/_errors.py new file mode 100644 index 000000000..6d0019808 --- /dev/null +++ b/src/pre_commit_terraform/_errors.py @@ -0,0 +1,19 @@ +"""App-specific exceptions.""" + + +class PreCommitTerraformBaseError(Exception): + """Base exception for all the in-app errors.""" + + +class PreCommitTerraformRuntimeError( + PreCommitTerraformBaseError, + RuntimeError, +): + """An exception representing a runtime error condition.""" + + +# N818 - The name mimics the built-in SystemExit and is meant to have exactly +# the same semantics. For this reason, it shouldn't have Error in the name to +# maintain resemblance. +class PreCommitTerraformExit(PreCommitTerraformBaseError, SystemExit): # noqa: N818 + """An exception for terminating execution from deep app layers.""" diff --git a/src/pre_commit_terraform/_structs.py b/src/pre_commit_terraform/_structs.py new file mode 100644 index 000000000..f701d71a9 --- /dev/null +++ b/src/pre_commit_terraform/_structs.py @@ -0,0 +1,18 @@ +"""Data structures to be reused across the app.""" + +from enum import IntEnum + + +class ReturnCode(IntEnum): + """POSIX-style return code values. + + To be used in check callable implementations. + """ + + # WPS115: "Require snake_case for naming class attributes". According to + # "Correct" example in docs - it's valid use case => false-positive + OK = 0 # noqa: WPS115 + ERROR = 1 # noqa: WPS115 + + +__all__ = ('ReturnCode',) diff --git a/src/pre_commit_terraform/_types.py b/src/pre_commit_terraform/_types.py new file mode 100644 index 000000000..69f71e690 --- /dev/null +++ b/src/pre_commit_terraform/_types.py @@ -0,0 +1,35 @@ +"""Composite types for annotating in-project code.""" + +from argparse import ArgumentParser, Namespace +from collections.abc import Callable +from typing import Protocol, Union + +from ._structs import ReturnCode + + +ReturnCodeType = Union[ReturnCode, int] # Union instead of pipe for Python 3.9 +CLIAppEntryPointCallableType = Callable[[Namespace], ReturnCodeType] + + +class CLISubcommandModuleProtocol(Protocol): + """A protocol for the subcommand-implementing module shape.""" + + # WPS115: "Require snake_case for naming class attributes". + # This protocol describes module shapes and not regular classes. + # It's a valid use case as then it's used as constants: + # "CLI_SUBCOMMAND_NAME: Final[str] = 'hook-name'"" on top level + CLI_SUBCOMMAND_NAME: str # noqa: WPS115 + """This constant contains a CLI.""" + + def populate_argument_parser( + self, + subcommand_parser: ArgumentParser, + ) -> None: + """Run a module hook for populating the subcommand parser.""" + + def invoke_cli_app(self, parsed_cli_args: Namespace) -> ReturnCodeType: + """Run a module hook implementing the subcommand logic.""" + ... # pylint: disable=unnecessary-ellipsis + + +__all__ = ('CLISubcommandModuleProtocol', 'ReturnCodeType') diff --git a/src/pre_commit_terraform/terraform_docs_replace.py b/src/pre_commit_terraform/terraform_docs_replace.py new file mode 100644 index 000000000..3bfb3bc43 --- /dev/null +++ b/src/pre_commit_terraform/terraform_docs_replace.py @@ -0,0 +1,118 @@ +"""Terraform Docs Replace Hook. + +This hook is deprecated and will be removed in the future. +Please, use 'terraform_docs' hook instead. +""" + +import os + +# S404 - Allow importing 'subprocess' module to call external tools +# needed by these hooks. FIXME - should be moved to separate module +# when more hooks will be introduced +import subprocess # noqa: S404 +import warnings +from argparse import ArgumentParser, Namespace +from typing import Final +from typing import cast as cast_to + +from ._structs import ReturnCode +from ._types import ReturnCodeType + + +CLI_SUBCOMMAND_NAME: Final[str] = 'replace-docs' + + +def populate_argument_parser(subcommand_parser: ArgumentParser) -> None: + """Populate the parser for the subcommand.""" + subcommand_parser.description = ( + 'Run terraform-docs on a set of files. Follows the standard ' + 'convention of pulling the documentation from main.tf in order to ' + 'replace the entire README.md file each time.' + ) + subcommand_parser.add_argument( + '--dest', + dest='dest', + default='README.md', + ) + subcommand_parser.add_argument( + '--sort-inputs-by-required', + dest='sort', + action='store_true', + help='[deprecated] use --sort-by-required instead', + ) + subcommand_parser.add_argument( + '--sort-by-required', + dest='sort', + action='store_true', + ) + subcommand_parser.add_argument( + '--with-aggregate-type-defaults', + dest='aggregate', + action='store_true', + help='[deprecated]', + ) + subcommand_parser.add_argument( + 'filenames', + nargs='*', + help='Filenames to check.', + ) + + +# WPS231 - Found function with too much cognitive complexity +# We will not spend time on fixing complexity in deprecated hook +def invoke_cli_app(parsed_cli_args: Namespace) -> ReturnCodeType: # noqa: WPS231 + """Run the entry-point of the CLI app. + + Returns: + ReturnCodeType: The return code of the app. + """ + warnings.warn( + '`terraform_docs_replace` hook is DEPRECATED.' + 'For migration instructions see ' + 'https://github.com/antonbabenko/pre-commit-terraform/issues/248' + '#issuecomment-1290829226', + category=UserWarning, + stacklevel=1, # It's should be 2, but tests are failing w/ values >1. + # As it's deprecated hook, it's safe to leave it as is w/o fixing it. + ) + + dirs: list[str] = [] + for filename in cast_to('list[str]', parsed_cli_args.filenames): + if os.path.realpath(filename) not in dirs and ( + filename.endswith(('.tf', '.tfvars')) + ): + # PTH120 - It should use 'pathlib', but this hook is deprecated and + # we don't want to spent time on testing fixes for it + dirs.append(os.path.dirname(filename)) # noqa: PTH120 + + retval = ReturnCode.OK + + for directory in dirs: + try: # noqa: WPS229 - ignore as it's deprecated hook + proc_args = [] + proc_args.append('terraform-docs') + if cast_to('bool', parsed_cli_args.sort): + proc_args.append('--sort-by-required') + proc_args.extend( + ( + 'md', + f'./{directory}', + '>', + './{dir}/{dest}'.format( + dir=directory, + dest=cast_to('str', parsed_cli_args.dest), + ), + ), + ) + # S602 - 'shell=True' is insecure, but this hook is deprecated and + # we don't want to spent time on testing fixes for it + subprocess.check_call(' '.join(proc_args), shell=True) # noqa: S602 + # PERF203 - try-except shouldn't be in a loop, but it's deprecated + # hook, so leave as is + # WPS111 - Too short var name, but it's deprecated hook, so leave as is + except subprocess.CalledProcessError as e: # noqa: PERF203,WPS111 + # T201,WPS421 - Leave print statement as is, as this is + # deprecated hook + print(e) # noqa: T201,WPS421,WPS111 + retval = ReturnCode.ERROR + return retval diff --git a/tests/Dockerfile b/tests/Dockerfile index ec77d18af..d64ba61e0 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,11 +1,9 @@ +# We use `latest` tag for tests proposes +# hadolint ignore=DL3007 FROM pre-commit-terraform:latest -RUN apt update && \ - apt install -y \ - datamash \ - time && \ - # Cleanup - rm -rf /var/lib/apt/lists/* +RUN apk add --no-cache \ + datamash=~1.8 WORKDIR /pct ENTRYPOINT [ "/pct/tests/hooks_performance_test.sh" ] diff --git a/tests/pytest/_cli_test.py b/tests/pytest/_cli_test.py new file mode 100644 index 000000000..773825269 --- /dev/null +++ b/tests/pytest/_cli_test.py @@ -0,0 +1,103 @@ +"""Tests for the high-level CLI entry point.""" + +from argparse import ArgumentParser, Namespace + +import pytest +from pre_commit_terraform import _cli_parsing as _cli_parsing_mod +from pre_commit_terraform._cli import invoke_cli_app +from pre_commit_terraform._errors import ( + PreCommitTerraformBaseError, + PreCommitTerraformExit, + PreCommitTerraformRuntimeError, +) +from pre_commit_terraform._structs import ReturnCode +from pre_commit_terraform._types import ReturnCodeType + + +pytestmark = pytest.mark.filterwarnings( + 'ignore:`terraform_docs_replace` hook is DEPRECATED.:UserWarning:' + 'pre_commit_terraform.terraform_docs_replace', +) + + +@pytest.mark.parametrize( + ('raised_error', 'expected_stderr'), + ( + pytest.param( + PreCommitTerraformRuntimeError('sentinel'), + 'App execution took an unexpected turn: sentinel. Exiting...', + id='app-runtime-exc', + ), + pytest.param( + PreCommitTerraformBaseError('sentinel'), + 'A surprising exception happened: sentinel. Exiting...', + id='app-base-exc', + ), + pytest.param( + KeyboardInterrupt('sentinel'), + 'User-initiated interrupt: sentinel. Exiting...', + id='ctrl-c', + ), + ), +) +def test_known_interrupts( + capsys: pytest.CaptureFixture[str], + expected_stderr: str, + monkeypatch: pytest.MonkeyPatch, + raised_error: BaseException, +) -> None: + """Check that known interrupts are turned into return code 1.""" + + class CustomCmdStub: + CLI_SUBCOMMAND_NAME = 'sentinel' + + def populate_argument_parser( + self, + subcommand_parser: ArgumentParser, + ) -> None: + pass # noqa: WPS420. This is a stub, docstring not needed so "pass" required. + + def invoke_cli_app(self, parsed_cli_args: Namespace) -> ReturnCodeType: + raise raised_error + + monkeypatch.setattr( + _cli_parsing_mod, + 'SUBCOMMAND_MODULES', + [CustomCmdStub()], + ) + + assert invoke_cli_app(['sentinel']) == ReturnCode.ERROR + + captured_outputs = capsys.readouterr() + assert captured_outputs.err == f'{expected_stderr!s}\n' + + +def test_app_exit( + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Check that an exit exception is re-raised.""" + + class CustomCmdStub: + CLI_SUBCOMMAND_NAME = 'sentinel' + + def populate_argument_parser( + self, + subcommand_parser: ArgumentParser, + ) -> None: + pass # noqa: WPS420. This is a stub, docstring not needed so "pass" required. + + def invoke_cli_app(self, parsed_cli_args: Namespace) -> ReturnCodeType: + raise PreCommitTerraformExit(self.CLI_SUBCOMMAND_NAME) + + monkeypatch.setattr( + _cli_parsing_mod, + 'SUBCOMMAND_MODULES', + [CustomCmdStub()], + ) + + with pytest.raises(PreCommitTerraformExit, match=r'^sentinel$'): + invoke_cli_app(['sentinel']) + + captured_outputs = capsys.readouterr() + assert captured_outputs.err == 'App exiting: sentinel\n' diff --git a/tests/pytest/terraform_docs_replace_test.py b/tests/pytest/terraform_docs_replace_test.py new file mode 100644 index 000000000..c4190c3c0 --- /dev/null +++ b/tests/pytest/terraform_docs_replace_test.py @@ -0,0 +1,125 @@ +"""Tests for the `replace-docs` subcommand.""" + +from argparse import ArgumentParser, Namespace +from subprocess import CalledProcessError + +import pytest_mock + +import pytest +from pre_commit_terraform._structs import ReturnCode +from pre_commit_terraform.terraform_docs_replace import ( + invoke_cli_app, + populate_argument_parser, +) +from pre_commit_terraform.terraform_docs_replace import ( + subprocess as replace_docs_subprocess_mod, +) + + +def test_arg_parser_populated() -> None: + """Check that `replace-docs` populates its parser.""" + test_arg_parser = ArgumentParser() + populate_argument_parser(test_arg_parser) + assert test_arg_parser.get_default('dest') == 'README.md' + + +def test_check_is_deprecated() -> None: + """Verify that `replace-docs` shows a deprecation warning.""" + deprecation_msg_regex = ( + r'^`terraform_docs_replace` hook is DEPRECATED\.For migration.*$' + ) + with pytest.warns(UserWarning, match=deprecation_msg_regex): + # not `pytest.deprecated_call()` due to this being a user warning + invoke_cli_app(Namespace(filenames=[])) + + +@pytest.mark.parametrize( + ('parsed_cli_args', 'expected_cmds'), + ( + pytest.param(Namespace(filenames=[]), [], id='no-files'), + pytest.param( + Namespace( + dest='SENTINEL.md', + filenames=['some.tf'], + sort=False, + ), + ['terraform-docs md ./ > .//SENTINEL.md'], + id='one-file', + ), + pytest.param( + Namespace( + dest='SENTINEL.md', + filenames=['some.tf', 'thing/weird.tfvars'], + sort=True, + ), + [ + 'terraform-docs --sort-by-required md ./ > .//SENTINEL.md', + 'terraform-docs --sort-by-required md ./thing ' + '> ./thing/SENTINEL.md', + ], + id='two-sorted-files', + ), + pytest.param( + Namespace(filenames=['some.thing', 'un.supported']), + [], + id='invalid-files', + ), + ), +) +@pytest.mark.filterwarnings( + 'ignore:`terraform_docs_replace` hook is DEPRECATED.:UserWarning:' + 'pre_commit_terraform.terraform_docs_replace', +) +def test_control_flow_positive( + expected_cmds: list[str], + mocker: pytest_mock.MockerFixture, + monkeypatch: pytest.MonkeyPatch, + parsed_cli_args: Namespace, +) -> None: + """Check that the subcommand's happy path works.""" + check_call_mock = mocker.Mock() + monkeypatch.setattr( + replace_docs_subprocess_mod, + 'check_call', + check_call_mock, + ) + + assert invoke_cli_app(parsed_cli_args) == ReturnCode.OK + + executed_commands = [ + cmd for ((cmd,), _shell) in check_call_mock.call_args_list + ] + + assert len(expected_cmds) == check_call_mock.call_count + assert expected_cmds == executed_commands + + +@pytest.mark.filterwarnings( + 'ignore:`terraform_docs_replace` hook is DEPRECATED.:UserWarning:' + 'pre_commit_terraform.terraform_docs_replace', +) +def test_control_flow_negative( + mocker: pytest_mock.MockerFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Check that the subcommand's error processing works.""" + parsed_cli_args = Namespace( + dest='SENTINEL.md', + filenames=['some.tf'], + sort=True, + ) + expected_cmd = 'terraform-docs --sort-by-required md ./ > .//SENTINEL.md' + + check_call_mock = mocker.Mock( + side_effect=CalledProcessError(ReturnCode.ERROR, expected_cmd), + ) + monkeypatch.setattr( + replace_docs_subprocess_mod, + 'check_call', + check_call_mock, + ) + + assert invoke_cli_app(parsed_cli_args) == ReturnCode.ERROR + # S604 - 'shell=True' is insecure, but this hook is deprecated and we don't + # want to spent time on testing fixes for it + check_call_mock.assert_called_once_with(expected_cmd, shell=True) # noqa: S604 diff --git a/tools/entrypoint.sh b/tools/entrypoint.sh index 86d5e369a..d7f0c9c83 100755 --- a/tools/entrypoint.sh +++ b/tools/entrypoint.sh @@ -39,7 +39,7 @@ wdir="$(pwd)" if ! su-exec "$USERID" "$BASHPATH" -c "test -w $wdir && test -r $wdir"; then echo_error_and_exit "uid:gid $USERID lacks permissions to $wdir/" fi -wdirgitindex="$wdir/.git/index" +wdirgitindex="$(git rev-parse --git-dir 2>&1)/index" || echo_error_and_exit "${wdirgitindex%/index}" if ! su-exec "$USERID" "$BASHPATH" -c "test -w $wdirgitindex && test -r $wdirgitindex"; then echo_error_and_exit "uid:gid $USERID cannot write to $wdirgitindex" fi diff --git a/tools/install/_common.sh b/tools/install/_common.sh new file mode 100755 index 000000000..70297f297 --- /dev/null +++ b/tools/install/_common.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +set -eo pipefail + +# Tool name, based on filename. +# Tool filename MUST BE same as in package manager/binary name +TOOL=${0##*/} +readonly TOOL=${TOOL%%.*} + +# Get "TOOL_VERSION" +# shellcheck disable=SC1091 # Created in Dockerfile before execution of this script +source /.env +env_var_name="${TOOL//-/_}" +env_var_name="${env_var_name^^}_VERSION" +# shellcheck disable=SC2034 # Used in other scripts +readonly VERSION="${!env_var_name}" + +# Skip tool installation if the version is set to "false" +if [[ $VERSION == false ]]; then + echo "'$TOOL' skipped" + exit 0 +fi + +####################################################################### +# Install the latest or specific version of the tool from GitHub release +# Globals: +# TOOL - Name of the tool +# VERSION - Version of the tool +# Arguments: +# GH_ORG - GitHub organization name where the tool is hosted +# DISTRIBUTED_AS - How the tool is distributed. +# Can be: 'tar.gz', 'zip' or 'binary' +# GH_RELEASE_REGEX_LATEST - Regular expression to match the latest +# release URL +# GH_RELEASE_REGEX_SPECIFIC_VERSION - Regular expression to match the +# specific version release URL +# UNUSUAL_TOOL_NAME_IN_PKG - If the tool in the tar.gz package is +# not in the root or named differently than the tool name itself, +# For example, includes the version number or is in a subdirectory +####################################################################### +function common::install_from_gh_release { + local -r GH_ORG=$1 + local -r DISTRIBUTED_AS=$2 + local -r GH_RELEASE_REGEX_LATEST=$3 + local -r GH_RELEASE_REGEX_SPECIFIC_VERSION=$4 + local -r UNUSUAL_TOOL_NAME_IN_PKG=$5 + + case $DISTRIBUTED_AS in + tar.gz | zip) + local -r PKG="${TOOL}.${DISTRIBUTED_AS}" + ;; + binary) + local -r PKG="$TOOL" + ;; + *) + echo "Unknown DISTRIBUTED_AS: '$DISTRIBUTED_AS'. Should be one of: 'tar.gz', 'zip' or 'binary'." >&2 + exit 1 + ;; + esac + + # Download tool + local -r RELEASES="https://api.github.com/repos/${GH_ORG}/${TOOL}/releases" + + if [[ $VERSION == latest ]]; then + curl -L "$(curl -s "${RELEASES}/latest" | grep -o -E -i -m 1 "$GH_RELEASE_REGEX_LATEST")" > "$PKG" + else + curl -L "$(curl -s "$RELEASES" | grep -o -E -i -m 1 "$GH_RELEASE_REGEX_SPECIFIC_VERSION")" > "$PKG" + fi + + # Make tool ready to use + if [[ $DISTRIBUTED_AS == tar.gz ]]; then + if [[ -z $UNUSUAL_TOOL_NAME_IN_PKG ]]; then + tar -xzf "$PKG" "$TOOL" + else + tar -xzf "$PKG" "$UNUSUAL_TOOL_NAME_IN_PKG" + mv "$UNUSUAL_TOOL_NAME_IN_PKG" "$TOOL" + fi + rm "$PKG" + + elif [[ $DISTRIBUTED_AS == zip ]]; then + unzip "$PKG" + rm "$PKG" + else + chmod +x "$PKG" + fi +} diff --git a/tools/install/checkov.sh b/tools/install/checkov.sh new file mode 100755 index 000000000..708e4772b --- /dev/null +++ b/tools/install/checkov.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR +# shellcheck source=_common.sh +. "$SCRIPT_DIR/_common.sh" + +# +# Unique part +# + +apk add --no-cache \ + gcc=~12 \ + libffi-dev=~3 \ + musl-dev=~1 + +if [[ $VERSION == latest ]]; then + pip3 install --no-cache-dir "${TOOL}" +else + pip3 install --no-cache-dir "${TOOL}==${VERSION}" +fi +pip3 check diff --git a/tools/install/hcledit.sh b/tools/install/hcledit.sh new file mode 100755 index 000000000..498e4fb6f --- /dev/null +++ b/tools/install/hcledit.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR +# shellcheck source=_common.sh +. "$SCRIPT_DIR/_common.sh" + +# +# Unique part +# + +GH_ORG="minamijoyo" +GH_RELEASE_REGEX_SPECIFIC_VERSION="https://.+?${VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz" +GH_RELEASE_REGEX_LATEST="https://.+?_${TARGETOS}_${TARGETARCH}.tar.gz" +DISTRIBUTED_AS="tar.gz" + +common::install_from_gh_release "$GH_ORG" "$DISTRIBUTED_AS" \ + "$GH_RELEASE_REGEX_LATEST" "$GH_RELEASE_REGEX_SPECIFIC_VERSION" diff --git a/tools/install/infracost.sh b/tools/install/infracost.sh new file mode 100755 index 000000000..9974ca1d1 --- /dev/null +++ b/tools/install/infracost.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR +# shellcheck source=_common.sh +. "$SCRIPT_DIR/_common.sh" + +# +# Unique part +# + +GH_ORG="infracost" +GH_RELEASE_REGEX_SPECIFIC_VERSION="https://.+?v${VERSION}/${TOOL}-${TARGETOS}-${TARGETARCH}.tar.gz" +GH_RELEASE_REGEX_LATEST="https://.+?-${TARGETOS}-${TARGETARCH}.tar.gz" +DISTRIBUTED_AS="tar.gz" +UNUSUAL_TOOL_NAME_IN_PKG="${TOOL}-${TARGETOS}-${TARGETARCH}" + +common::install_from_gh_release "$GH_ORG" "$DISTRIBUTED_AS" \ + "$GH_RELEASE_REGEX_LATEST" "$GH_RELEASE_REGEX_SPECIFIC_VERSION" \ + "$UNUSUAL_TOOL_NAME_IN_PKG" diff --git a/tools/install/opentofu.sh b/tools/install/opentofu.sh new file mode 100755 index 000000000..bda0fe399 --- /dev/null +++ b/tools/install/opentofu.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR +# shellcheck source=_common.sh +. "$SCRIPT_DIR/_common.sh" + +# +# Unique part +# +GH_ORG="opentofu" +GH_RELEASE_REGEX_SPECIFIC_VERSION="https://.+?v${VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz" +GH_RELEASE_REGEX_LATEST="https://.+?_${TARGETOS}_${TARGETARCH}.tar.gz" +DISTRIBUTED_AS="tar.gz" +UNUSUAL_TOOL_NAME_IN_PKG="tofu" + +common::install_from_gh_release "$GH_ORG" "$DISTRIBUTED_AS" \ + "$GH_RELEASE_REGEX_LATEST" "$GH_RELEASE_REGEX_SPECIFIC_VERSION" \ + "$UNUSUAL_TOOL_NAME_IN_PKG" + +# restore original binary name +mv "$TOOL" "$UNUSUAL_TOOL_NAME_IN_PKG" diff --git a/tools/install/pre-commit.sh b/tools/install/pre-commit.sh new file mode 100755 index 000000000..ca46e679d --- /dev/null +++ b/tools/install/pre-commit.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR + +# shellcheck source=_common.sh +. "$SCRIPT_DIR/_common.sh" + +# +# Unique part +# + +if [[ $VERSION == latest ]]; then + pip3 install --no-cache-dir "$TOOL" +else + pip3 install --no-cache-dir "${TOOL}==${VERSION}" +fi +pip3 check diff --git a/tools/install/terraform-docs.sh b/tools/install/terraform-docs.sh new file mode 100755 index 000000000..9eec05394 --- /dev/null +++ b/tools/install/terraform-docs.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR +# shellcheck source=_common.sh +. "$SCRIPT_DIR/_common.sh" + +# +# Unique part +# + +GH_ORG="terraform-docs" +GH_RELEASE_REGEX_SPECIFIC_VERSION="https://.+?v${VERSION}-${TARGETOS}-${TARGETARCH}.tar.gz" +GH_RELEASE_REGEX_LATEST="https://.+?-${TARGETOS}-${TARGETARCH}.tar.gz" +DISTRIBUTED_AS="tar.gz" + +common::install_from_gh_release "$GH_ORG" "$DISTRIBUTED_AS" \ + "$GH_RELEASE_REGEX_LATEST" "$GH_RELEASE_REGEX_SPECIFIC_VERSION" diff --git a/tools/install/terraform.sh b/tools/install/terraform.sh new file mode 100755 index 000000000..65ec21c2b --- /dev/null +++ b/tools/install/terraform.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR +# shellcheck source=_common.sh +. "$SCRIPT_DIR/_common.sh" + +# +# Unique part +# +# shellcheck disable=SC2153 # We are using the variable from _common.sh +if [[ $VERSION == latest ]]; then + version="$(curl -s https://api.github.com/repos/hashicorp/terraform/releases/latest | grep tag_name | grep -o -E -m 1 "[0-9.]+")" +else + version=$VERSION +fi +readonly version + +curl -L "https://releases.hashicorp.com/terraform/${version}/${TOOL}_${version}_${TARGETOS}_${TARGETARCH}.zip" > "${TOOL}.zip" +unzip "${TOOL}.zip" "$TOOL" +rm "${TOOL}.zip" diff --git a/tools/install/terragrunt.sh b/tools/install/terragrunt.sh new file mode 100755 index 000000000..20cc60ff7 --- /dev/null +++ b/tools/install/terragrunt.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR +# shellcheck source=_common.sh +. "$SCRIPT_DIR/_common.sh" + +# +# Unique part +# + +GH_ORG="gruntwork-io" +GH_RELEASE_REGEX_SPECIFIC_VERSION="https://.+?v${VERSION}/${TOOL}_${TARGETOS}_${TARGETARCH}" +GH_RELEASE_REGEX_LATEST="https://.+?/${TOOL}_${TARGETOS}_${TARGETARCH}" +DISTRIBUTED_AS="binary" + +common::install_from_gh_release "$GH_ORG" "$DISTRIBUTED_AS" \ + "$GH_RELEASE_REGEX_LATEST" "$GH_RELEASE_REGEX_SPECIFIC_VERSION" diff --git a/tools/install/terrascan.sh b/tools/install/terrascan.sh new file mode 100755 index 000000000..4393159d3 --- /dev/null +++ b/tools/install/terrascan.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR +# shellcheck source=_common.sh +. "$SCRIPT_DIR/_common.sh" + +# +# Unique part +# + +[[ $TARGETARCH == amd64 ]] && ARCH="x86_64" || ARCH="$TARGETARCH" +readonly ARCH +# Convert the first letter to Uppercase +OS="${TARGETOS^}" + +GH_ORG="tenable" +GH_RELEASE_REGEX_SPECIFIC_VERSION="https://.+?${VERSION}_${OS}_${ARCH}.tar.gz" +GH_RELEASE_REGEX_LATEST="https://.+?_${OS}_${ARCH}.tar.gz" +DISTRIBUTED_AS="tar.gz" + +common::install_from_gh_release "$GH_ORG" "$DISTRIBUTED_AS" \ + "$GH_RELEASE_REGEX_LATEST" "$GH_RELEASE_REGEX_SPECIFIC_VERSION" + +# Download (caching) terrascan rego policies to save time during terrascan run +# https://runterrascan.io/docs/usage/_print/#pg-2cba380a2ef14e4ae3c674e02c5f9f53 +./"$TOOL" init diff --git a/tools/install/tflint.sh b/tools/install/tflint.sh new file mode 100755 index 000000000..ac2556b81 --- /dev/null +++ b/tools/install/tflint.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR +# shellcheck source=_common.sh +. "$SCRIPT_DIR/_common.sh" + +# +# Unique part +# + +GH_ORG="terraform-linters" +GH_RELEASE_REGEX_SPECIFIC_VERSION="https://.+?/v${VERSION}/${TOOL}_${TARGETOS}_${TARGETARCH}.zip" +GH_RELEASE_REGEX_LATEST="https://.+?_${TARGETOS}_${TARGETARCH}.zip" +DISTRIBUTED_AS="zip" + +common::install_from_gh_release "$GH_ORG" "$DISTRIBUTED_AS" \ + "$GH_RELEASE_REGEX_LATEST" "$GH_RELEASE_REGEX_SPECIFIC_VERSION" diff --git a/tools/install/tfsec.sh b/tools/install/tfsec.sh new file mode 100755 index 000000000..3c9c2430d --- /dev/null +++ b/tools/install/tfsec.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR +# shellcheck source=_common.sh +. "$SCRIPT_DIR/_common.sh" + +# +# Unique part +# + +GH_ORG="aquasecurity" +GH_RELEASE_REGEX_SPECIFIC_VERSION="https://.+?v${VERSION}/${TOOL}-${TARGETOS}-${TARGETARCH}" +GH_RELEASE_REGEX_LATEST="https://.+?/${TOOL}-${TARGETOS}-${TARGETARCH}" +DISTRIBUTED_AS="binary" + +common::install_from_gh_release "$GH_ORG" "$DISTRIBUTED_AS" \ + "$GH_RELEASE_REGEX_LATEST" "$GH_RELEASE_REGEX_SPECIFIC_VERSION" diff --git a/tools/install/tfupdate.sh b/tools/install/tfupdate.sh new file mode 100755 index 000000000..498e4fb6f --- /dev/null +++ b/tools/install/tfupdate.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR +# shellcheck source=_common.sh +. "$SCRIPT_DIR/_common.sh" + +# +# Unique part +# + +GH_ORG="minamijoyo" +GH_RELEASE_REGEX_SPECIFIC_VERSION="https://.+?${VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz" +GH_RELEASE_REGEX_LATEST="https://.+?_${TARGETOS}_${TARGETARCH}.tar.gz" +DISTRIBUTED_AS="tar.gz" + +common::install_from_gh_release "$GH_ORG" "$DISTRIBUTED_AS" \ + "$GH_RELEASE_REGEX_LATEST" "$GH_RELEASE_REGEX_SPECIFIC_VERSION" diff --git a/tools/install/trivy.sh b/tools/install/trivy.sh new file mode 100755 index 000000000..c07625b53 --- /dev/null +++ b/tools/install/trivy.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +readonly SCRIPT_DIR +# shellcheck source=_common.sh +. "$SCRIPT_DIR/_common.sh" + +# +# Unique part +# + +[[ $TARGETARCH == amd64 ]] && ARCH="64bit" || ARCH="$TARGETARCH" +readonly ARCH + +GH_ORG="aquasecurity" +GH_RELEASE_REGEX_SPECIFIC_VERSION="https://.+?/v${VERSION}/${TOOL}_.+?_${TARGETOS}-${ARCH}.tar.gz" +GH_RELEASE_REGEX_LATEST="https://.+?/${TOOL}_.+?_${TARGETOS}-${ARCH}.tar.gz" +DISTRIBUTED_AS="tar.gz" + +common::install_from_gh_release "$GH_ORG" "$DISTRIBUTED_AS" \ + "$GH_RELEASE_REGEX_LATEST" "$GH_RELEASE_REGEX_SPECIFIC_VERSION" diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..0b0f26b6b --- /dev/null +++ b/tox.ini @@ -0,0 +1,303 @@ +[tox] +isolated_build = true + + +[python-cli-options] +byte-warnings = -b +byte-errors = -bb +max-isolation = -E -s -I +some-isolation = -E -s +warnings-to-errors = -Werror + + +[testenv] +description = Run pytest under {envpython} +dependency_groups = + testing + +# In: +# 'tox run -e py -- --lf', 'tox run -- --lf', 'tox run -e py313,py312 -- --lf' +# '{posargs}' (positional arguments) == '--lf' +commands = + {envpython} \ + {[python-cli-options]byte-errors} \ + {[python-cli-options]max-isolation} \ + {[python-cli-options]warnings-to-errors} \ + -W 'ignore:Coverage failure::pytest_cov.plugin' \ + -m pytest \ + {tty:--color=yes} \ + {posargs:--cov-report=html:{envtmpdir}{/}htmlcov{/}} +commands_post = + # Create GHA Job Summary markdown table of the coverage report + # https://github.blog/news-insights/product-news/supercharging-github-actions-with-job-summaries/ + # a leading '-' suppresses non-zero return codes + -{envpython} \ + {[python-cli-options]byte-errors} \ + {[python-cli-options]max-isolation} \ + {[python-cli-options]warnings-to-errors} \ + -c \ + 'import atexit, os, sys; \ + os.getenv("GITHUB_ACTIONS") == "true" or sys.exit(); \ + import coverage; \ + gh_summary_fd = open(\ + os.environ["GITHUB_STEP_SUMMARY"], encoding="utf-8", mode="a",\ + ); \ + atexit.register(gh_summary_fd.close); \ + cov = coverage.Coverage(); \ + cov.load(); \ + cov.report(file=gh_summary_fd, output_format="markdown")' + # Expose the coverage & test run XML report paths into GHA + {envpython} \ + {[python-cli-options]byte-errors} \ + {[python-cli-options]max-isolation} \ + {[python-cli-options]warnings-to-errors} \ + -c \ + 'import os, pathlib, sys; \ + os.getenv("GITHUB_ACTIONS") == "true" or sys.exit(); \ + cov_report_arg_prefix = "--cov-report=xml:"; \ + test_report_arg_prefix = "--junitxml="; \ + cov_reports = [\ + arg[len(cov_report_arg_prefix):] for arg in sys.argv \ + if arg.startswith(cov_report_arg_prefix)\ + ]; \ + test_reports = [\ + arg[len(test_report_arg_prefix):] for arg in sys.argv \ + if arg.startswith(test_report_arg_prefix)\ + ]; \ + cov_report_file = cov_reports[-1] if cov_reports else None; \ + test_report_file = test_reports[-1] if test_reports else None; \ + gh_output_fd = open(\ + os.environ["GITHUB_OUTPUT"], encoding="utf-8", mode="a",\ + ); \ + cov_report_file and \ + print(f"cov-report-files={cov_report_file !s}", file=gh_output_fd); \ + test_report_file and \ + print(f"test-result-files={test_report_file !s}", file=gh_output_fd); \ + print("codecov-flags=pytest", file=gh_output_fd); \ + gh_output_fd.close()' \ + {posargs} + # Print out the output coverage dir and a way to serve html: + {envpython} \ + {[python-cli-options]byte-errors} \ + {[python-cli-options]max-isolation} \ + {[python-cli-options]warnings-to-errors} \ + -c\ + 'import pathlib, shlex, sys; \ + cov_html_report_arg_prefix = "--cov-report=html:"; \ + cov_html_reports = [\ + arg[len(cov_html_report_arg_prefix):] for arg in sys.argv \ + if arg.startswith(cov_html_report_arg_prefix)\ + ]; \ + cov_html_reports or sys.exit(); \ + cov_html_report_dir = pathlib.Path(cov_html_reports[-1]); \ + index_file = cov_html_report_dir / "index.html";\ + index_file.exists() or sys.exit(); \ + html_url = f"file://\{index_file\}";\ + browse_cmd = shlex.join(("python3", "-Im", "webbrowser", html_url)); \ + serve_cmd = shlex.join((\ + "python3", "-Im", "http.server", \ + "--directory", str(cov_html_report_dir), "0", \ + )); \ + print(f"\nTo open the HTML coverage report, run\n\n\ + \t\{browse_cmd !s\}\n");\ + print(f"To serve \ + the HTML coverage report with a local web server, use\n\n\ + \t\{serve_cmd !s\}\n")' \ + {posargs:--cov-report=html:{envtmpdir}{/}htmlcov{/}} +package = editable +pass_env = + CI + GITHUB_* + SSH_AUTH_SOCK + TERM +set_env = + COVERAGE_PROCESS_START = {toxinidir}{/}.coveragerc +wheel_build_env = .pkg + +# Duplicate default 'py' env to 'pytest' to be able run pytest with 'tox run -e pytest' +[testenv:pytest] + + +[testenv:cleanup-dists] +description = + Wipe the the dist{/} folder +dependency_groups = +commands_pre = +commands = + {envpython} \ + {[python-cli-options]byte-errors} \ + {[python-cli-options]max-isolation} \ + {[python-cli-options]warnings-to-errors} \ + -c \ + 'import os, shutil, sys; \ + dists_dir = "{toxinidir}{/}dist{/}"; \ + shutil.rmtree(dists_dir, ignore_errors=True); \ + sys.exit(os.path.exists(dists_dir))' +commands_post = +package = skip + + +[testenv:build-dists] +description = + Build dists with {basepython} and put them into the dist{/} folder +dependency_groups = + building +depends = + cleanup-dists +commands = + {envpython} \ + {[python-cli-options]byte-errors} \ + {[python-cli-options]max-isolation} \ + {[python-cli-options]warnings-to-errors} \ + -m build \ + {posargs:} +commands_post = +package = skip + + +[testenv:metadata-validation] +description = + Verify that dists under the `dist{/}` dir + have valid metadata +dependency_groups = + upstreaming +depends = + build-dists +commands = + {envpython} \ + {[python-cli-options]byte-errors} \ + {[python-cli-options]max-isolation} \ + {[python-cli-options]warnings-to-errors} \ + -m twine \ + check \ + --strict \ + dist{/}* +commands_post = +package = skip + +# In: +# 'tox run -e pre-commit -- mypy-py313 --all' +# '{posargs}' == 'mypy-py313 --all' +[testenv:pre-commit] +description = + Run the quality checks under {basepython}; run as + `SKIP=check-id1,check-id2 tox r -e pre-commit` to instruct the underlying + `pre-commit` invocation avoid running said checks; Use + `tox r -e pre-commit -- check-id1 --all-files` to select checks matching IDs + aliases{:} `tox r -e pre-commit -- mypy --all-files` will run 3 MyPy + invocations, but `tox r -e pre-commit -- mypy-py313 --all-files` runs one. +commands = + {envpython} \ + {[python-cli-options]byte-errors} \ + {[python-cli-options]max-isolation} \ + {[python-cli-options]warnings-to-errors} \ + -m pre_commit \ + run \ + --color=always \ + --show-diff-on-failure \ + {posargs:--all-files} + + # Print out the advice on how to install pre-commit from this env into Git: + # a leading '-' suppresses non-zero return codes + -{envpython} \ + {[python-cli-options]byte-errors} \ + {[python-cli-options]max-isolation} \ + {[python-cli-options]warnings-to-errors} \ + -c \ + 'cmd = "{envpython} -m pre_commit install"; \ + scr_width = len(cmd) + 10; \ + sep = "=" * scr_width; \ + cmd_str = " $ \{cmd\}";' \ + 'print(f"\n\{sep\}\nTo install pre-commit hooks into the Git repo, run:\ + \n\n\{cmd_str\}\n\n\{sep\}\n")' +commands_post = + {envpython} \ + {[python-cli-options]byte-errors} \ + {[python-cli-options]max-isolation} \ + {[python-cli-options]warnings-to-errors} \ + -c \ + 'import os, pathlib, sys; \ + os.getenv("GITHUB_ACTIONS") == "true" or sys.exit(); \ + project_root_path = pathlib.Path(r"{toxinidir}"); \ + test_results_dir = pathlib.Path(r"{temp_dir}") / ".test-results"; \ + coverage_result_files = ",".join(\ + str(xml_path.relative_to(project_root_path)) \ + for xml_path in test_results_dir.glob("mypy--py-*{/}cobertura.xml")\ + ); \ + gh_output_fd = open(\ + os.environ["GITHUB_OUTPUT"], encoding="utf-8", mode="a",\ + ); \ + print(\ + f"cov-report-files={coverage_result_files !s}", file=gh_output_fd\ + ); \ + print("codecov-flags=MyPy", file=gh_output_fd); \ + gh_output_fd.close()' + # Publish available MyPy-produced text and JSON reports wrapped as Markdown code blocks, to a GHA job summary + {envpython} \ + {[python-cli-options]byte-errors} \ + {[python-cli-options]max-isolation} \ + {[python-cli-options]warnings-to-errors} \ + -c \ + 'import itertools, os, pathlib, shlex, sys; \ + os.getenv("GITHUB_ACTIONS") == "true" or sys.exit(); \ + test_results_dir = pathlib.Path(r"{temp_dir}") / ".test-results"; \ + text_and_json_reports = itertools.chain( \ + test_results_dir.glob("mypy--py-*{/}*.json"), \ + test_results_dir.glob("mypy--py-*{/}*.txt"), \ + ); \ + report_contents = { \ + report{:} report.read_text() \ + for report in text_and_json_reports \ + }; \ + reports_summary_text_blob = "\n\n".join( \ + f"\N\{NUMBER SIGN\}\N\{NUMBER SIGN\} {report_path.parent.name}{:} " \ + f"`{report_path.name}`\n\n" \ + f"```{report_path.suffix[1:]}\n{report_text}\n```\n" \ + for report_path, report_text in report_contents.items() \ + ); \ + gh_summary_fd = open( \ + os.environ["GITHUB_STEP_SUMMARY"], encoding="utf-8", mode="a", \ + ); \ + print(reports_summary_text_blob, file=gh_summary_fd); \ + gh_summary_fd.close()' + # Print out the output coverage dir and a way to serve html: + {envpython} \ + {[python-cli-options]byte-errors} \ + {[python-cli-options]max-isolation} \ + {[python-cli-options]warnings-to-errors} \ + -c\ + 'import os, pathlib, sys; \ + os.getenv("GITHUB_ACTIONS") == "true" and sys.exit(); \ + len(sys.argv) >= 3 and all(\ + arg != "mypy" and not arg.startswith("mypy-py3") \ + for arg in sys.argv \ + ) and sys.exit(); \ + project_root_path = pathlib.Path(r"{toxinidir}"); \ + test_results_dir = pathlib.Path(r"{temp_dir}") / ".test-results"; \ + coverage_html_report_urls = [\ + f"file://\{xml_path !s\}" \ + for xml_path in test_results_dir.glob("mypy--py-*{/}index.html")\ + ]; \ + coverage_html_report_open_cmds = [\ + f"python3 -Im webbrowser \N\{QUOTATION MARK\}\{html_url !s\}\N\{QUOTATION MARK\}" \ + for html_url in coverage_html_report_urls\ + ]; \ + coverage_html_report_open_cmds_blob = "\n\n\t".join(\ + coverage_html_report_open_cmds,\ + ); \ + print(\ + f"\nTo open the HTML coverage reports, run\n\n\ + \t\{coverage_html_report_open_cmds_blob !s\}\n"\ + ); \ + print(\ + f"[*] Find rest of JSON and text reports, are in the same directories."\ + )\ + ' \ + {posargs:--all-files} +dependency_groups = + linting +isolated_build = true +package = skip +pass_env = + {[testenv]pass_env} + SKIP # set this variable