From d5ecdcfaa1175158f73809cec5d9bcf1506a89ad Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 20 Nov 2024 12:35:10 +0100 Subject: [PATCH 01/17] add .pre-commit-config.yaml --- .pre-commit-config.yaml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..8c585ad --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,33 @@ +--- +ci: + skip: [flake8] +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.7.4 + hooks: + # Run the linter. + - id: ruff + args: [--fix, --preview] + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: v0.42.0 + hooks: + - id: markdownlint-fix + # Using this mirror lets us use mypyc-compiled black, which is about 2x faster + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.10.0 + hooks: + - id: black-jupyter + - repo: https://github.com/pycqa/flake8 + rev: 7.1.1 + hooks: + - id: flake8 + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.35.1 + hooks: + - id: yamllint From 8618d170cffd612ff79320c57d7fc12129bd5a29 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 20 Nov 2024 13:20:26 +0100 Subject: [PATCH 02/17] add ruff config --- pyproject.toml | 107 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index de28677..5b5bd16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,4 +31,109 @@ dependencies = [ where = ["src"] [tool.setuptools.package-data] -actinia = ["templates/*.json"] \ No newline at end of file +actinia = ["templates/*.json"] + +[tool.black] +required-version = '24' +line-length = 88 +target-version = ['py39', 'py310', 'py311', 'py312', 'py313'] +# 'extend-exclude' excludes files or directories in addition to the defaults + +[tool.ruff] + +required-version = ">=0.6.0" + +# Same as Black. +line-length = 88 +indent-width = 4 + +[tool.ruff.lint] +# See https://docs.astral.sh/ruff/rules/ +select = [ + # "A", # flake8-builtins (A) + "AIR", # Airflow (AIR) + # "ANN", # flake8-annotations (ANN) + "ARG", # flake8-unused-arguments (ARG) + # "B", # flake8-bugbear (B) + # "BLE", # flake8-blind-except (BLE) + # "C4", # flake8-comprehensions (C4) + "COM", # flake8-commas (COM) + # "D", # pydocstyle (D) + # "DTZ", # flake8-datetimez (DTZ) + "E", # pycodestyle (E, W) + "F", # Pyflakes (F) + "FA", # flake8-future-annotations (FA) + "FBT", # flake8-boolean-trap (FBT) + "FLY", # flynt (FLY) + # "FURB", # refurb (FURB) + "G", # flake8-logging-format (G) + "I", # isort (I) + "ICN", # flake8-import-conventions (ICN) + "INT", # flake8-gettext (INT) + "ISC", # flake8-implicit-str-concat (ISC) + "LOG", # flake8-logging (LOG) + # "N", # pep8-naming (N) + "NPY", # NumPy-specific rules (NPY) + "PERF", # Perflint (PERF) + "PGH", # pygrep-hooks (PGH) + "PIE", # flake8-pie (PIE) + "PLC", # Pylint (PL) Convention (C) + "PLE", # Pylint (PL) Error (E) + # "PLR", # Pylint (PL) Refactor (R) + # "PLW", # Pylint (PL) Warning (W) + # "PT", # flake8-pytest-style (PT) + # "PTH", # flake8-use-pathlib (PTH) + "PYI", # flake8-pyi (PYI) + "Q", # flake8-quotes (Q) + # "RET", # flake8-return (RET) + "RSE", # flake8-raise (RSE) + "RUF", # Ruff-specific rules (RUF) + # "S", # flake8-bandit (S) + # "SIM", # flake8-simplify (SIM) + "SLF", # flake8-self (SLF) + "SLOT", # flake8-slots (SLOT) + "T10", # flake8-debugger (T10) + "TCH", # flake8-type-checking (TCH) + "TID", # flake8-tidy-imports (TID) + # "TRY", # tryceratops (TRY) + "UP", # pyupgrade (UP) + "W", # pycodestyle (E, W) + "YTT", # flake8-2020 (YTT) +] + +ignore = [ + # See https://docs.astral.sh/ruff/rules/ + "ANN201", + "E501", + "FBT002", + "PLC0414", + "PLR6301", + "S101", +] + +[tool.ruff.format] +# Like Black, use double quotes for strings. +quote-style = "double" + +# Like Black, indent with spaces, rather than tabs. +indent-style = "space" + +# Like Black, respect magic trailing commas. +skip-magic-trailing-comma = false + +# Like Black, automatically detect the appropriate line ending. +line-ending = "auto" + +# Enable auto-formatting of code examples in docstrings. Markdown, +# reStructuredText code/literal blocks and doctests are all supported. +# +# This is currently disabled by default, but it is planned for this +# to be opt-out in the future. +docstring-code-format = false + +# Set the line length limit used when formatting code snippets in +# docstrings. +# +# This only has an effect when the `docstring-code-format` setting is +# enabled. +docstring-code-line-length = "dynamic" From 821ed4eb91bf499e059ba223bc112f2f70687f32 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 20 Nov 2024 14:36:31 +0100 Subject: [PATCH 03/17] code ql workflow --- .github/workflows/python-code-quality.yml | 140 ++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 .github/workflows/python-code-quality.yml diff --git a/.github/workflows/python-code-quality.yml b/.github/workflows/python-code-quality.yml new file mode 100644 index 0000000..0bfbe97 --- /dev/null +++ b/.github/workflows/python-code-quality.yml @@ -0,0 +1,140 @@ +--- +name: Python Code Quality + +on: + push: + branches: + - main + - pre_commit_ruff + pull_request: + - main + +jobs: + python-checks: + name: Python Code Quality Checks + + concurrency: + group: ${{ github.workflow }}-${{ github.job }}-${{ + github.event_name == 'pull_request' && + github.head_ref || github.sha }} + cancel-in-progress: true + + strategy: + matrix: + include: + - os: ubuntu-22.04 + + env: + # renovate: datasource=python-version depName=python + PYTHON_VERSION: "3.10" + MIN_PYTHON_VERSION: "3.8" + # renovate: datasource=pypi depName=black + BLACK_VERSION: "24.10.0" + # renovate: datasource=pypi depName=flake8 + FLAKE8_VERSION: "7.1.1" + # renovate: datasource=pypi depName=pylint + PYLINT_VERSION: "2.12.2" + # renovate: datasource=pypi depName=bandit + BANDIT_VERSION: "1.7.10" + # renovate: datasource=pypi depName=ruff + RUFF_VERSION: "0.7.4" + + runs-on: ${{ matrix.os }} + permissions: + security-events: write + + steps: + - name: Versions + run: | + echo OS: ${{ matrix.os }} + echo Python: ${{ env.PYTHON_VERSION }} + echo Minimal Python version: ${{ env.MIN_PYTHON_VERSION }} + echo Black: ${{ env.BLACK_VERSION }} + echo Flake8: ${{ env.FLAKE8_VERSION }} + echo Pylint: ${{ env.PYLINT_VERSION }} + echo Bandit: ${{ env.BANDIT_VERSION }} + echo Ruff: ${{ env.RUFF_VERSION }} + + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: pip + - name: Upgrade pip + run: python -m pip install --upgrade pip + + - name: Install Ruff + run: pip install ruff==${{ env.RUFF_VERSION }} + - name: Run Ruff (output annotations on fixable errors) + run: ruff check --output-format=github . --preview --unsafe-fixes + continue-on-error: true + - name: Run Ruff (apply fixes for suggestions) + run: ruff check . --preview --fix --unsafe-fixes + - name: Create and uploads code suggestions to apply for Ruff + # Will fail fast here if there are changes required + id: diff-ruff + # To run after ruff step exits with failure when rules don't have fixes available + if: ${{ !cancelled() }} + uses: ./.github/actions/create-upload-suggestions + with: + tool-name: ruff + # To keep repo's file structure in formatted changes artifact + extra-upload-changes: pyproject.toml + + - name: Install Black only + run: pip install black[jupyter]==${{ env.BLACK_VERSION }} + + - name: Run Black + run: black . + + - name: Create and uploads code suggestions to apply for Black + # Will fail fast here if there are changes required + id: diff-black + uses: ./.github/actions/create-upload-suggestions + with: + tool-name: black + # To keep repo's file structure in formatted changes artifact + extra-upload-changes: .clang-format + + - name: Install Python dependencies + run: | + pip install --user pipx + pipx ensurepath + pipx install flake8==${{ env.FLAKE8_VERSION }} + pipx install pylint==${{ env.PYLINT_VERSION }} + # The extra toml is only needed before Python 3.11 + pipx install bandit[sarif,toml]==${{ env.BANDIT_VERSION }} + + - name: Run Flake8 + run: | + flake8 --count --statistics --show-source --jobs=$(nproc) . + + - name: Run Flake8 on additional files + run: | + flake8 --count --statistics --show-source --jobs=$(nproc) python/grass/{script,jupyter}/testsuite/ + + - name: Bandit Vulnerability Scan + run: | + bandit -c pyproject.toml -iii -r . -f sarif -o bandit.sarif --exit-zero + + - name: Upload Bandit Scan Results + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: bandit.sarif + path: bandit.sarif + + - name: Upload SARIF File into Security Tab + uses: github/codeql-action/upload-sarif@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4 + with: + sarif_file: bandit.sarif + + python-success: + name: Python Code Quality Result + needs: + - python-checks + if: ${{ always() }} + uses: ./.github/workflows/verify-success.yml + with: + needs_context: ${{ toJson(needs) }} From 9aede8faa71ec431be2d36ded0627ff947ea7d05 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 20 Nov 2024 14:37:01 +0100 Subject: [PATCH 04/17] add yamllint config --- .yamllint | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .yamllint diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..1b60099 --- /dev/null +++ b/.yamllint @@ -0,0 +1,15 @@ +--- + +extends: default + +rules: + comments: + level: error + min-spaces-from-content: 1 + document-start: + level: error + line-length: + level: error + max: 120 + allow-non-breakable-inline-mappings: true + truthy: disable From 66655d324fc450728aa9dbf4672fe88447c558cc Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 20 Nov 2024 14:40:55 +0100 Subject: [PATCH 05/17] fix on section --- .github/workflows/python-code-quality.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/python-code-quality.yml b/.github/workflows/python-code-quality.yml index 0bfbe97..23d9ec4 100644 --- a/.github/workflows/python-code-quality.yml +++ b/.github/workflows/python-code-quality.yml @@ -7,7 +7,6 @@ on: - main - pre_commit_ruff pull_request: - - main jobs: python-checks: From a439d28a8ab9d80425f39fd394c9a1463c692af3 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 20 Nov 2024 14:43:53 +0100 Subject: [PATCH 06/17] add verify-success from GRASS GIS --- .github/workflows/verify-success.yml | 155 +++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 .github/workflows/verify-success.yml diff --git a/.github/workflows/verify-success.yml b/.github/workflows/verify-success.yml new file mode 100644 index 0000000..237c239 --- /dev/null +++ b/.github/workflows/verify-success.yml @@ -0,0 +1,155 @@ +--- +name: Verify Success reusable workflow + +# Use this reusable workflow as a job of workflow to check that +# all jobs, including ones ran through a matrix, were successful. +# This job can then be used as a required status check in the +# repo's rulesets, that allows to change the required jobs or +# the matrix values of a required job without needing to change +# the rulesets settings. In the future, GitHub might have a +# solution to this natively. + +# This reusable workflow has inputs to change what is required +# to have this workflow pass. It handles the cases were there were +# skipped jobs, and no successful jobs. + +# The jobs to check must set as the `needs` for the job calling this +# reusable workflow. This also means that the job ids should be in the +# same workflow file. The calling job must be set to always run to be +# triggered when jobs are skipped or cancelled. +# Then, set the `needs_context` input like: +# `needs_context: ${{ toJson(needs) }}` + +# Example usage, as a job inside a workflow: +# ``` +# jobs: +# a-job-id: +# ... +# another-job-id: +# ... +# some-job-success: +# name: Some Job Result +# needs: +# - a-job-id +# - another-job-id +# if: ${{ always() }} +# uses: ./.github/workflows/verify-success.yml +# with: +# needs_context: ${{ toJson(needs) }} +# ``` + +on: + workflow_call: + inputs: + needs_context: + type: string + required: true + # Can't escape the handlebars in the description + description: + In the calling job that defines all the needed jobs, + send `toJson(needs)` inside `$` followed by `{{ }}` + fail_if_failure: + type: boolean + default: true + description: + If true, this workflow will fail if any job from 'needs_context was + failed + fail_if_cancelled: + type: boolean + default: true + description: + If true, this workflow will fail if any job from 'needs_context' was + cancelled + fail_if_skipped: + type: boolean + default: false + description: + If true, this workflow will fail if any job from 'needs_context' was + skipped + require_success: + type: boolean + default: true + description: + If true, this workflow will fail if no job from 'needs_context' was + successful + +jobs: + verify-success: + name: Success + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Set outputs for each job result type + id: has-result + run: | + echo "failure=${{ + contains(env.NEEDS_RESULT, 'failure') }}" >> "$GITHUB_OUTPUT" + echo "cancelled=${{ + contains(env.NEEDS_RESULT, 'cancelled') }}" >> "$GITHUB_OUTPUT" + echo "skipped=${{ + contains(env.NEEDS_RESULT, 'skipped') }}" >> "$GITHUB_OUTPUT" + echo "success=${{ + contains(env.NEEDS_RESULT, 'success') }}" >> "$GITHUB_OUTPUT" + env: + NEEDS_RESULT: ${{ toJson(fromJson(inputs.needs_context).*.result) }} + - name: Set exit codes for each job result type + id: exit-code + run: | + echo "failure=${{ inputs.fail_if_failure && + fromJson(steps.has-result.outputs.failure) && 1 || 0 + }}" >> "$GITHUB_OUTPUT" + echo "cancelled=${{ inputs.fail_if_cancelled && + fromJson(steps.has-result.outputs.cancelled) && 1 || 0 + }}" >> "$GITHUB_OUTPUT" + echo "skipped=${{ inputs.fail_if_skipped && + fromJson(steps.has-result.outputs.skipped) && 1 || 0 + }}" >> "$GITHUB_OUTPUT" + echo "success=${{ inputs.require_success && + !fromJson(steps.has-result.outputs.success) && 1 || 0 + }}" >> "$GITHUB_OUTPUT" + - name: Set messages for each job result type + id: message + run: | + echo "failure=${{ format('{0}{1} were failed', + (steps.exit-code.outputs.failure == 1) && env.P1 || env.P2, + (steps.has-result.outputs.failure == 'true') && env.M1 || env.M2) + }}" >> "$GITHUB_OUTPUT" + echo "cancelled=${{ format('{0}{1} were cancelled', + (steps.exit-code.outputs.cancelled == 1) && env.P1 || env.P2, + (steps.has-result.outputs.cancelled == 'true') && env.M1 || env.M2) + }}" >> "$GITHUB_OUTPUT" + echo "skipped=${{ format('{0}{1} were skipped', + (steps.exit-code.outputs.skipped == 1) && env.P1 || env.P2, + (steps.has-result.outputs.skipped == 'true') && env.M1 || env.M2) + }}" >> "$GITHUB_OUTPUT" + echo "success=${{ format('{0}{1} were successful', + (steps.exit-code.outputs.success == 1) && env.P1 || env.P2, + (steps.has-result.outputs.success == 'true') && env.M1 || env.M2) + }}" >> "$GITHUB_OUTPUT" + env: + P1: "::error ::" # Common message prefix if step will fail + P2: "" # Common message prefix if step will not fail + M1: "Some jobs" # Common message if result is true + M2: "No jobs" # Common message if result is false + + - name: Check for failed jobs + run: | + echo "${{ steps.message.outputs.failure }}" + exit ${{ steps.exit-code.outputs.failure }} + - name: Check for cancelled jobs + run: | + echo "${{ steps.message.outputs.cancelled }}" + exit ${{ steps.exit-code.outputs.cancelled }} + - name: Check for skipped jobs + run: | + echo "${{ steps.message.outputs.skipped }}" + exit ${{ steps.exit-code.outputs.skipped }} + - name: Check for successful jobs + run: | + echo "${{ steps.message.outputs.success }}" + exit ${{ steps.exit-code.outputs.success }} + + - run: echo "Checks passed successfully" + if: ${{ success() }} + - run: echo "Checks failed" + if: ${{ !success() }} From d4ab76b95f2704aace2b84bdfa3243a1e5e1abc9 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 20 Nov 2024 14:46:41 +0100 Subject: [PATCH 07/17] add more excludes --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 5b5bd16..d680933 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,6 +104,9 @@ select = [ ignore = [ # See https://docs.astral.sh/ruff/rules/ "ANN201", + "COM812", + "E265", + "E266", "E501", "FBT002", "PLC0414", From e4afcc1a8268dff456eb2bc61c8cd6131e888aa3 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 20 Nov 2024 14:48:56 +0100 Subject: [PATCH 08/17] add more excludes --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index d680933..9407294 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,6 +108,7 @@ ignore = [ "E265", "E266", "E501", + "E721", "FBT002", "PLC0414", "PLR6301", From eb28c3e98d8ae5a7ff03a3f787c254a795c49da8 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 20 Nov 2024 14:51:16 +0100 Subject: [PATCH 09/17] add more excludes --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 9407294..56b4456 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,10 +109,14 @@ ignore = [ "E266", "E501", "E721", + "F401", "FBT002", + "I001", "PLC0414", "PLR6301", "S101", + "UP009", + "UP015", ] [tool.ruff.format] From 43260e96a3a0c03e78a3a420eff6f10e4e918b7f Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 20 Nov 2024 14:53:04 +0100 Subject: [PATCH 10/17] add more excludes --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 56b4456..a67ea93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,6 +115,7 @@ ignore = [ "PLC0414", "PLR6301", "S101", + "UP004", "UP009", "UP015", ] From f61350aac43caf8ef6194e09ae31de99c1e7e5de Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 20 Nov 2024 14:57:28 +0100 Subject: [PATCH 11/17] add ruff suggestion from GRASS GIS --- .../create-upload-suggestions/action.yml | 240 ++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 .github/actions/create-upload-suggestions/action.yml diff --git a/.github/actions/create-upload-suggestions/action.yml b/.github/actions/create-upload-suggestions/action.yml new file mode 100644 index 0000000..b80c08c --- /dev/null +++ b/.github/actions/create-upload-suggestions/action.yml @@ -0,0 +1,240 @@ +--- +name: "Create and upload suggestions" +description: "Creates a file containing changes applied by a tool" +author: echoix +inputs: + tool-name: + description: "The tool name, used for creating the file names" + required: true + upload-suggestions: + description: >- + If "true", the suggestions diff file will be uploaded here. + Otherwise, with "false" or any other value, the diff artifact won't be + uploaded. Not uploading directly could be used when multiple tools are + used, and each has a diff file that would be uploaded into one artifact. + default: "true" + required: false + extra-upload-suggestions: + description: >- + Extra paths to add to the diff artifact upload containing suggestions. + default: "" + required: false + upload-changes: + description: >- + If "true", the changed files will be uploaded to an artifact. + Otherwise, with "false" or any other value, the artifact containing + changed files won't be uploaded. + default: "true" + required: false + extra-upload-changes: + description: >- + Extra paths to add to the artifact containing changed files. + It is recommended to add at least one file from the root of the repo to + prevent flattening of the repo structure. + default: "" + required: false + fail-if-changed: + description: >- + If "true", the action will exit with a failure if there are some + uncommitted changed files. + default: "true" + required: false + fail-message: + description: >- + Message to display when files have changed and the `fail-if-changed` + input is set to `true`. + default: "Files have changed." + required: false +outputs: + changed-files: + description: List of files that were changed, line separated + value: ${{ steps.git-changed-files.outputs.CHANGED_FILES }} + files_changed: + description: '"true" if some files were changed, "false" otherwise.' + value: ${{ steps.files_changed.outputs.files_changed }} + diff-file-name: + description: Name of the diff file created + value: ${{ steps.diff-patch.outputs.diff-file-name }} + tool-name: + description: Name of the tool, sanitized + value: ${{ steps.tool-name-safe.outputs.tool-name }} + suggestions-artifact-id: + description: Artifact id of the suggestions if uploaded + value: ${{ steps.upload-diff.outputs.artifact-id }} + suggestions-artifact-url: + description: Artifact url of the suggestions if uploaded + value: ${{ steps.upload-diff.outputs.artifact-url }} + changes-artifact-id: + description: Artifact id of the changes if uploaded + value: ${{ steps.upload-changes.outputs.artifact-id }} + changes-artifact-url: + description: Artifact url of the changes if uploaded + value: ${{ steps.upload-changes.outputs.artifact-url }} +runs: + using: "composite" + steps: + - name: Pass inputs through environment variables + # To avoid directly using the inputs in scripts + id: inputs + shell: bash + run: | + echo "tool-name=${INPUT_TOOL_NAME}" >> "${GITHUB_OUTPUT}" + echo "upload-suggestions=${INPUT_UPLOAD_SUGGESTIONS}" >> "${GITHUB_OUTPUT}" + { + echo 'extra-upload-suggestions<> "${GITHUB_OUTPUT}" + echo "upload-changes=${INPUT_UPLOAD_CHANGES}" >> "${GITHUB_OUTPUT}" + { + echo 'extra-upload-changes<> "${GITHUB_OUTPUT}" + echo "fail-if-changed=${INPUT_FAIL_IF_CHANGED}" >> "${GITHUB_OUTPUT}" + { + echo 'fail-message<> "${GITHUB_OUTPUT}" + env: + INPUT_TOOL_NAME: ${{ inputs.tool-name }} + INPUT_UPLOAD_SUGGESTIONS: ${{ inputs.upload-suggestions }} + INPUT_EXTRA_UPLOAD_SUGGESTIONS: ${{ inputs.extra-upload-suggestions }} + INPUT_UPLOAD_CHANGES: ${{ inputs.upload-changes }} + INPUT_EXTRA_UPLOAD_CHANGES: ${{ inputs.extra-upload-changes }} + INPUT_FAIL_IF_CHANGED: ${{ inputs.fail-if-changed }} + INPUT_FAIL_MESSAGE: ${{ inputs.fail-message }} + - name: Sanitize tool name for file name and diff file name + id: tool-name-safe + shell: bash + env: + INPUT_TOOL_NAME: ${{ steps.inputs.outputs.tool-name }} + run: | + sanitize() { # https://stackoverflow.com/a/44811468 + local s="${1?need a string}" # receive input in first argument + s="${s//[^[:alnum:]]/-}" # replace all non-alnum characters to - + s="${s//+(-)/-}" # convert multiple - to single - + s="${s/#-}" # remove - from start + s="${s/%-}" # remove - from end + echo "${s,,}" # convert to lowercase + } + tool_name_safe=$(sanitize "${INPUT_TOOL_NAME}") + echo "New tool name value: ${tool_name_safe}" + echo "tool-name=${tool_name_safe}" >> "${GITHUB_OUTPUT}" + echo "diff-file-name=diff-${tool_name_safe}.patch" >> "${GITHUB_OUTPUT}" + - id: git-changed-files + shell: bash + run: | + { + echo 'CHANGED_FILES<> "${GITHUB_OUTPUT}" + - name: Get if has changed files (list of changed files is not empty) + id: files_changed + shell: bash + run: | + if [[ -n "$CHANGED_FILES" ]]; then + echo "files_changed=true" >> "${GITHUB_OUTPUT}" + else + echo "files_changed=false" >> "${GITHUB_OUTPUT}" + fi + env: + CHANGED_FILES: ${{ steps.git-changed-files.outputs.CHANGED_FILES }} + - name: List all changed files tracked and untracked files + shell: bash + run: | + echo "Changed files: ${{ steps.git-changed-files.outputs.CHANGED_FILES }}" + - name: Add job summary without changed files + shell: bash + if: ${{ steps.files_changed.outputs.files_changed == 'false' }} + run: | + { + echo "### Changed files:" + echo "No files were changed by ${INPUT_TOOL_NAME}" + } >> "${GITHUB_STEP_SUMMARY}" + env: + INPUT_TOOL_NAME: ${{ steps.inputs.outputs.tool-name }} + - name: Add job summary with changed files + if: ${{ steps.files_changed.outputs.files_changed == 'true' }} + shell: bash + run: | + { + echo '### Changed files:' + echo '```' + echo "${CHANGED_FILES}" + echo '```' + } >> "${GITHUB_STEP_SUMMARY}" + env: + CHANGED_FILES: ${{ steps.git-changed-files.outputs.CHANGED_FILES }} + - name: Create unified diff of changes + if: ${{ steps.files_changed.outputs.files_changed == 'true' }} + id: diff-patch + shell: bash + run: | + git diff --unified=0 --no-color --output="${INPUT_DIFF_FILE_NAME}" + echo "diff-file-name=${INPUT_DIFF_FILE_NAME}" >> "${GITHUB_OUTPUT}" + env: + INPUT_DIFF_FILE_NAME: ${{ steps.tool-name-safe.outputs.diff-file-name }} + - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + id: upload-diff + if: >- + ${{ (steps.files_changed.outputs.files_changed == 'true') && + (steps.inputs.outputs.upload-suggestions == 'true') }} + with: + name: diff + if-no-files-found: ignore + retention-days: 1 + path: | + ${{ steps.diff-patch.outputs.diff-file-name }} + ${{ steps.inputs.outputs.extra-upload-suggestions }} + - name: Add note to summary explaining that code suggestions will be applied if it is a PR + if: >- + ${{ (github.event_name == 'pull_request') && + (steps.files_changed.outputs.files_changed == 'true') }} + shell: bash + run: | + { + echo '' + echo 'Suggestions can only be added near to lines changed in this PR.' + echo 'If any fixes can be added as code suggestions, they will be added shortly from another workflow.' + } >> "${GITHUB_STEP_SUMMARY}" + - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + id: upload-changes + if: >- + ${{ always() && + (steps.inputs.outputs.upload-changes == 'true') }} + with: + name: formatted-${{ steps.tool-name-safe.outputs.tool-name }} + if-no-files-found: ignore + retention-days: 10 + path: | + ${{ steps.inputs.outputs.extra-upload-changes }} + ${{ steps.git-changed-files.outputs.CHANGED_FILES }} + - name: Explain that more files need to be fixed + if: ${{ steps.files_changed.outputs.files_changed == 'true' }} + shell: bash + run: | + { + echo '' + echo 'All fixed files are included in the '"${FORMATTED_URL}"' artifact. \ + This artifact can be downloaded and copied to the repository to replace \ + unformatted files with the formatted files.' + } >> "${GITHUB_STEP_SUMMARY}" + env: + FORMATTED_URL: >- + [`formatted-${{ steps.tool-name-safe.outputs.tool-name }}`](${{ + steps.upload-changes.outputs.artifact-url }}) + - name: Fail action if some files were changed + if: >- + ${{ (steps.files_changed.outputs.files_changed == 'true') && + (steps.inputs.outputs.fail-if-changed == 'true') }} + shell: bash + run: | + if [[ -n "$INPUT_FAIL_MESSAGE" ]]; then + echo "::error::$INPUT_FAIL_MESSAGE" + fi + exit 1 + env: + INPUT_FAIL_MESSAGE: ${{ steps.inputs.outputs.fail-message }} From 7ba8c1d07876d0c6244f6b52c9456ca0a05dab20 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 20 Nov 2024 15:03:53 +0100 Subject: [PATCH 12/17] deactivate black in actions --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index a67ea93..91dbaea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,11 @@ required-version = '24' line-length = 88 target-version = ['py39', 'py310', 'py311', 'py312', 'py313'] # 'extend-exclude' excludes files or directories in addition to the defaults +extend-exclude = ''' +( + * +) +''' [tool.ruff] From 180e6b4fb38ddd1ddfa71a59aafce6e330131db9 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 20 Nov 2024 15:07:00 +0100 Subject: [PATCH 13/17] deactivate black in actions --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 91dbaea..08ae9c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,9 +38,12 @@ required-version = '24' line-length = 88 target-version = ['py39', 'py310', 'py311', 'py312', 'py313'] # 'extend-exclude' excludes files or directories in addition to the defaults +# Temporarily deactivate black checks extend-exclude = ''' ( - * + notebooks/ + src/ + tests/ ) ''' From 8023193177c8ffc0e6c09c27376c412bbe71645d Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 20 Nov 2024 15:08:41 +0100 Subject: [PATCH 14/17] deactivate black in actions --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 08ae9c4..f49f046 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,9 +41,9 @@ target-version = ['py39', 'py310', 'py311', 'py312', 'py313'] # Temporarily deactivate black checks extend-exclude = ''' ( - notebooks/ - src/ - tests/ + notebooks/, + src/, + tests/, ) ''' From 15151ae6aec503aaaaa5645596a62148ada27bc2 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 20 Nov 2024 15:16:36 +0100 Subject: [PATCH 15/17] deactivate black in actions --- pyproject.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f49f046..e8e3335 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,9 +41,7 @@ target-version = ['py39', 'py310', 'py311', 'py312', 'py313'] # Temporarily deactivate black checks extend-exclude = ''' ( - notebooks/, - src/, - tests/, + ^/.*.py ) ''' From b1415b72b0da26edf5f41cd651f146ff94ef9c09 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 20 Nov 2024 15:19:10 +0100 Subject: [PATCH 16/17] remove remnants from GRASS --- .github/workflows/python-code-quality.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/python-code-quality.yml b/.github/workflows/python-code-quality.yml index 23d9ec4..dfab758 100644 --- a/.github/workflows/python-code-quality.yml +++ b/.github/workflows/python-code-quality.yml @@ -110,10 +110,6 @@ jobs: run: | flake8 --count --statistics --show-source --jobs=$(nproc) . - - name: Run Flake8 on additional files - run: | - flake8 --count --statistics --show-source --jobs=$(nproc) python/grass/{script,jupyter}/testsuite/ - - name: Bandit Vulnerability Scan run: | bandit -c pyproject.toml -iii -r . -f sarif -o bandit.sarif --exit-zero From de0e435b1b13b4383bd6fe70c1e86b63b87b30a0 Mon Sep 17 00:00:00 2001 From: Stefan Blumentrath Date: Thu, 21 Nov 2024 09:35:11 +0100 Subject: [PATCH 17/17] Update tests.yml use docker v2 cmd --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 399ac57..53bee76 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,10 +14,10 @@ jobs: steps: - uses: actions/checkout@v4 - name: Start containers - run: docker-compose -f "docker/docker-compose-test.yml" up -d --build + run: docker compose -f "docker/docker-compose-test.yml" up -d --build - name: List running docker run: docker ps - name: Running tests run: make test - name: Stop containers - run: docker-compose -f "docker/docker-compose-test.yml" down + run: docker compose -f "docker/docker-compose-test.yml" down