From f5b153099049c4f9a925f59f58fc152bb60e6aae Mon Sep 17 00:00:00 2001 From: aidenvaines-bjss Date: Wed, 23 Jul 2025 16:20:59 +0100 Subject: [PATCH 1/4] CCM-11345 Manual Repo Sync --- .github/actions/check-todo-usage/action.yaml | 10 + .../create-lines-of-code-report/action.yaml | 2 +- .github/actions/scan-dependencies/action.yaml | 2 +- .github/workflows/cicd-3-deploy.yaml | 1 - .../scheduled-repository-template-sync.yaml | 2 +- .github/workflows/scorecard.yml | 14 +- .github/workflows/stage-1-commit.yaml | 25 +- .github/workflows/stage-3-build.yaml | 4 +- .gitignore | 12 +- .tool-versions | 14 +- infrastructure/terraform/.gitignore | 1 - infrastructure/terraform/bin/terraform.sh | 42 ++-- .../config/.repository-template-sync-ignore | 21 +- .../config/.repository-template-sync-merge | 9 + scripts/config/gitleaks.toml | 1 + scripts/config/pre-commit.yaml | 8 +- scripts/config/terraform-docs.yml | 2 + .../config/vocabularies/words/accept.txt | 2 +- scripts/githooks/check-todos.sh | 238 ++++++++++++++++++ scripts/githooks/sort-dictionary.sh | 3 +- scripts/githooks/sync-template-repo.sh | 76 ++++-- scripts/init.mk | 4 +- scripts/maintenance/merge.js | 81 ++++++ scripts/terraform/terraform-docs.sh | 2 +- 24 files changed, 498 insertions(+), 78 deletions(-) create mode 100644 .github/actions/check-todo-usage/action.yaml create mode 100644 scripts/config/.repository-template-sync-merge create mode 100755 scripts/githooks/check-todos.sh create mode 100644 scripts/maintenance/merge.js diff --git a/.github/actions/check-todo-usage/action.yaml b/.github/actions/check-todo-usage/action.yaml new file mode 100644 index 00000000..a403d588 --- /dev/null +++ b/.github/actions/check-todo-usage/action.yaml @@ -0,0 +1,10 @@ +name: "Check Todo usage" +description: "Check Todo usage" +runs: + using: "composite" + steps: + - name: "Check Todo usage" + shell: bash + run: | + export BRANCH_NAME=origin/${{ github.event.repository.default_branch }} + check=branch ./scripts/githooks/check-todos.sh diff --git a/.github/actions/create-lines-of-code-report/action.yaml b/.github/actions/create-lines-of-code-report/action.yaml index b21f0667..86396f7a 100644 --- a/.github/actions/create-lines-of-code-report/action.yaml +++ b/.github/actions/create-lines-of-code-report/action.yaml @@ -44,7 +44,7 @@ runs: echo "secrets_exist=${{ inputs.idp_aws_report_upload_role_name != '' && inputs.idp_aws_report_upload_bucket_endpoint != '' }}" >> $GITHUB_OUTPUT - name: "Authenticate to send the report" if: steps.check.outputs.secrets_exist == 'true' - uses: aws-actions/configure-aws-credentials@v2 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: arn:aws:iam::${{ inputs.idp_aws_report_upload_account_id }}:role/${{ inputs.idp_aws_report_upload_role_name }} aws-region: ${{ inputs.idp_aws_report_upload_region }} diff --git a/.github/actions/scan-dependencies/action.yaml b/.github/actions/scan-dependencies/action.yaml index f8ed605d..1000df14 100644 --- a/.github/actions/scan-dependencies/action.yaml +++ b/.github/actions/scan-dependencies/action.yaml @@ -58,7 +58,7 @@ runs: run: echo "secrets_exist=${{ inputs.idp_aws_report_upload_role_name != '' && inputs.idp_aws_report_upload_bucket_endpoint != '' }}" >> $GITHUB_OUTPUT - name: "Authenticate to send the reports" if: steps.check.outputs.secrets_exist == 'true' - uses: aws-actions/configure-aws-credentials@v2 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: arn:aws:iam::${{ inputs.idp_aws_report_upload_account_id }}:role/${{ inputs.idp_aws_report_upload_role_name }} aws-region: ${{ inputs.idp_aws_report_upload_region }} diff --git a/.github/workflows/cicd-3-deploy.yaml b/.github/workflows/cicd-3-deploy.yaml index ff2ad644..5230577e 100644 --- a/.github/workflows/cicd-3-deploy.yaml +++ b/.github/workflows/cicd-3-deploy.yaml @@ -48,7 +48,6 @@ jobs: echo "nodejs_version=$(grep "^nodejs\s" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT echo "python_version=$(grep "^python\s" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT echo "terraform_version=$(grep "^terraform\s" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT - # TODO: Get the version, but it may not be the .version file as this should come from the CI/CD Pull Request Workflow echo "version=$(head -n 1 .version 2> /dev/null || echo unknown)" >> $GITHUB_OUTPUT # echo "tag=${{ github.event.inputs.tag }}" >> $GITHUB_OUTPUT - name: "List variables" diff --git a/.github/workflows/scheduled-repository-template-sync.yaml b/.github/workflows/scheduled-repository-template-sync.yaml index 9df295a9..e9114865 100644 --- a/.github/workflows/scheduled-repository-template-sync.yaml +++ b/.github/workflows/scheduled-repository-template-sync.yaml @@ -27,7 +27,7 @@ jobs: - name: Run syncronisation script run: | - ./scripts/githooks/sync-template-repo.sh + ./nhs-notify-repository-template/scripts/githooks/sync-template-repo.sh rm -Rf ./nhs-notify-repository-template - name: Create Pull Request diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 9c6fdd97..03bf1714 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -27,17 +27,17 @@ jobs: # Needed to publish results and get a badge (see publish_results below). id-token: write # Uncomment the permissions below if installing in a private repository. - # contents: read - # actions: read + contents: read + actions: read steps: - name: "Checkout code" - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 with: results_file: results.sarif results_format: sarif @@ -45,7 +45,7 @@ jobs: # - you want to enable the Branch-Protection check on a *public* repository, or # - you are installing Scorecard on a *private* repository # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional. - # repo_token: ${{ secrets.SCORECARD_TOKEN }} + repo_token: ${{ secrets.SCORECARD_TOKEN }} # Public repositories: # - Publish results to OpenSSF REST API for easy access by consumers @@ -59,7 +59,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@97a0fba1372883ab732affbe8f94b823f91727db # v3.pre.node20 + uses: actions/upload-artifact@v4 with: name: SARIF file path: results.sarif @@ -68,6 +68,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@60168efe1c415ce0f5521ea06d5c2062adbeed1b # v3.28.17 + uses: github/codeql-action/upload-sarif@9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0 # v3.28.9 with: sarif_file: results.sarif diff --git a/.github/workflows/stage-1-commit.yaml b/.github/workflows/stage-1-commit.yaml index e168ba96..e56f2f3e 100644 --- a/.github/workflows/stage-1-commit.yaml +++ b/.github/workflows/stage-1-commit.yaml @@ -36,7 +36,7 @@ jobs: scan-secrets: name: "Scan secrets" runs-on: ubuntu-latest - timeout-minutes: 2 + timeout-minutes: 5 steps: - name: "Checkout code" uses: actions/checkout@v4 @@ -47,7 +47,7 @@ jobs: check-file-format: name: "Check file format" runs-on: ubuntu-latest - timeout-minutes: 2 + timeout-minutes: 5 steps: - name: "Checkout code" uses: actions/checkout@v4 @@ -58,7 +58,7 @@ jobs: check-markdown-format: name: "Check Markdown format" runs-on: ubuntu-latest - timeout-minutes: 2 + timeout-minutes: 5 steps: - name: "Checkout code" uses: actions/checkout@v4 @@ -93,7 +93,7 @@ jobs: check-english-usage: name: "Check English usage" runs-on: ubuntu-latest - timeout-minutes: 2 + timeout-minutes: 5 steps: - name: "Checkout code" uses: actions/checkout@v4 @@ -101,6 +101,17 @@ jobs: fetch-depth: 0 # Full history is needed to compare branches - name: "Check English usage" uses: ./.github/actions/check-english-usage + check-todo-usage: + name: "Check TODO usage" + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: "Checkout code" + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Full history is needed to compare branches + - name: "Check TODO usage" + uses: ./.github/actions/check-todo-usage detect-terraform-changes: name: "Detect Terraform Changes" runs-on: ubuntu-latest @@ -127,7 +138,7 @@ jobs: lint-terraform: name: "Lint Terraform" runs-on: ubuntu-latest - timeout-minutes: 2 + timeout-minutes: 5 needs: detect-terraform-changes if: needs.detect-terraform-changes.outputs.terraform_changed == 'true' steps: @@ -156,7 +167,7 @@ jobs: permissions: id-token: write contents: read - timeout-minutes: 2 + timeout-minutes: 5 steps: - name: "Checkout code" uses: actions/checkout@v4 @@ -175,7 +186,7 @@ jobs: permissions: id-token: write contents: read - timeout-minutes: 2 + timeout-minutes: 5 steps: - name: "Checkout code" uses: actions/checkout@v4 diff --git a/.github/workflows/stage-3-build.yaml b/.github/workflows/stage-3-build.yaml index 201d5b19..fabfe110 100644 --- a/.github/workflows/stage-3-build.yaml +++ b/.github/workflows/stage-3-build.yaml @@ -60,7 +60,7 @@ jobs: - name: "Upload artefact 1" run: | echo "Uploading artefact 1 ..." - # TODO: Use either action/cache or action/upload-artifact + # Use either action/cache or action/upload-artifact artefact-n: name: "Artefact n" runs-on: ubuntu-latest @@ -77,4 +77,4 @@ jobs: - name: "Upload artefact n" run: | echo "Uploading artefact n ..." - # TODO: Use either action/cache or action/upload-artifact + # Use either action/cache or action/upload-artifact diff --git a/.gitignore b/.gitignore index adb1a949..e97bd341 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,16 @@ version.json *.code-workspace !project.code-workspace -node_modules # Please, add your custom content below! + +# dependencies +node_modules +.node-version +*/node_modules +/.pnp +.pnp.js +/build +dist +.DS_Store +.reports diff --git a/.tool-versions b/.tool-versions index 6b801709..34d9b8a5 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,19 +1,21 @@ act 0.2.64 -gitleaks 8.18.4 +gitleaks 8.24.0 +jq 1.6 +nodejs 22.11.0 pre-commit 3.6.0 terraform 1.9.2 terraform-docs 0.19.0 trivy 0.61.0 vale 3.6.0 -nodejs 22.15.1 +# python 3.13.2 # ============================================================================== # The section below is reserved for Docker image versions. # TODO: Move this section - consider using a different file for the repository template dependencies. -# docker/ghcr.io/anchore/grype v0.69.1@sha256:d41fcb371d0af59f311e72123dff46900ebd6d0482391b5a830853ee4f9d1a76 # SEE: https://github.com/anchore/grype/pkgs/container/grype -# docker/ghcr.io/anchore/syft v0.92.0@sha256:63c60f0a21efb13e80aa1359ab243e49213b6cc2d7e0f8179da38e6913b997e0 # SEE: https://github.com/anchore/syft/pkgs/container/syft -# docker/ghcr.io/gitleaks/gitleaks v8.18.0@sha256:fd2b5cab12b563d2cc538b14631764a1c25577780e3b7dba71657d58da45d9d9 # SEE: https://github.com/gitleaks/gitleaks/pkgs/container/gitleaks +# docker/ghcr.io/anchore/grype v0.92.2@sha256:651e558f9ba84f2a790b3449c8a57cbbf4f34e004f7d3f14ae8f8cbeede4cd33 # SEE: https://github.com/anchore/grype/pkgs/container/grype +# docker/ghcr.io/anchore/syft v1.26.0@sha256:de078f51704a213906970b1475edd6006b8af50aa159852e125518237487b8c6 # SEE: https://github.com/anchore/syft/pkgs/container/syft +# docker/ghcr.io/gitleaks/gitleaks:v8.24.0@sha256:b8e9bf46893c2f20e10bfb4b2e783adaef519dea981b01ca6221ac325e836040 # SEE: https://github.com/gitleaks/gitleaks/pkgs/container/gitleaks # docker/ghcr.io/igorshubovych/markdownlint-cli v0.37.0@sha256:fb3e79946fce78e1cde84d6798c6c2a55f2de11fc16606a40d49411e281d950d # SEE: https://github.com/igorshubovych/markdownlint-cli/pkgs/container/markdownlint-cli # docker/ghcr.io/make-ops-tools/gocloc latest@sha256:6888e62e9ae693c4ebcfed9f1d86c70fd083868acb8815fe44b561b9a73b5032 # SEE: https://github.com/make-ops-tools/gocloc/pkgs/container/gocloc # docker/ghcr.io/nhs-england-tools/github-runner-image 20230909-321fd1e-rt@sha256:ce4fd6035dc450a50d3cbafb4986d60e77cb49a71ab60a053bb1b9518139a646 # SEE: https://github.com/nhs-england-tools/github-runner-image/pkgs/container/github-runner-image @@ -22,4 +24,4 @@ nodejs 22.15.1 # docker/jdkato/vale v3.6.0@sha256:0ef22c8d537f079633cfff69fc46f69a2196072f69cab1ab232e8a79a388e425 # SEE: https://hub.docker.com/r/jdkato/vale/tags # docker/koalaman/shellcheck latest@sha256:e40388688bae0fcffdddb7e4dea49b900c18933b452add0930654b2dea3e7d5c # SEE: https://hub.docker.com/r/koalaman/shellcheck/tags # docker/mstruebing/editorconfig-checker 2.7.1@sha256:dd3ca9ea50ef4518efe9be018d669ef9cf937f6bb5cfe2ef84ff2a620b5ddc24 # SEE: https://hub.docker.com/r/mstruebing/editorconfig-checker/tags -# docker/sonarsource/sonar-scanner-cli 5.0.1@sha256:494ecc3b5b1ee1625bd377b3905c4284e4f0cc155cff397805a244dee1c7d575 # SEE: https://hub.docker.com/r/sonarsource/sonar-scanner-cli/tags +# docker/sonarsource/sonar-scanner-cli 11.3@sha256:7462f132388135e32b948f8f18ff0db9ae28a87c6777f1df5b2207e04a6d7c5c # SEE: https://hub.docker.com/r/sonarsource/sonar-scanner-cli/tags diff --git a/infrastructure/terraform/.gitignore b/infrastructure/terraform/.gitignore index 54175d30..579b6414 100644 --- a/infrastructure/terraform/.gitignore +++ b/infrastructure/terraform/.gitignore @@ -23,7 +23,6 @@ components/**/backend_tfscaffold.tf # Scaffold Plugin Cache plugin-cache/* -bootstrap # PyCache **/__pycache__ diff --git a/infrastructure/terraform/bin/terraform.sh b/infrastructure/terraform/bin/terraform.sh index c932a77e..72e6c571 100755 --- a/infrastructure/terraform/bin/terraform.sh +++ b/infrastructure/terraform/bin/terraform.sh @@ -539,24 +539,26 @@ fi; [ -f "${dynamic_file_path}" ] && tf_var_file_paths+=("${dynamic_file_path}"); # Warn on duplication -duplicate_variables="$(cat "${tf_var_file_paths[@]}" | sed -n -e 's/\(^[a-zA-Z0-9_\-]\+\)\s*=.*$/\1/p' | sort | uniq -d)"; -[ -n "${duplicate_variables}" ] \ - && echo -e " -################################################################### -# WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING # -################################################################### -The following input variables appear to be duplicated: - -${duplicate_variables} - -This could lead to unexpected behaviour. Overriding of variables -has previously been unpredictable and is not currently supported, -but it may work. - -Recent changes to terraform might give you useful overriding and -map-merging functionality, please use with caution and report back -on your successes & failures. -###################################################################"; +if [ ${#tf_var_file_paths[@]} -gt 0 ]; then + duplicate_variables="$(cat "${tf_var_file_paths[@]}" | sed -n -e 's/\(^[a-zA-Z0-9_\-]\+\)\s*=.*$/\1/p' | sort | uniq -d)"; + [ -n "${duplicate_variables}" ] \ + && echo -e " + ################################################################### + # WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING # + ################################################################### + The following input variables appear to be duplicated: + + ${duplicate_variables} + + This could lead to unexpected behaviour. Overriding of variables + has previously been unpredictable and is not currently supported, + but it may work. + + Recent changes to terraform might give you useful overriding and + map-merging functionality, please use with caution and report back + on your successes & failures. + ###################################################################"; +fi # Build up the tfvars arguments for terraform command line for file_path in "${tf_var_file_paths[@]}"; do @@ -791,8 +793,8 @@ case "${action}" in ;; *) echo -e "Generic action case invoked. Only the additional arguments will be passed to terraform, you break it you fix it:"; - echo -e "\tterraform ${action} ${extra_args}"; - terraform "${action}" ${extra_args} \ + echo -e "\tterraform ${action} ${extra_args} | tee terraform_output"; + terraform "${action}" ${extra_args} | tee terraform_output \ || error_and_die "Terraform ${action} failed."; ;; esac; diff --git a/scripts/config/.repository-template-sync-ignore b/scripts/config/.repository-template-sync-ignore index 7564f3e8..7b6cbfc1 100644 --- a/scripts/config/.repository-template-sync-ignore +++ b/scripts/config/.repository-template-sync-ignore @@ -1,18 +1,33 @@ # Files and folders to ignore when syncing nhs-notify-repository-template back in to this repository -scripts/config/.repository-template-sync-ignore -.github/workflows/ nhs-notify-repository-template/ # Files and Folders in this repository to ignore +.editorconfig +.github/CODEOWNERS +.github/ISSUE_TEMPLATE +.github/workflows/cicd-*.yaml +.github/workflows/stage-*.yaml +.gitleaksignore .vscode/ +Makefile CHANGELOG.md project.code-workspace README.md +scripts/config/sonar-scanner.properties +scripts/tests/ VERSION # Files and Folders in the template repository to disregard .devcontainer/ -.github/workflows/cicd-*.yaml +.github/actions/build-docs +.github/workflows/*.disabled */examples/ docs/ +eslint.config.mjs infrastructure/terraform/components/ +lambdas/example-lambda/ +package-lock.json +package.json +scripts/**/examples/ +scripts/terraform/terraform.mk +src/.vscode/ diff --git a/scripts/config/.repository-template-sync-merge b/scripts/config/.repository-template-sync-merge new file mode 100644 index 00000000..a42f50a9 --- /dev/null +++ b/scripts/config/.repository-template-sync-merge @@ -0,0 +1,9 @@ +# Files and folders to merge when syncing nhs-notify-repository-template back in to this repository +.github/workflows/cicd-*.yaml +.gitignore +.tool-versions +scripts/config/.repository-template-sync-ignore +scripts/config/.repository-template-sync-merge +scripts/config/vale/styles/config/vocabularies/words/accept.txt +scripts/config/vale/styles/config/vocabularies/words/reject.txt +scripts/config/vale/vale.ini diff --git a/scripts/config/gitleaks.toml b/scripts/config/gitleaks.toml index 188bfdf0..7b8360ae 100644 --- a/scripts/config/gitleaks.toml +++ b/scripts/config/gitleaks.toml @@ -1,4 +1,5 @@ # SEE: https://github.com/gitleaks/gitleaks/#configuration +# Do not edit this file directly as it will be overwritten by changes from the nhs-notify-repository-template on next sync [extend] useDefault = true # SEE: https://github.com/gitleaks/gitleaks/blob/master/config/gitleaks.toml diff --git a/scripts/config/pre-commit.yaml b/scripts/config/pre-commit.yaml index de68260b..8843370f 100644 --- a/scripts/config/pre-commit.yaml +++ b/scripts/config/pre-commit.yaml @@ -9,7 +9,6 @@ repos: - id: check-symlinks - id: detect-private-key - id: end-of-file-fixer - exclude: .+\.cs - id: forbid-new-submodules - id: mixed-line-ending - id: pretty-format-json @@ -64,3 +63,10 @@ repos: entry: ./scripts/githooks/check-terraform-docs.sh language: script pass_filenames: false + - repo: local + hooks: + - id: check-todo-usage + name: Check TODO usage + entry: /usr/bin/env check=branch ./scripts/githooks/check-todos.sh + language: script + pass_filenames: false diff --git a/scripts/config/terraform-docs.yml b/scripts/config/terraform-docs.yml index b38d1289..e49d91f8 100644 --- a/scripts/config/terraform-docs.yml +++ b/scripts/config/terraform-docs.yml @@ -16,6 +16,7 @@ content: |- {{ .Modules }} {{ .Outputs }} {{ .Footer }} + output: file: 'README.md' mode: inject @@ -27,6 +28,7 @@ output: + output-values: enabled: false from: '' diff --git a/scripts/config/vale/styles/config/vocabularies/words/accept.txt b/scripts/config/vale/styles/config/vocabularies/words/accept.txt index 3b07d50c..3aa82c3a 100644 --- a/scripts/config/vale/styles/config/vocabularies/words/accept.txt +++ b/scripts/config/vale/styles/config/vocabularies/words/accept.txt @@ -9,7 +9,7 @@ drawio endcapture endfor endraw -GitHub +Git[Hh]ub Gitleaks Grype idempotence diff --git a/scripts/githooks/check-todos.sh b/scripts/githooks/check-todos.sh new file mode 100755 index 00000000..83b7a80e --- /dev/null +++ b/scripts/githooks/check-todos.sh @@ -0,0 +1,238 @@ +#!/bin/bash + +# WARNING: Please, DO NOT edit this file! It is maintained in the Repository Template (https://github.com/nhs-england-tools/repository-template). Raise a PR instead. + +set -euo pipefail + +# Pre-commit git hook to scan for secrets hard-coded in the codebase. This is a +# gitleaks command wrapper. It will run gitleaks natively if it is installed, +# otherwise it will run it in a Docker container. +# +# Usage: +# $ [options] ./scan-secrets.sh +# +# Options: +# check=all # check all files in the repository +# check=staged-changes # check only files staged for commit. +# check=working-tree-changes # check modified, unstaged files. This is the default. +# check=branch # check for all changes since branching from $BRANCH_NAME +# VERBOSE=true # Show all the executed commands, default is 'false' +# +# Exit codes: +# 0 - No Todos +# 1 - Todos found or error encountered +# 126 - Unknown flag + +# ============================================================================== + +EXCLUDED_FILES=( + ".devcontainer/devcontainer.json" + ".tool-versions" + ".vscode/extensions.json" + "infrastructure/terraform/bin/terraform.sh" + "Makefile" + "project.code-workspace" + "src/jekyll-devcontainer/src/.devcontainer/devcontainer.json" +) + +EXCLUDED_DIRS=( + ".git/" + ".venv/" + "docs/" + "node_modules/" +) + + +# Get files to check based on mode +function get_files_to_check() { + local mode="$1" + case "$mode" in + staged-changes) + git diff --diff-filter=ACMRT --name-only --cached # ACMRT only show files added, copied, modified, renamed or that had their type changed (eg. file → symlink) in this commit. This leaves out deleted files. + ;; + working-tree-changes) + git ls-files --others --exclude-standard && git diff --diff-filter=ACMRT --name-only + ;; + branch) + git diff --diff-filter=ACMRT --name-only ${BRANCH_NAME:-origin/main} + ;; + all) + git ls-files && git ls-files --others --exclude-standard + ;; + *) + echo "Unknown check mode: $mode" >&2 + exit 126 + ;; + esac +} + + +function build_exclude_args() { + local args=( + --exclude=".github/actions/check-todo-usage/action.yaml" + --exclude=".github/workflows/stage-1-commit.yaml" + --exclude="scripts/config/pre-commit.yaml" + --exclude="scripts/githooks/check-todos.sh" + ) # Exclude this script and its references by default, as it naturally contains TODOs. Todo todo todo <- see? + + if [ ${#EXCLUDED_DIRS[@]} -gt 0 ]; then + for dir in "${EXCLUDED_DIRS[@]}"; do + args+=(--exclude-dir="$dir") + done + fi + + if [ ${#EXCLUDED_FILES[@]} -gt 0 ]; then + for file in "${EXCLUDED_FILES[@]}"; do + args+=(--exclude="$file") + done + fi + echo "${args[@]}" +} + + +function search_todos() { + local mode="$1" + shift # Shift positional parameters so $@ contains only exclude_args + local exclude_args=("$@") + local todos="" + + local files + files=$(get_files_to_check "$mode") + # flatten files to unique list + files=$(echo "$files" | tr ' ' '\n' | sort -u) + + for file in $files; do + skip=false + + # Check if the file matches any exclude patterns + # Exclude files based on provided arguments and predefined directories + for ex in "${exclude_args[@]}"; do + if [[ "$ex" == --exclude* ]]; then + pattern=${ex#--exclude=} + [[ "$file" == $pattern ]] && skip=true && break + fi + done + + # Check if the file is in any of the excluded directories + for exdir in "${EXCLUDED_DIRS[@]}"; do + [[ "$file" == $exdir* ]] && skip=true && break + done + + # If the file is excluded, skip it + if [ "$skip" = false ] && [ -f "$file" ]; then + file_todos=$(grep -nHiE '\bTODO\b' "$file" || true) + [ -n "$file_todos" ] && todos+="$file_todos\n" + fi + done + + echo -e "$todos" +} + + +function filter_todos_with_valid_jira_ticket() { + local todos="$1" + local jira_regex="[A-Z][A-Z0-9]+-[0-9]+" + local todos_without_ticket="" + + while IFS= read -r line; do + # Only lines with TODO but without a valid JIRA ticket + if grep -qnHiE '\bTODO\b' <<< "$line"; then + if ! [[ "$line" =~ $jira_regex ]]; then + todos_without_ticket+="$line\n" + fi + fi + done <<< "$(echo -e "$todos")" + + # Output only TODOs without a valid JIRA ticket + echo -e "$todos_without_ticket" +} + + +function print_output() { + local todos="$1" + local exclude_args="$2" + local todo_count=$(line_count "$todos") + + echo "TODO Check Configuration:" + echo "=========================================" + echo " Check Mode: ${check:-working-tree-changes}" + echo " Total TODOs found: $todo_count" + + if [ ${#EXCLUDED_DIRS[@]} -gt 0 ]; then + echo " Excluded Directories: ${EXCLUDED_DIRS[*]}" + else + echo " Excluded Directories: (none)" + fi + + if [ ${#EXCLUDED_FILES[@]} -gt 0 ]; then + echo " Excluded Files: ${EXCLUDED_FILES[*]}" + else + echo " Excluded Files: (none)" + fi + + if is-arg-true "${VERBOSE:-false}"; then + echo "Grep Exclude Args: $exclude_args" + fi + + echo -e "\n=========================================" + echo "All TODOs found: $todo_count" + echo "=========================================" + + if [ "$todo_count" -gt 0 ]; then + echo "$todos" + else + echo "No TODOs found." + fi + + local results=$(filter_todos_with_valid_jira_ticket "$todos") + local results_count=$(line_count "$results") + + echo -e "\n=========================================" + echo "TODOs without a Jira ticket: $results_count" + echo "=========================================" + + if [ "$results_count" -gt 0 ]; then + echo "$results" + exit 1 + else + echo "No TODOs found without a Jira reference." + fi +} + + +function main() { + cd "$(git rev-parse --show-toplevel)" + + local check_mode="${check:-working-tree-changes}" + local exclude_args=$(build_exclude_args) + local todos=$(search_todos "$check_mode" $exclude_args) + print_output "$todos" "$exclude_args" +} + +# ============================================================================== + +# Count non-empty lines in a string +function line_count() { + local input="$1" + if [ -n "$input" ]; then + echo -e "$input" | wc -l + else + echo 0 + fi +} + +function is-arg-true() { + if [[ "$1" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$ ]]; then + return 0 + else + return 1 + fi +} + +# ============================================================================== + +is-arg-true "${VERBOSE:-false}" && set -x + +main "$@" + +exit 0 diff --git a/scripts/githooks/sort-dictionary.sh b/scripts/githooks/sort-dictionary.sh index 45a39529..2fea1399 100755 --- a/scripts/githooks/sort-dictionary.sh +++ b/scripts/githooks/sort-dictionary.sh @@ -25,7 +25,8 @@ function main() { mv $root/accept.sorted.txt $root/accept.txt mv $root/reject.sorted.txt $root/reject.txt - git add -uv $root/* + # Update the sorted files in the staged git index + git add --update --verbose $root/* } # ============================================================================== diff --git a/scripts/githooks/sync-template-repo.sh b/scripts/githooks/sync-template-repo.sh index 060aaa4c..6e731db7 100755 --- a/scripts/githooks/sync-template-repo.sh +++ b/scripts/githooks/sync-template-repo.sh @@ -5,7 +5,7 @@ set -euo pipefail # Script to synchronise the nhs-notify-template-repository with this repository # # Usage: -# $ [options] ./check-terraform-format.sh +# $ [options] ./sync-template-repo.sh # # Options: # new_only=true # Only identify new files from the template-repository @@ -13,13 +13,16 @@ set -euo pipefail # ============================================================================== -# Command line prameters +scriptdir=$(realpath "$(dirname "$0")") + +# Command line parameters new_only=${new_only:-false} changes_only=${changes_only:-false} # Set variables TEMPLATE_REPO_DIR="nhs-notify-repository-template" IGNORE_FILE="scripts/config/.repository-template-sync-ignore" +MERGE_FILE="scripts/config/.repository-template-sync-merge" # Check if the template directory exists if [ ! -d "${TEMPLATE_REPO_DIR}" ]; then @@ -34,10 +37,18 @@ if [ ! -f "${IGNORE_FILE}" ]; then echo "# Files and Folders in the template repository to disregard" >> ${IGNORE_FILE} fi -# Read the .template-ignore file into an array -while IFS= read -r line || [ -n "$line" ]; do - IGNORED_PATHS+=("$line") -done < "$IGNORE_FILE" +# Check if the .template-merge file exists, create an empty one if not +if [ ! -f "${MERGE_FILE}" ]; then + echo "# Files and folders to merge when syncing ${TEMPLATE_REPO_DIR} back in to this repository" > ${MERGE_FILE} +fi + +TMP_SYNC_IGNORE=${PWD}/tmp-sync-ignore +mkdir -p "${TMP_SYNC_IGNORE}" +cp "${IGNORE_FILE}" "${TMP_SYNC_IGNORE}/.gitignore" + +TMP_SYNC_MERGE=${PWD}/tmp-sync-merge +mkdir -p "${TMP_SYNC_MERGE}" +cp "${MERGE_FILE}" "${TMP_SYNC_MERGE}/.gitignore" # Check if a file is ignored. is_ignored() { @@ -48,21 +59,30 @@ is_ignored() { return 0 fi - for ignored in "${IGNORED_PATHS[@]}"; do - if [[ -n "$ignored" && "$file" =~ $ignored ]]; then - return 0 - fi - done - return 1 + pushd "${TMP_SYNC_IGNORE}" > /dev/null + git check-ignore -q "${file}" + R=$? + popd > /dev/null + return $R +} + +is_merge() { + local file=${1} + + pushd "${TMP_SYNC_MERGE}" > /dev/null + git check-ignore -q "${file}" + R=$? + popd > /dev/null + return $R } # Navigate to the template directory -cd "${TEMPLATE_REPO_DIR}" || exit +pushd "${TEMPLATE_REPO_DIR}" || exit FILES_ADDED=() FILES_WITH_CHANGES=() # Loop through all files in the template directory -while IFS= read -r -d '' file; do +while IFS= read -r -d '' file || [[ -n $file ]]; do relative_path="${file#./}" # Remove leading './' # Check if the file is ignored @@ -84,27 +104,41 @@ while IFS= read -r -d '' file; do # If the file exists, check if it's different if [ "$new_only" == false ]; then if ! diff -q "$file" "$target_path" > /dev/null 2>&1; then - echo "Merging changes from $relative_path" - FILES_WITH_CHANGES+=("${relative_path}") - cp "$file" "$target_path" + if is_merge "$relative_path"; then + echo "Merging changes from $relative_path" + cp "$target_path" "${target_path}.bak" + node "${scriptdir}/../maintenance/merge.js" "$target_path" "$file" > "${target_path}.merged" + if ! cmp -s "${target_path}.merged" "${target_path}.bak"; then + FILES_WITH_CHANGES+=("${relative_path}") + mv "${target_path}.merged" "$target_path" + fi + rm -f "${target_path}.merged" "${target_path}.bak" + else + echo "Copying changes from $relative_path" + cp "$file" "$target_path" + FILES_WITH_CHANGES+=("${relative_path}") + fi fi fi fi done < <(find . -type f -print0) -echo "${#FILES_ADDED[@]}" files added, "${#FILES_WITH_CHANGES[@]}" files with changes detected. +popd +rm -rf "${TMP_SYNC_IGNORE}" "${TMP_SYNC_MERGE}" echo ------------------------------------------ +echo "${#FILES_ADDED[@]} files added, ${#FILES_WITH_CHANGES[@]} files with changes detected." -if [ "$changes_only" == false ]; then +if [[ "$changes_only" == false && ${#FILES_ADDED[@]} -gt 0 ]]; then echo ------------------------------------------ echo "New files added:" printf ' - %s\n' "${FILES_ADDED[@]}" fi - -if [ "$new_only" == false ]; then +if [[ "$new_only" == false && ${#FILES_WITH_CHANGES[@]} -gt 0 ]]; then echo ------------------------------------------ echo "Changed files:" printf ' - %s\n' "${FILES_WITH_CHANGES[@]}" fi + +echo ------------------------------------------ diff --git a/scripts/init.mk b/scripts/init.mk index 373f8a4f..e12255c3 100644 --- a/scripts/init.mk +++ b/scripts/init.mk @@ -7,7 +7,7 @@ include scripts/tests/test.mk # ============================================================================== runner-act: # Run GitHub Actions locally - mandatory: workflow=[workflow file name], job=[job name] @Development - source ./scripts/docker/docker.lib.sh + . ./scripts/docker/docker.lib.sh act $(shell [[ "${VERBOSE}" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$$ ]] && echo --verbose) \ --container-architecture linux/amd64 \ --platform ubuntu-latest=$$(name="ghcr.io/nhs-england-tools/github-runner-image" docker-get-image-version-and-pull) \ @@ -21,7 +21,7 @@ runner-act: # Run GitHub Actions locally - mandatory: workflow=[workflow file na --job ${job} version-create-effective-file: # Create effective version file - optional: dir=[path to the VERSION file to use, default is '.'], BUILD_DATETIME=[build date and time in the '%Y-%m-%dT%H:%M:%S%z' format generated by the CI/CD pipeline, default is current date and time] @Development - source scripts/docker/docker.lib.sh + . ./scripts/docker/docker.lib.sh version-create-effective-file shellscript-lint-all: # Lint all shell scripts in this project, do not fail on error, just print the error messages @Quality diff --git a/scripts/maintenance/merge.js b/scripts/maintenance/merge.js new file mode 100644 index 00000000..8f5bd6cc --- /dev/null +++ b/scripts/maintenance/merge.js @@ -0,0 +1,81 @@ +/******************************************************************************* + * Script to merge all added lines from source to target. + * Modified lines will be left intact. + * + * This is intended for updating config files like .tool-versions and .gitignore + * where the first token on the line remains the same + * + * Usage: + * $ node merge.js + * + * Output: + * Outputs the merged file to stdout + * + *******************************************************************************/ + +const fs = require("fs"); + +// Read files +const [file1, file2] = process.argv.slice(2); +const lines1 = fs.readFileSync(file1).toString().split('\n'); +const lines2 = fs.readFileSync(file2).toString().split('\n'); + +// Tokenize lines in file1 for later comparison +const tokenize = line => { + if (line === '') { + return []; + } + return line.split(/\s+|[:=]/).filter(x => x !== '#' && x !== '').slice(0, 1); +}; +const lines1Tokens = lines1.flatMap(tokenize); + +// Step through the files +let pos1 = 0, pos2 = 0; +while (pos1 < lines1.length || pos2 < lines2.length) { + + const l1 = pos1 < lines1.length && lines1[pos1] || ''; + const l2 = pos2 < lines2.length && lines2[pos2] || ''; + + // If the lines match, print l1 and skip l2 + if (l1 !== '' && l1 === l2) { + process.stdout.write(`${l1}\n`); + pos1++; + pos2++; + continue; + } + + if (pos2 < lines2.length) { + // If l2 is empty, skip l2 + if (l2 === '') { + pos2++; + continue; + } + + const [l2token] = tokenize(l2); + + // If l2 token matches l1 token, print l1 + if (pos1 < lines1.length && lines1Tokens[pos1] === l2token) { + process.stdout.write(`${l1}\n`); + pos1++; + pos2++; + continue; + } + + // If l2 doesn't match any lines in file1, print l2 + if (l2token && !lines1Tokens.includes(l2token)) { + process.stdout.write(`${l2}\n`); + } + pos2++ + continue; + } + + // If we're not at the end of file1, print l1 + if (pos1 < lines1.length) { + if (pos1 === lines1.length - 1 && l1 === '') { + // Don't print tailing newline + } else { + process.stdout.write(`${l1}\n`); + } + pos1++; + } +} diff --git a/scripts/terraform/terraform-docs.sh b/scripts/terraform/terraform-docs.sh index 7711c48e..446d30d0 100755 --- a/scripts/terraform/terraform-docs.sh +++ b/scripts/terraform/terraform-docs.sh @@ -44,7 +44,7 @@ function run-terraform-docs-natively() { function run-terraform-docs-in-docker() { - # shellcheck disable=SC1091 + # shellcheck disable=SC1091 source ./scripts/docker/docker.lib.sh local dir_to_scan="$1" From 00d574b7b5cb2e7a280a3cfb770116425d492482 Mon Sep 17 00:00:00 2001 From: aidenvaines-bjss Date: Wed, 23 Jul 2025 16:40:10 +0100 Subject: [PATCH 2/4] CCM-11345 Manual Repo Sync --- .github/SECURITY.md | 4 ++-- .github/workflows/stage-1-commit.yaml | 2 +- LICENCE.md | 2 +- scripts/config/pre-commit.yaml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/SECURITY.md b/.github/SECURITY.md index 241f1e33..0382fd5d 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -21,8 +21,8 @@ If you wish to notify us of a vulnerability via email, please include detailed i You can reach us at: -- _[ A product team email address ]_ -- [cybersecurity@nhs.net](cybersecurity@nhs.net) +- [england.nhsnotify@nhs.net](mailto:england.nhsnotify@nhs.net) +- [cybersecurity@nhs.net](mailto:cybersecurity@nhs.net) ### NCSC diff --git a/.github/workflows/stage-1-commit.yaml b/.github/workflows/stage-1-commit.yaml index e56f2f3e..ececcb36 100644 --- a/.github/workflows/stage-1-commit.yaml +++ b/.github/workflows/stage-1-commit.yaml @@ -156,7 +156,7 @@ jobs: - name: "Checkout code" uses: actions/checkout@v4 - name: "Setup ASDF" - uses: asdf-vm/actions/setup@v3 + uses: asdf-vm/actions/setup@v4 - name: "Perform Setup" uses: ./.github/actions/setup - name: "Trivy Scan" diff --git a/LICENCE.md b/LICENCE.md index ed56eb21..02174c4c 100644 --- a/LICENCE.md +++ b/LICENCE.md @@ -1,6 +1,6 @@ # MIT Licence -Copyright (c) 2024 Crown Copyright NHS England. +Copyright (c) 2025 Crown Copyright NHS England. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/scripts/config/pre-commit.yaml b/scripts/config/pre-commit.yaml index 8843370f..29397e23 100644 --- a/scripts/config/pre-commit.yaml +++ b/scripts/config/pre-commit.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 # Use the ref you want to point at + rev: v5.0.0 # Use the ref you want to point at hooks: - id: trailing-whitespace - id: detect-aws-credentials From c1adcd33fe31c6e500305e048719606f88d6efb3 Mon Sep 17 00:00:00 2001 From: aidenvaines-bjss Date: Thu, 24 Jul 2025 16:45:44 +0100 Subject: [PATCH 3/4] CCM-11345 Manual Repo Sync --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 03bf1714..091fd473 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -68,6 +68,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0 # v3.28.9 + uses: github/codeql-action/upload-sarif@fca7ace96b7d713c7035871441bd52efbe39e27e # v3.28.19 with: sarif_file: results.sarif From 31608e8b50b1b15a94d297ad08fe04a36e866458 Mon Sep 17 00:00:00 2001 From: aidenvaines-bjss Date: Fri, 25 Jul 2025 09:29:45 +0100 Subject: [PATCH 4/4] CCM-11345 Manual Repo Sync --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 091fd473..5552785d 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -37,7 +37,7 @@ jobs: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 + uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 with: results_file: results.sarif results_format: sarif